IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Prometheus: Migrate metadata queries to use resource calls (#49921)

Browse Source 
* Sent resource calls for metadata to the backend

* moved resource calls to the backend

* code review feedback

* fixed post with body

* statuscode >= 300

* cleanup

* fixed tests

* fixed datasource tests

* code review feedback

* force some other endpoints to only GET

* fix linting errors

* fixed tests

* was able to remove section of redundant code

* cleanup and code review feedback

* moved query_exemplars to get request

* fixed return on error

* went back to resource calls, but using the backendsrv directly

* moved to a resource call with fallback

* fixed tests

* check for proper messages

* proper check for invalid calls

* code review changes
This commit is contained in:
Stephanie Closson 2022-06-03 15:56:13 -03:00 committed by GitHub
parent d2f3631a47
commit 53ee72d15d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 181 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
pkg/tsdb/prometheus/client/client.go
View File

@ -42,7 +42,7 @@ func (c *Client) QueryRange(ctx context.Context, q *models.Query) (*http.Respons
qs.Set("end", formatTime(tr.End))
qs.Set("step", strconv.FormatFloat(tr.Step.Seconds(), 'f', -1, 64))
return c.fetch(ctx, u, qs)
return c.fetch(ctx, c.method, u, qs)
}
func (c *Client) QueryInstant(ctx context.Context, q *models.Query) (*http.Response, error) {
@ -60,7 +60,7 @@ func (c *Client) QueryInstant(ctx context.Context, q *models.Query) (*http.Respo
qs.Set("time", formatTime(tr.End))
}
return c.fetch(ctx, u, qs)
return c.fetch(ctx, c.method, u, qs)
}
func (c *Client) QueryExemplars(ctx context.Context, q *models.Query) (*http.Response, error) {
@ -77,20 +77,31 @@ func (c *Client) QueryExemplars(ctx context.Context, q *models.Query) (*http.Res
qs.Set("start", formatTime(tr.Start))
qs.Set("end", formatTime(tr.End))
return c.fetch(ctx, u, qs)
return c.fetch(ctx, c.method, u, qs)
}
func (c *Client) fetch(ctx context.Context, u *url.URL, qs url.Values) (*http.Response, error) {
if strings.ToUpper(c.method) == http.MethodGet {
u.RawQuery = qs.Encode()
}
r, err := http.NewRequestWithContext(ctx, c.method, u.String(), nil)
func (c *Client) QueryResource(ctx context.Context, method string, p string, qs url.Values) (*http.Response, error) {
u, err := url.ParseRequestURI(c.baseUrl)
if err != nil {
return nil, err
}
if strings.ToUpper(c.method) == http.MethodPost {
u.Path = path.Join(u.Path, p)
return c.fetch(ctx, method, u, qs)
}
func (c *Client) fetch(ctx context.Context, method string, u *url.URL, qs url.Values) (*http.Response, error) {
if strings.ToUpper(method) == http.MethodGet {
u.RawQuery = qs.Encode()
}
r, err := http.NewRequestWithContext(ctx, method, u.String(), nil)
if err != nil {
return nil, err
}
if strings.ToUpper(method) == http.MethodPost {
r.Body = ioutil.NopCloser(strings.NewReader(qs.Encode()))
r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
}

29
pkg/tsdb/prometheus/prometheus.go
View File

@ -16,6 +16,7 @@ import (
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb/prometheus/buffered"
"github.com/grafana/grafana/pkg/tsdb/prometheus/querydata"
"github.com/grafana/grafana/pkg/tsdb/prometheus/resource"
apiv1 "github.com/prometheus/client_golang/api/prometheus/v1"
)
@ -29,6 +30,7 @@ type Service struct {
type instance struct {
buffered *buffered.Buffered
queryData *querydata.QueryData
resource *resource.Resource
}
func ProvideService(httpClientProvider httpclient.Provider, cfg *setting.Cfg, features featuremgmt.FeatureToggles, tracer tracing.Tracer) *Service {
@ -57,9 +59,15 @@ func newInstanceSettings(httpClientProvider httpclient.Provider, cfg *setting.Cf
return nil, err
}
r, err := resource.New(httpClientProvider, cfg, features, settings, plog)
if err != nil {
return nil, err
}
return instance{
buffered: b,
queryData: qd,
resource: r,
}, nil
}
}
@ -81,6 +89,27 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
return i.buffered.ExecuteTimeSeriesQuery(ctx, req)
}
func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error {
i, err := s.getInstance(req.PluginContext)
if err != nil {
return err
}
statusCode, bytes, err := i.resource.Execute(ctx, req)
body := bytes
if err != nil {
body = []byte(err.Error())
}
return sender.Send(&backend.CallResourceResponse{
Status: statusCode,
Headers: map[string][]string{
"content-type": {"application/json"},
},
Body: body,
})
}
func (s *Service) getInstance(pluginCtx backend.PluginContext) (*instance, error) {
i, err := s.im.Get(pluginCtx)
if err != nil {

85
pkg/tsdb/prometheus/resource/resource.go Normal file
View File

@ -0,0 +1,85 @@
package resource
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/url"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tsdb/prometheus/client"
)
type Resource struct {
provider *client.Provider
log log.Logger
}
func New(
httpClientProvider httpclient.Provider,
cfg *setting.Cfg,
features featuremgmt.FeatureToggles,
settings backend.DataSourceInstanceSettings,
plog log.Logger,
) (*Resource, error) {
var jsonData map[string]interface{}
if err := json.Unmarshal(settings.JSONData, &jsonData); err != nil {
return nil, fmt.Errorf("error reading settings: %w", err)
}
p := client.NewProvider(settings, jsonData, httpClientProvider, cfg, features, plog)
return &Resource{
log: plog,
provider: p,
}, nil
}
func (r *Resource) Execute(ctx context.Context, req *backend.CallResourceRequest) (int, []byte, error) {
client, err := r.provider.GetClient(reqHeaders(req.Headers))
if err != nil {
return 500, nil, err
}
return r.fetch(ctx, client, req)
}
func (r *Resource) fetch(ctx context.Context, client *client.Client, req *backend.CallResourceRequest) (int, []byte, error) {
r.log.Debug("Sending resource query", "URL", req.URL)
u, err := url.Parse(req.URL)
if err != nil {
return 500, nil, err
}
resp, err := client.QueryResource(ctx, req.Method, u.Path, u.Query())
if err != nil {
return resp.StatusCode, nil, err
}
defer resp.Body.Close() //nolint (we don't care about the error being returned by resp.Body.Close())
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return 500, nil, err
}
return resp.StatusCode, data, err
}
func reqHeaders(headers map[string][]string) map[string]string {
// Keep only the authorization header, incase downstream the authorization header is required.
// Strip all the others out as appropriate headers will be applied to speak with prometheus.
h := make(map[string]string)
accessValues := headers["Authorization"]
if len(accessValues) > 0 {
h["Authorization"] = accessValues[0]
}
return h
}

11
public/app/plugins/datasource/prometheus/datasource.test.ts
View File

@ -59,6 +59,7 @@ describe('PrometheusDatasource', () => {
let ds: PrometheusDatasource;
const instanceSettings = {
url: 'proxied',
id: 1,
directUrl: 'direct',
user: 'test',
password: 'mupp',
@ -149,7 +150,7 @@ describe('PrometheusDatasource', () => {
it('added to metadata request', () => {
promDs.metadataRequest('/foo');
expect(fetchMock.mock.calls.length).toBe(1);
expect(fetchMock.mock.calls[0][0].url).toBe('proxied/foo?customQuery=123');
expect(fetchMock.mock.calls[0][0].url).toBe('/api/datasources/1/resources/foo?customQuery=123');
});
it('adds params to timeseries query', () => {
@ -184,13 +185,13 @@ describe('PrometheusDatasource', () => {
it('added to metadata request with non-POST endpoint', () => {
promDs.metadataRequest('/foo');
expect(fetchMock.mock.calls.length).toBe(1);
expect(fetchMock.mock.calls[0][0].url).toBe('proxied/foo?customQuery=123');
expect(fetchMock.mock.calls[0][0].url).toBe('/api/datasources/1/resources/foo?customQuery=123');
});
it('added to metadata request with POST endpoint', () => {
promDs.metadataRequest('/api/v1/labels');
expect(fetchMock.mock.calls.length).toBe(1);
expect(fetchMock.mock.calls[0][0].url).toBe('proxied/api/v1/labels');
expect(fetchMock.mock.calls[0][0].url).toBe('/api/datasources/1/resources/api/v1/labels');
expect(fetchMock.mock.calls[0][0].data.customQuery).toBe('123');
});
@ -431,7 +432,7 @@ describe('PrometheusDatasource', () => {
});
});
describe('Prometheus regular escaping', () => {
describe('Prometheus regular escaping', () => {
it('should not escape non-string', () => {
expect(prometheusRegularEscape(12)).toEqual(12);
});
@ -457,7 +458,7 @@ describe('PrometheusDatasource', () => {
});
});
describe('Prometheus regexes escaping', () => {
describe('Prometheus regexes escaping', () => {
it('should not escape simple string', () => {
expect(prometheusSpecialRegexEscape('cryptodepression')).toEqual('cryptodepression');
});

33
public/app/plugins/datasource/prometheus/datasource.tsx
View File

@ -109,7 +109,7 @@ export class PrometheusDatasource
this.withCredentials = instanceSettings.withCredentials;
this.interval = instanceSettings.jsonData.timeInterval || '15s';
this.queryTimeout = instanceSettings.jsonData.queryTimeout;
this.httpMethod = instanceSettings.jsonData.httpMethod || 'POST';
this.httpMethod = instanceSettings.jsonData.httpMethod || 'GET';
// `directUrl` is never undefined, we set it at https://github.com/grafana/grafana/blob/main/pkg/api/frontendsettings.go#L108
// here we "fall back" to this.url to make typescript happy, but it should never happen
this.directUrl = instanceSettings.jsonData.directUrl ?? this.url;
@ -165,8 +165,14 @@ export class PrometheusDatasource
}
}
let queryUrl = this.url + url;
if (url.startsWith(`/api/datasources/${this.id}`)) {
// This url is meant to be a replacement for the whole URL. Replace the entire URL
queryUrl = url;
}
const options: BackendSrvRequest = defaults(overrides, {
url: this.url + url,
url: queryUrl,
method: this.httpMethod,
headers: {},
});
@ -209,10 +215,16 @@ export class PrometheusDatasource
// If URL includes endpoint that supports POST and GET method, try to use configured method. This might fail as POST is supported only in v2.10+.
if (GET_AND_POST_METADATA_ENDPOINTS.some((endpoint) => url.includes(endpoint))) {
try {
return await lastValueFrom(this._request<T>(url, params, { method: this.httpMethod, hideFromInspector: true }));
return await lastValueFrom(
this._request<T>(`/api/datasources/${this.id}/resources${url}`, params, {
method: this.httpMethod,
hideFromInspector: true,
showErrorAlert: false,
})
);
} catch (err) {
// If status code of error is Method Not Allowed (405) and HTTP method is POST, retry with GET
if (this.httpMethod === 'POST' && err.status === 405) {
if (this.httpMethod === 'POST' && (err.status === 405 || err.status === 400)) {
console.warn(`Couldn't use configured POST HTTP method for this request. Trying to use GET method instead.`);
} else {
throw err;
@ -220,7 +232,12 @@ export class PrometheusDatasource
}
}
return await lastValueFrom(this._request<T>(url, params, { method: 'GET', hideFromInspector: true })); // toPromise until we change getTagValues, getTagKeys to Observable
return await lastValueFrom(
this._request<T>(`/api/datasources/${this.id}/resources${url}`, params, {
method: 'GET',
hideFromInspector: true,
})
); // toPromise until we change getTagValues, getTagKeys to Observable
}
interpolateQueryExpr(value: string | string[] = [], variable: any) {
@ -995,7 +1012,11 @@ export class PrometheusDatasource
async areExemplarsAvailable() {
try {
const res = await this.metadataRequest('/api/v1/query_exemplars', { query: 'test' });
const res = await this.getResource('/api/v1/query_exemplars', {
query: 'test',
start: dateTime().subtract(30, 'minutes').valueOf(),
end: dateTime().valueOf(),
});
if (res.data.status === 'success') {
return true;
}

20
public/app/plugins/datasource/prometheus/metric_find_query.test.ts
View File

@ -18,6 +18,7 @@ const fetchMock = jest.spyOn(backendSrv, 'fetch');
const instanceSettings = {
url: 'proxied',
id: 1,
directUrl: 'direct',
user: 'test',
password: 'mupp',
@ -75,8 +76,9 @@ describe('PrometheusMetricFindQuery', () => {
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(fetchMock).toHaveBeenCalledWith({
method: 'GET',
url: `proxied/api/v1/labels?start=${raw.from.unix()}&end=${raw.to.unix()}`,
url: `/api/datasources/1/resources/api/v1/labels?start=${raw.from.unix()}&end=${raw.to.unix()}`,
hideFromInspector: true,
showErrorAlert: false,
headers: {},
});
});
@ -94,7 +96,7 @@ describe('PrometheusMetricFindQuery', () => {
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(fetchMock).toHaveBeenCalledWith({
method: 'GET',
url: `proxied/api/v1/label/resource/values?start=${raw.from.unix()}&end=${raw.to.unix()}`,
url: `/api/datasources/1/resources/api/v1/label/resource/values?start=${raw.from.unix()}&end=${raw.to.unix()}`,
hideFromInspector: true,
headers: {},
});
@ -117,10 +119,11 @@ describe('PrometheusMetricFindQuery', () => {
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(fetchMock).toHaveBeenCalledWith({
method: 'GET',
url: `proxied/api/v1/series?match${encodeURIComponent(
url: `/api/datasources/1/resources/api/v1/series?match${encodeURIComponent(
'[]'
)}=metric&start=${raw.from.unix()}&end=${raw.to.unix()}`,
hideFromInspector: true,
showErrorAlert: false,
headers: {},
});
});
@ -142,8 +145,9 @@ describe('PrometheusMetricFindQuery', () => {
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(fetchMock).toHaveBeenCalledWith({
method: 'GET',
url: 'proxied/api/v1/series?match%5B%5D=metric%7Blabel1%3D%22foo%22%2C%20label2%3D%22bar%22%2C%20label3%3D%22baz%22%7D&start=1524650400&end=1524654000',
url: '/api/datasources/1/resources/api/v1/series?match%5B%5D=metric%7Blabel1%3D%22foo%22%2C%20label2%3D%22bar%22%2C%20label3%3D%22baz%22%7D&start=1524650400&end=1524654000',
hideFromInspector: true,
showErrorAlert: false,
headers: {},
});
});
@ -167,10 +171,11 @@ describe('PrometheusMetricFindQuery', () => {
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(fetchMock).toHaveBeenCalledWith({
method: 'GET',
url: `proxied/api/v1/series?match${encodeURIComponent(
url: `/api/datasources/1/resources/api/v1/series?match${encodeURIComponent(
'[]'
)}=metric&start=${raw.from.unix()}&end=${raw.to.unix()}`,
hideFromInspector: true,
showErrorAlert: false,
headers: {},
});
});
@ -188,7 +193,7 @@ describe('PrometheusMetricFindQuery', () => {
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(fetchMock).toHaveBeenCalledWith({
method: 'GET',
url: `proxied/api/v1/label/__name__/values?start=${raw.from.unix()}&end=${raw.to.unix()}`,
url: `/api/datasources/1/resources/api/v1/label/__name__/values?start=${raw.from.unix()}&end=${raw.to.unix()}`,
hideFromInspector: true,
headers: {},
});
@ -242,10 +247,11 @@ describe('PrometheusMetricFindQuery', () => {
expect(fetchMock).toHaveBeenCalledTimes(1);
expect(fetchMock).toHaveBeenCalledWith({
method: 'GET',
url: `proxied/api/v1/series?match${encodeURIComponent('[]')}=${encodeURIComponent(
url: `/api/datasources/1/resources/api/v1/series?match${encodeURIComponent('[]')}=${encodeURIComponent(
'up{job="job1"}'
)}&start=${raw.from.unix()}&end=${raw.to.unix()}`,
hideFromInspector: true,
showErrorAlert: false,
headers: {},
});
});