From 1a281ac49dd6b9ee6964badb832918507bf8ba97 Mon Sep 17 00:00:00 2001 From: Jo Date: Mon, 28 Aug 2023 10:49:23 +0200 Subject: [PATCH] Experiment: tag UI devices for anon stats (#73748) * experiment: attempt to tag only UI devices * lint frontend * use await * use shorthand check * do not assume build info exists * do not assume build info exists * Apply suggestions from code review Co-authored-by: Alexander Zobnin --------- Co-authored-by: Alexander Zobnin --- package.json | 1 + pkg/services/anonymous/anonimpl/impl.go | 97 +++++++++++++++++++- pkg/services/anonymous/anonimpl/impl_test.go | 90 ++++++++++++------ pkg/services/anonymous/service.go | 6 +- public/app/core/services/backend_srv.ts | 31 ++++++- yarn.lock | 10 ++ 6 files changed, 201 insertions(+), 34 deletions(-) diff --git a/package.json b/package.json index f19661282a5..c2366f86e61 100644 --- a/package.json +++ b/package.json @@ -232,6 +232,7 @@ "@daybrush/utils": "1.13.0", "@emotion/css": "11.11.2", "@emotion/react": "11.11.1", + "@fingerprintjs/fingerprintjs": "^3.4.2", "@glideapps/glide-data-grid": "^5.2.1", "@grafana/aws-sdk": "0.1.2", "@grafana/data": "workspace:*", diff --git a/pkg/services/anonymous/anonimpl/impl.go b/pkg/services/anonymous/anonimpl/impl.go index f8867643af8..29cdc6dc3ad 100644 --- a/pkg/services/anonymous/anonimpl/impl.go +++ b/pkg/services/anonymous/anonimpl/impl.go @@ -16,10 +16,12 @@ import ( "github.com/grafana/grafana/pkg/infra/remotecache" "github.com/grafana/grafana/pkg/infra/usagestats" "github.com/grafana/grafana/pkg/services/anonymous" + "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/web" ) const thirtyDays = 30 * 24 * time.Hour +const deviceIDHeader = "X-Grafana-Device-Id" type Device struct { Kind anonymous.DeviceKind `json:"kind"` @@ -41,6 +43,10 @@ func (a *Device) Key() (string, error) { return strings.Join([]string{string(a.Kind), hex.EncodeToString(hash.Sum(nil))}, ":"), nil } +func (a *Device) UIKey(deviceID string) (string, error) { + return strings.Join([]string{string(a.Kind), deviceID}, ":"), nil +} + type AnonDeviceService struct { remoteCache remotecache.CacheStorage log log.Logger @@ -70,9 +76,21 @@ func (a *AnonDeviceService) usageStatFn(ctx context.Context) (map[string]interfa return nil, nil } + anonUIDeviceCount, err := a.remoteCache.Count(ctx, string(anonymous.AnonDeviceUI)) + if err != nil { + return nil, nil + } + + authedUIDeviceCount, err := a.remoteCache.Count(ctx, string(anonymous.AuthedDeviceUI)) + if err != nil { + return nil, nil + } + return map[string]interface{}{ - "stats.anonymous.session.count": anonDeviceCount, // keep session for legacy data - "stats.users.device.count": authedDeviceCount, + "stats.anonymous.session.count": anonDeviceCount, // keep session for legacy data + "stats.users.device.count": authedDeviceCount, + "stats.anonymous.device.ui.count": anonUIDeviceCount, + "stats.users.device.ui.count": authedUIDeviceCount, }, nil } @@ -89,6 +107,72 @@ func (a *AnonDeviceService) untagDevice(ctx context.Context, device *Device) err return nil } +func (a *AnonDeviceService) untagUIDevice(ctx context.Context, deviceID string, device *Device) error { + key, err := device.UIKey(deviceID) + if err != nil { + return err + } + + if err := a.remoteCache.Delete(ctx, key); err != nil { + return err + } + + return nil +} + +func (a *AnonDeviceService) tagDeviceUI(ctx context.Context, httpReq *http.Request, device Device) error { + deviceID := httpReq.Header.Get(deviceIDHeader) + if deviceID == "" { + return nil + } + + if device.Kind == anonymous.AnonDevice { + device.Kind = anonymous.AnonDeviceUI + } else if device.Kind == anonymous.AuthedDevice { + device.Kind = anonymous.AuthedDeviceUI + } + + key, err := device.UIKey(deviceID) + if err != nil { + return err + } + + if setting.Env == setting.Dev { + a.log.Debug("tagging device for UI", "deviceID", deviceID, "device", device, "key", key) + } + + if _, ok := a.localCache.Get(key); ok { + return nil + } + + a.localCache.SetDefault(key, struct{}{}) + + deviceJSON, err := json.Marshal(device) + if err != nil { + return err + } + + if err := a.remoteCache.Set(ctx, key, deviceJSON, thirtyDays); err != nil { + return err + } + + // remove existing tag when device switches to another kind + untagKind := anonymous.AnonDeviceUI + if device.Kind == anonymous.AnonDeviceUI { + untagKind = anonymous.AuthedDeviceUI + } + + if err := a.untagUIDevice(ctx, deviceID, &Device{ + Kind: untagKind, + IP: device.IP, + UserAgent: device.UserAgent, + }); err != nil { + return err + } + + return nil +} + func (a *AnonDeviceService) TagDevice(ctx context.Context, httpReq *http.Request, kind anonymous.DeviceKind) error { addr := web.RemoteAddr(httpReq) ip, err := network.GetIPFromAddress(addr) @@ -109,11 +193,20 @@ func (a *AnonDeviceService) TagDevice(ctx context.Context, httpReq *http.Request LastSeen: time.Now().UTC(), } + err = a.tagDeviceUI(ctx, httpReq, *taggedDevice) + if err != nil { + a.log.Debug("failed to tag device for UI", "error", err) + } + key, err := taggedDevice.Key() if err != nil { return err } + if setting.Env == setting.Dev { + a.log.Debug("tagging device", "device", taggedDevice, "key", key) + } + if _, ok := a.localCache.Get(key); ok { return nil } diff --git a/pkg/services/anonymous/anonimpl/impl_test.go b/pkg/services/anonymous/anonimpl/impl_test.go index dd59f6767b4..ec8a0cbd928 100644 --- a/pkg/services/anonymous/anonimpl/impl_test.go +++ b/pkg/services/anonymous/anonimpl/impl_test.go @@ -70,11 +70,13 @@ func TestIntegrationDeviceService_tag(t *testing.T) { kind anonymous.DeviceKind } testCases := []struct { - name string - req []tagReq - expectedAnonCount int64 - expectedAuthedCount int64 - expectedDevice *Device + name string + req []tagReq + expectedAnonCount int64 + expectedAuthedCount int64 + expectedAnonUICount int64 + expectedAuthedUICount int64 + expectedDevice *Device }{ { name: "no requests", @@ -112,73 +114,104 @@ func TestIntegrationDeviceService_tag(t *testing.T) { IP: "10.30.30.1", UserAgent: "test"}, }, + { + name: "should tag device ID once", + req: []tagReq{{httpReq: &http.Request{ + Header: http.Header{ + "User-Agent": []string{"test"}, + "X-Forwarded-For": []string{"10.30.30.1"}, + http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"}, + }, + }, + kind: anonymous.AnonDevice, + }, + }, + expectedAnonUICount: 1, + expectedAuthedUICount: 0, + expectedAnonCount: 1, + expectedAuthedCount: 0, + expectedDevice: &Device{ + Kind: anonymous.AnonDevice, + IP: "10.30.30.1", + UserAgent: "test"}, + }, { name: "repeat request should not tag", req: []tagReq{{httpReq: &http.Request{ Header: http.Header{ - "User-Agent": []string{"test"}, - "X-Forwarded-For": []string{"10.30.30.1"}, + "User-Agent": []string{"test"}, + http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"}, + "X-Forwarded-For": []string{"10.30.30.1"}, }, }, kind: anonymous.AnonDevice, }, {httpReq: &http.Request{ Header: http.Header{ - "User-Agent": []string{"test"}, - "X-Forwarded-For": []string{"10.30.30.1"}, + "User-Agent": []string{"test"}, + http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"}, + "X-Forwarded-For": []string{"10.30.30.1"}, }, }, kind: anonymous.AnonDevice, }, }, expectedAnonCount: 1, + expectedAnonUICount: 1, expectedAuthedCount: 0, }, { name: "authed request should untag anon", req: []tagReq{{httpReq: &http.Request{ Header: http.Header{ - "User-Agent": []string{"test"}, - "X-Forwarded-For": []string{"10.30.30.1"}, + "User-Agent": []string{"test"}, + http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"}, + "X-Forwarded-For": []string{"10.30.30.1"}, }, }, kind: anonymous.AnonDevice, }, {httpReq: &http.Request{ Header: http.Header{ - "User-Agent": []string{"test"}, - "X-Forwarded-For": []string{"10.30.30.1"}, + "User-Agent": []string{"test"}, + http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"}, + "X-Forwarded-For": []string{"10.30.30.1"}, }, }, kind: anonymous.AuthedDevice, }, }, - expectedAnonCount: 0, - expectedAuthedCount: 1, + expectedAnonCount: 0, + expectedAuthedCount: 1, + expectedAuthedUICount: 1, }, { name: "anon request should untag authed", req: []tagReq{{httpReq: &http.Request{ Header: http.Header{ - "User-Agent": []string{"test"}, - "X-Forwarded-For": []string{"10.30.30.1"}, + "User-Agent": []string{"test"}, + http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"}, + "X-Forwarded-For": []string{"10.30.30.1"}, }, }, kind: anonymous.AuthedDevice, }, {httpReq: &http.Request{ Header: http.Header{ - "User-Agent": []string{"test"}, - "X-Forwarded-For": []string{"10.30.30.1"}, + "User-Agent": []string{"test"}, + http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"}, + "X-Forwarded-For": []string{"10.30.30.1"}, }, }, kind: anonymous.AnonDevice, }, }, expectedAnonCount: 1, + expectedAnonUICount: 1, expectedAuthedCount: 0, }, { - name: "tag 4 different requests", + name: "tag 4 different requests - 2 are UI", req: []tagReq{{httpReq: &http.Request{ Header: http.Header{ - "User-Agent": []string{"test"}, - "X-Forwarded-For": []string{"10.30.30.1"}, + http.CanonicalHeaderKey("User-Agent"): []string{"test"}, + http.CanonicalHeaderKey("X-Forwarded-For"): []string{"10.30.30.1"}, + http.CanonicalHeaderKey(deviceIDHeader): []string{"a"}, }, }, kind: anonymous.AnonDevice, @@ -191,8 +224,9 @@ func TestIntegrationDeviceService_tag(t *testing.T) { kind: anonymous.AnonDevice, }, {httpReq: &http.Request{ Header: http.Header{ - "User-Agent": []string{"test"}, - "X-Forwarded-For": []string{"10.30.30.3"}, + "User-Agent": []string{"test"}, + "X-Forwarded-For": []string{"10.30.30.3"}, + http.CanonicalHeaderKey(deviceIDHeader): []string{"c"}, }, }, kind: anonymous.AuthedDevice, @@ -205,8 +239,10 @@ func TestIntegrationDeviceService_tag(t *testing.T) { kind: anonymous.AuthedDevice, }, }, - expectedAnonCount: 2, - expectedAuthedCount: 2, + expectedAnonCount: 2, + expectedAuthedCount: 2, + expectedAnonUICount: 1, + expectedAuthedUICount: 1, }, } @@ -226,6 +262,8 @@ func TestIntegrationDeviceService_tag(t *testing.T) { assert.Equal(t, tc.expectedAnonCount, stats["stats.anonymous.session.count"].(int64)) assert.Equal(t, tc.expectedAuthedCount, stats["stats.users.device.count"].(int64)) + assert.Equal(t, tc.expectedAnonUICount, stats["stats.anonymous.device.ui.count"].(int64)) + assert.Equal(t, tc.expectedAuthedUICount, stats["stats.users.device.ui.count"].(int64)) if tc.expectedDevice != nil { key, err := tc.expectedDevice.Key() diff --git a/pkg/services/anonymous/service.go b/pkg/services/anonymous/service.go index 5100800ce26..8a60c240bae 100644 --- a/pkg/services/anonymous/service.go +++ b/pkg/services/anonymous/service.go @@ -8,8 +8,10 @@ import ( type DeviceKind string const ( - AnonDevice DeviceKind = "anon-session" - AuthedDevice DeviceKind = "authed-session" + AnonDevice DeviceKind = "anon-session" + AuthedDevice DeviceKind = "authed-session" + AnonDeviceUI DeviceKind = "ui-anon-session" + AuthedDeviceUI DeviceKind = "ui-authed-session" ) type Service interface { diff --git a/public/app/core/services/backend_srv.ts b/public/app/core/services/backend_srv.ts index 594bc15346e..994e9613815 100644 --- a/public/app/core/services/backend_srv.ts +++ b/public/app/core/services/backend_srv.ts @@ -1,3 +1,4 @@ +import FingerprintJS from '@fingerprintjs/fingerprintjs'; import { from, lastValueFrom, MonoTypeOperatorFunction, Observable, Subject, Subscription, throwError } from 'rxjs'; import { fromFetch } from 'rxjs/fetch'; import { @@ -15,6 +16,7 @@ import { import { v4 as uuidv4 } from 'uuid'; import { AppEvents, DataQueryErrorType } from '@grafana/data'; +import { GrafanaEdition } from '@grafana/data/src/types/config'; import { BackendSrv as BackendService, BackendSrvRequest, config, FetchError, FetchResponse } from '@grafana/runtime'; import appEvents from 'app/core/app_events'; import { getConfig } from 'app/core/config'; @@ -61,6 +63,7 @@ export class BackendSrv implements BackendService { private readonly fetchQueue: FetchQueue; private readonly responseQueue: ResponseQueue; private _tokenRotationInProgress?: Observable | null = null; + private deviceID?: string | null = null; private dependencies: BackendSrvDependencies = { fromFetch: fromFetch, @@ -83,9 +86,26 @@ export class BackendSrv implements BackendService { this.internalFetch = this.internalFetch.bind(this); this.fetchQueue = new FetchQueue(); this.responseQueue = new ResponseQueue(this.fetchQueue, this.internalFetch); + + this.initGrafanaDeviceID(); + new FetchQueueWorker(this.fetchQueue, this.responseQueue, getConfig()); } + private async initGrafanaDeviceID() { + if (config.buildInfo?.edition === GrafanaEdition.OpenSource) { + return; + } + + try { + const fp = await FingerprintJS.load(); + const result = await fp.get(); + this.deviceID = result.visitorId; + } catch (error) { + console.error(error); + } + } + async request(options: BackendSrvRequest): Promise { return await lastValueFrom(this.fetch(options).pipe(map((response: FetchResponse) => response.data))); } @@ -134,15 +154,18 @@ export class BackendSrv implements BackendService { const token = loadUrlToken(); if (token !== null && token !== '') { - if (!options.headers) { - options.headers = {}; - } - if (config.jwtUrlLogin && config.jwtHeaderName) { + options.headers = options.headers ?? {}; options.headers[config.jwtHeaderName] = `${token}`; } } + // Add device id header if not OSS build + if (config.buildInfo?.edition !== GrafanaEdition.OpenSource && this.deviceID) { + options.headers = options.headers ?? {}; + options.headers['X-Grafana-Device-Id'] = `${this.deviceID}`; + } + return this.getFromFetchStream(options).pipe( this.handleStreamResponse(options), this.handleStreamError(options), diff --git a/yarn.lock b/yarn.lock index 6c0c75ae464..c0806de5d28 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3477,6 +3477,15 @@ __metadata: languageName: node linkType: hard +"@fingerprintjs/fingerprintjs@npm:^3.4.2": + version: 3.4.2 + resolution: "@fingerprintjs/fingerprintjs@npm:3.4.2" + dependencies: + tslib: ^2.4.1 + checksum: 3b9dc81e4186f1aaa39e208c17939f5747bf9a8eb1c8175264a352e46e263abd81bcf89439240bd1a4755d7e3dfb4a83164e294f940abc290e7f2076d3b603ce + languageName: node + linkType: hard + "@floating-ui/core@npm:^1.0.1": version: 1.0.1 resolution: "@floating-ui/core@npm:1.0.1" @@ -19234,6 +19243,7 @@ __metadata: "@emotion/css": 11.11.2 "@emotion/eslint-plugin": 11.11.0 "@emotion/react": 11.11.1 + "@fingerprintjs/fingerprintjs": ^3.4.2 "@glideapps/glide-data-grid": ^5.2.1 "@grafana/aws-sdk": 0.1.2 "@grafana/data": "workspace:*"