IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Experiment: tag UI devices for anon stats (#73748)

Browse Source 
* experiment: attempt to tag only UI devices

* lint frontend

* use await

* use shorthand check

* do not assume build info exists

* do not assume build info exists

* Apply suggestions from code review

Co-authored-by: Alexander Zobnin <alexanderzobnin@gmail.com>

---------

Co-authored-by: Alexander Zobnin <alexanderzobnin@gmail.com>
This commit is contained in:
Jo 2023-08-28 10:49:23 +02:00 committed by GitHub
parent 5eed495cce
commit 1a281ac49d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 201 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -232,6 +232,7 @@
"@daybrush/utils": "1.13.0",
"@emotion/css": "11.11.2",
"@emotion/react": "11.11.1",
"@fingerprintjs/fingerprintjs": "^3.4.2",
"@glideapps/glide-data-grid": "^5.2.1",
"@grafana/aws-sdk": "0.1.2",
"@grafana/data": "workspace:*",

93
pkg/services/anonymous/anonimpl/impl.go
View File

@ -16,10 +16,12 @@ import (
"github.com/grafana/grafana/pkg/infra/remotecache"
"github.com/grafana/grafana/pkg/infra/usagestats"
"github.com/grafana/grafana/pkg/services/anonymous"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/web"
)
const thirtyDays = 30 * 24 * time.Hour
const deviceIDHeader = "X-Grafana-Device-Id"
type Device struct {
Kind anonymous.DeviceKind `json:"kind"`
@ -41,6 +43,10 @@ func (a *Device) Key() (string, error) {
return strings.Join([]string{string(a.Kind), hex.EncodeToString(hash.Sum(nil))}, ":"), nil
}
func (a *Device) UIKey(deviceID string) (string, error) {
return strings.Join([]string{string(a.Kind), deviceID}, ":"), nil
}
type AnonDeviceService struct {
remoteCache remotecache.CacheStorage
log log.Logger
@ -70,9 +76,21 @@ func (a *AnonDeviceService) usageStatFn(ctx context.Context) (map[string]interfa
return nil, nil
}
anonUIDeviceCount, err := a.remoteCache.Count(ctx, string(anonymous.AnonDeviceUI))
if err != nil {
return nil, nil
}
authedUIDeviceCount, err := a.remoteCache.Count(ctx, string(anonymous.AuthedDeviceUI))
if err != nil {
return nil, nil
}
return map[string]interface{}{
"stats.anonymous.session.count": anonDeviceCount, // keep session for legacy data
"stats.users.device.count": authedDeviceCount,
"stats.anonymous.device.ui.count": anonUIDeviceCount,
"stats.users.device.ui.count": authedUIDeviceCount,
}, nil
}
@ -89,6 +107,72 @@ func (a *AnonDeviceService) untagDevice(ctx context.Context, device *Device) err
return nil
}
func (a *AnonDeviceService) untagUIDevice(ctx context.Context, deviceID string, device *Device) error {
key, err := device.UIKey(deviceID)
if err != nil {
return err
}
if err := a.remoteCache.Delete(ctx, key); err != nil {
return err
}
return nil
}
func (a *AnonDeviceService) tagDeviceUI(ctx context.Context, httpReq *http.Request, device Device) error {
deviceID := httpReq.Header.Get(deviceIDHeader)
if deviceID == "" {
return nil
}
if device.Kind == anonymous.AnonDevice {
device.Kind = anonymous.AnonDeviceUI
} else if device.Kind == anonymous.AuthedDevice {
device.Kind = anonymous.AuthedDeviceUI
}
key, err := device.UIKey(deviceID)
if err != nil {
return err
}
if setting.Env == setting.Dev {
a.log.Debug("tagging device for UI", "deviceID", deviceID, "device", device, "key", key)
}
if _, ok := a.localCache.Get(key); ok {
return nil
}
a.localCache.SetDefault(key, struct{}{})
deviceJSON, err := json.Marshal(device)
if err != nil {
return err
}
if err := a.remoteCache.Set(ctx, key, deviceJSON, thirtyDays); err != nil {
return err
}
// remove existing tag when device switches to another kind
untagKind := anonymous.AnonDeviceUI
if device.Kind == anonymous.AnonDeviceUI {
untagKind = anonymous.AuthedDeviceUI
}
if err := a.untagUIDevice(ctx, deviceID, &Device{
Kind: untagKind,
IP: device.IP,
UserAgent: device.UserAgent,
}); err != nil {
return err
}
return nil
}
func (a *AnonDeviceService) TagDevice(ctx context.Context, httpReq *http.Request, kind anonymous.DeviceKind) error {
addr := web.RemoteAddr(httpReq)
ip, err := network.GetIPFromAddress(addr)
@ -109,11 +193,20 @@ func (a *AnonDeviceService) TagDevice(ctx context.Context, httpReq *http.Request
LastSeen: time.Now().UTC(),
}
err = a.tagDeviceUI(ctx, httpReq, *taggedDevice)
if err != nil {
a.log.Debug("failed to tag device for UI", "error", err)
}
key, err := taggedDevice.Key()
if err != nil {
return err
}
if setting.Env == setting.Dev {
a.log.Debug("tagging device", "device", taggedDevice, "key", key)
}
if _, ok := a.localCache.Get(key); ok {
return nil
}

44
pkg/services/anonymous/anonimpl/impl_test.go
View File

@ -74,6 +74,8 @@ func TestIntegrationDeviceService_tag(t *testing.T) {
req []tagReq
expectedAnonCount int64
expectedAuthedCount int64
expectedAnonUICount int64
expectedAuthedUICount int64
expectedDevice *Device
}{
{
@ -112,11 +114,33 @@ func TestIntegrationDeviceService_tag(t *testing.T) {
IP: "10.30.30.1",
UserAgent: "test"},
},
{
name: "should tag device ID once",
req: []tagReq{{httpReq: &http.Request{
Header: http.Header{
"User-Agent": []string{"test"},
"X-Forwarded-For": []string{"10.30.30.1"},
http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"},
},
},
kind: anonymous.AnonDevice,
},
},
expectedAnonUICount: 1,
expectedAuthedUICount: 0,
expectedAnonCount: 1,
expectedAuthedCount: 0,
expectedDevice: &Device{
Kind: anonymous.AnonDevice,
IP: "10.30.30.1",
UserAgent: "test"},
},
{
name: "repeat request should not tag",
req: []tagReq{{httpReq: &http.Request{
Header: http.Header{
"User-Agent": []string{"test"},
http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"},
"X-Forwarded-For": []string{"10.30.30.1"},
},
},
@ -124,6 +148,7 @@ func TestIntegrationDeviceService_tag(t *testing.T) {
}, {httpReq: &http.Request{
Header: http.Header{
"User-Agent": []string{"test"},
http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"},
"X-Forwarded-For": []string{"10.30.30.1"},
},
},
@ -131,12 +156,14 @@ func TestIntegrationDeviceService_tag(t *testing.T) {
},
},
expectedAnonCount: 1,
expectedAnonUICount: 1,
expectedAuthedCount: 0,
}, {
name: "authed request should untag anon",
req: []tagReq{{httpReq: &http.Request{
Header: http.Header{
"User-Agent": []string{"test"},
http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"},
"X-Forwarded-For": []string{"10.30.30.1"},
},
},
@ -144,6 +171,7 @@ func TestIntegrationDeviceService_tag(t *testing.T) {
}, {httpReq: &http.Request{
Header: http.Header{
"User-Agent": []string{"test"},
http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"},
"X-Forwarded-For": []string{"10.30.30.1"},
},
},
@ -152,11 +180,13 @@ func TestIntegrationDeviceService_tag(t *testing.T) {
},
expectedAnonCount: 0,
expectedAuthedCount: 1,
expectedAuthedUICount: 1,
}, {
name: "anon request should untag authed",
req: []tagReq{{httpReq: &http.Request{
Header: http.Header{
"User-Agent": []string{"test"},
http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"},
"X-Forwarded-For": []string{"10.30.30.1"},
},
},
@ -164,6 +194,7 @@ func TestIntegrationDeviceService_tag(t *testing.T) {
}, {httpReq: &http.Request{
Header: http.Header{
"User-Agent": []string{"test"},
http.CanonicalHeaderKey(deviceIDHeader): []string{"32mdo31deeqwes"},
"X-Forwarded-For": []string{"10.30.30.1"},
},
},
@ -171,14 +202,16 @@ func TestIntegrationDeviceService_tag(t *testing.T) {
},
},
expectedAnonCount: 1,
expectedAnonUICount: 1,
expectedAuthedCount: 0,
},
{
name: "tag 4 different requests",
name: "tag 4 different requests - 2 are UI",
req: []tagReq{{httpReq: &http.Request{
Header: http.Header{
"User-Agent": []string{"test"},
"X-Forwarded-For": []string{"10.30.30.1"},
http.CanonicalHeaderKey("User-Agent"): []string{"test"},
http.CanonicalHeaderKey("X-Forwarded-For"): []string{"10.30.30.1"},
http.CanonicalHeaderKey(deviceIDHeader): []string{"a"},
},
},
kind: anonymous.AnonDevice,
@ -193,6 +226,7 @@ func TestIntegrationDeviceService_tag(t *testing.T) {
Header: http.Header{
"User-Agent": []string{"test"},
"X-Forwarded-For": []string{"10.30.30.3"},
http.CanonicalHeaderKey(deviceIDHeader): []string{"c"},
},
},
kind: anonymous.AuthedDevice,
@ -207,6 +241,8 @@ func TestIntegrationDeviceService_tag(t *testing.T) {
},
expectedAnonCount: 2,
expectedAuthedCount: 2,
expectedAnonUICount: 1,
expectedAuthedUICount: 1,
},
}
@ -226,6 +262,8 @@ func TestIntegrationDeviceService_tag(t *testing.T) {
assert.Equal(t, tc.expectedAnonCount, stats["stats.anonymous.session.count"].(int64))
assert.Equal(t, tc.expectedAuthedCount, stats["stats.users.device.count"].(int64))
assert.Equal(t, tc.expectedAnonUICount, stats["stats.anonymous.device.ui.count"].(int64))
assert.Equal(t, tc.expectedAuthedUICount, stats["stats.users.device.ui.count"].(int64))
if tc.expectedDevice != nil {
key, err := tc.expectedDevice.Key()

2
pkg/services/anonymous/service.go
View File

@ -10,6 +10,8 @@ type DeviceKind string
const (
AnonDevice DeviceKind = "anon-session"
AuthedDevice DeviceKind = "authed-session"
AnonDeviceUI DeviceKind = "ui-anon-session"
AuthedDeviceUI DeviceKind = "ui-authed-session"
)
type Service interface {

31
public/app/core/services/backend_srv.ts
View File

@ -1,3 +1,4 @@
import FingerprintJS from '@fingerprintjs/fingerprintjs';
import { from, lastValueFrom, MonoTypeOperatorFunction, Observable, Subject, Subscription, throwError } from 'rxjs';
import { fromFetch } from 'rxjs/fetch';
import {
@ -15,6 +16,7 @@ import {
import { v4 as uuidv4 } from 'uuid';
import { AppEvents, DataQueryErrorType } from '@grafana/data';
import { GrafanaEdition } from '@grafana/data/src/types/config';
import { BackendSrv as BackendService, BackendSrvRequest, config, FetchError, FetchResponse } from '@grafana/runtime';
import appEvents from 'app/core/app_events';
import { getConfig } from 'app/core/config';
@ -61,6 +63,7 @@ export class BackendSrv implements BackendService {
private readonly fetchQueue: FetchQueue;
private readonly responseQueue: ResponseQueue;
private _tokenRotationInProgress?: Observable<FetchResponse> | null = null;
private deviceID?: string | null = null;
private dependencies: BackendSrvDependencies = {
fromFetch: fromFetch,
@ -83,9 +86,26 @@ export class BackendSrv implements BackendService {
this.internalFetch = this.internalFetch.bind(this);
this.fetchQueue = new FetchQueue();
this.responseQueue = new ResponseQueue(this.fetchQueue, this.internalFetch);
this.initGrafanaDeviceID();
new FetchQueueWorker(this.fetchQueue, this.responseQueue, getConfig());
}
private async initGrafanaDeviceID() {
if (config.buildInfo?.edition === GrafanaEdition.OpenSource) {
return;
}
try {
const fp = await FingerprintJS.load();
const result = await fp.get();
this.deviceID = result.visitorId;
} catch (error) {
console.error(error);
}
}
async request<T = any>(options: BackendSrvRequest): Promise<T> {
return await lastValueFrom(this.fetch<T>(options).pipe(map((response: FetchResponse<T>) => response.data)));
}
@ -134,15 +154,18 @@ export class BackendSrv implements BackendService {
const token = loadUrlToken();
if (token !== null && token !== '') {
if (!options.headers) {
options.headers = {};
}
if (config.jwtUrlLogin && config.jwtHeaderName) {
options.headers = options.headers ?? {};
options.headers[config.jwtHeaderName] = `${token}`;
}
}
// Add device id header if not OSS build
if (config.buildInfo?.edition !== GrafanaEdition.OpenSource && this.deviceID) {
options.headers = options.headers ?? {};
options.headers['X-Grafana-Device-Id'] = `${this.deviceID}`;
}
return this.getFromFetchStream<T>(options).pipe(
this.handleStreamResponse<T>(options),
this.handleStreamError(options),

10
yarn.lock
View File

@ -3477,6 +3477,15 @@ __metadata:
languageName: node
linkType: hard
"@fingerprintjs/fingerprintjs@npm:^3.4.2":
version: 3.4.2
resolution: "@fingerprintjs/fingerprintjs@npm:3.4.2"
dependencies:
tslib: ^2.4.1
checksum: 3b9dc81e4186f1aaa39e208c17939f5747bf9a8eb1c8175264a352e46e263abd81bcf89439240bd1a4755d7e3dfb4a83164e294f940abc290e7f2076d3b603ce
languageName: node
linkType: hard
"@floating-ui/core@npm:^1.0.1":
version: 1.0.1
resolution: "@floating-ui/core@npm:1.0.1"
@ -19234,6 +19243,7 @@ __metadata:
"@emotion/css": 11.11.2
"@emotion/eslint-plugin": 11.11.0
"@emotion/react": 11.11.1
"@fingerprintjs/fingerprintjs": ^3.4.2
"@glideapps/glide-data-grid": ^5.2.1
"@grafana/aws-sdk": 0.1.2
"@grafana/data": "workspace:*"