AzureMonitor: Automate retrieval of supported Metrics namespaces (#53203)

* Automate retrieval of supported Metrics namespaces

- Pass AzMonitor datasource to resource picker
- Add function to fetch unique namespaces for all subscriptions
- Update tests

* Remove unused file and reference

* Remove final namespace reference

* Add missing arguments

* Remove spaces added to generated file

public/app/plugins/datasource/grafana-azure-monitor-datasource/azureMetadata/index.ts

@@ -1,4 +1,3 @@
 export * from './locations';
 export * from './resourceTypes';
 export * from './logsResourceTypes';
-export * from './metricNamespaces';

public/app/plugins/datasource/grafana-azure-monitor-datasource/azureMetadata/metricNamespaces.ts

@@ -1,183 +0,0 @@
-/**
-This list is obtained parsing https://docs.microsoft.com/en-us/azure/azure-monitor/essentials/metrics-supported
-To programatically get the list, execute in the browser console:
- 
- const i = [];
- document.querySelectorAll('*[id]').forEach((el) => { 
-  if(el.textContent.match(/^(microsoft|Microsoft|Wandisco)/)) {
-    i.push(el.textContent)
-  }});
-  i;
- 
-Note: Validate that the output makes sense, the format of the page may change.
- */
-export const supportedMetricNamespaces = [
-  'Microsoft.AAD/DomainServices',
-  'microsoft.aadiam/azureADMetrics',
-  'Microsoft.AnalysisServices/servers',
-  'Microsoft.ApiManagement/service',
-  'Microsoft.App/containerapps',
-  'Microsoft.AppConfiguration/configurationStores',
-  'Microsoft.AppPlatform/Spring',
-  'Microsoft.Automation/automationAccounts',
-  'microsoft.avs/privateClouds',
-  'Microsoft.Batch/batchAccounts',
-  'Microsoft.BatchAI/workspaces',
-  'microsoft.bing/accounts',
-  'Microsoft.Blockchain/blockchainMembers',
-  'microsoft.botservice/botservices',
-  'Microsoft.Cache/redis',
-  'Microsoft.Cache/redisEnterprise',
-  'Microsoft.Cdn/cdnwebapplicationfirewallpolicies',
-  'Microsoft.Cdn/profiles',
-  'Microsoft.ClassicCompute/domainNames/slots/roles',
-  'Microsoft.ClassicCompute/virtualMachines',
-  'Microsoft.ClassicStorage/storageAccounts',
-  'Microsoft.ClassicStorage/storageAccounts/blobServices',
-  'Microsoft.ClassicStorage/storageAccounts/fileServices',
-  'Microsoft.ClassicStorage/storageAccounts/queueServices',
-  'Microsoft.ClassicStorage/storageAccounts/tableServices',
-  'Microsoft.Cloudtest/hostedpools',
-  'Microsoft.Cloudtest/pools',
-  'Microsoft.ClusterStor/nodes',
-  'Microsoft.CognitiveServices/accounts',
-  'Microsoft.Communication/CommunicationServices',
-  'Microsoft.Compute/cloudServices',
-  'Microsoft.Compute/cloudServices/roles',
-  'microsoft.compute/disks',
-  'Microsoft.Compute/virtualMachines',
-  'Microsoft.Compute/virtualMachineScaleSets',
-  'Microsoft.Compute/virtualMachineScaleSets/virtualMachines',
-  'Microsoft.ConnectedCache/CacheNodes',
-  'Microsoft.ConnectedVehicle/platformAccounts',
-  'Microsoft.ContainerInstance/containerGroups',
-  'Microsoft.ContainerRegistry/registries',
-  'Microsoft.ContainerService/managedClusters',
-  'Microsoft.CustomProviders/resourceproviders',
-  'Microsoft.DataBoxEdge/dataBoxEdgeDevices',
-  'Microsoft.DataCollaboration/workspaces',
-  'Microsoft.DataFactory/datafactories',
-  'Microsoft.DataFactory/factories',
-  'Microsoft.DataLakeAnalytics/accounts',
-  'Microsoft.DataLakeStore/accounts',
-  'Microsoft.DataProtection/BackupVaults',
-  'Microsoft.DataShare/accounts',
-  'Microsoft.DBforMariaDB/servers',
-  'Microsoft.DBforMySQL/flexibleServers',
-  'Microsoft.DBforMySQL/servers',
-  'Microsoft.DBforPostgreSQL/flexibleServers',
-  'Microsoft.DBForPostgreSQL/serverGroupsv2',
-  'Microsoft.DBforPostgreSQL/servers',
-  'Microsoft.DBforPostgreSQL/serversv2',
-  'Microsoft.Devices/ElasticPools',
-  'Microsoft.Devices/ElasticPools/IotHubTenants',
-  'Microsoft.Devices/IotHubs',
-  'Microsoft.Devices/provisioningServices',
-  'Microsoft.DigitalTwins/digitalTwinsInstances',
-  'Microsoft.DocumentDB/cassandraClusters',
-  'Microsoft.DocumentDB/DatabaseAccounts',
-  'microsoft.edgezones/edgezones',
-  'Microsoft.EventGrid/domains',
-  'Microsoft.EventGrid/eventSubscriptions',
-  'Microsoft.EventGrid/extensionTopics',
-  'Microsoft.EventGrid/partnerNamespaces',
-  'Microsoft.EventGrid/partnerTopics',
-  'Microsoft.EventGrid/systemTopics',
-  'Microsoft.EventGrid/topics',
-  'Microsoft.EventHub/clusters',
-  'Microsoft.EventHub/Namespaces',
-  'Microsoft.HDInsight/clusters',
-  'Microsoft.HealthcareApis/services',
-  'Microsoft.HealthcareApis/workspaces/fhirservices',
-  'Microsoft.HealthcareApis/workspaces/iotconnectors',
-  'microsoft.hybridnetwork/networkfunctions',
-  'microsoft.hybridnetwork/virtualnetworkfunctions',
-  'microsoft.insights/autoscalesettings',
-  'Microsoft.Insights/Components',
-  'Microsoft.IoTCentral/IoTApps',
-  'microsoft.keyvault/managedhsms',
-  'Microsoft.KeyVault/vaults',
-  'microsoft.kubernetes/connectedClusters',
-  'Microsoft.Kusto/Clusters',
-  'Microsoft.Logic/integrationServiceEnvironments',
-  'Microsoft.Logic/Workflows',
-  'Microsoft.MachineLearningServices/workspaces',
-  'Microsoft.Maps/accounts',
-  'Microsoft.Media/mediaservices',
-  'Microsoft.Media/mediaservices/liveEvents',
-  'Microsoft.Media/mediaservices/streamingEndpoints',
-  'Microsoft.Media/videoanalyzers',
-  'Microsoft.MixedReality/remoteRenderingAccounts',
-  'Microsoft.MixedReality/spatialAnchorsAccounts',
-  'Microsoft.NetApp/netAppAccounts/capacityPools',
-  'Microsoft.NetApp/netAppAccounts/capacityPools/volumes',
-  'Microsoft.Network/applicationgateways',
-  'Microsoft.Network/azureFirewalls',
-  'microsoft.network/bastionHosts',
-  'Microsoft.Network/connections',
-  'Microsoft.Network/dnsForwardingRulesets',
-  'Microsoft.Network/dnsResolvers',
-  'Microsoft.Network/dnszones',
-  'Microsoft.Network/expressRouteCircuits',
-  'Microsoft.Network/expressRouteCircuits/peerings',
-  'Microsoft.Network/expressRouteGateways',
-  'Microsoft.Network/expressRoutePorts',
-  'Microsoft.Network/frontdoors',
-  'Microsoft.Network/loadBalancers',
-  'Microsoft.Network/natGateways',
-  'Microsoft.Network/networkInterfaces',
-  'Microsoft.Network/networkWatchers/connectionMonitors',
-  'microsoft.network/p2svpngateways',
-  'Microsoft.Network/privateDnsZones',
-  'Microsoft.Network/privateEndpoints',
-  'Microsoft.Network/privateLinkServices',
-  'Microsoft.Network/publicIPAddresses',
-  'Microsoft.Network/trafficManagerProfiles',
-  'Microsoft.Network/virtualHubs',
-  'microsoft.network/virtualnetworkgateways',
-  'Microsoft.Network/virtualNetworks',
-  'Microsoft.Network/virtualRouters',
-  'microsoft.network/vpngateways',
-  'Microsoft.NotificationHubs/Namespaces/NotificationHubs',
-  'Microsoft.OperationalInsights/workspaces',
-  'Microsoft.Peering/peerings',
-  'Microsoft.Peering/peeringServices',
-  'Microsoft.PowerBIDedicated/capacities',
-  'microsoft.purview/accounts',
-  'Microsoft.RecoveryServices/Vaults',
-  'Microsoft.Relay/namespaces',
-  'microsoft.resources/subscriptions',
-  'Microsoft.Search/searchServices',
-  'microsoft.securitydetonation/chambers',
-  'Microsoft.ServiceBus/Namespaces',
-  'Microsoft.SignalRService/SignalR',
-  'Microsoft.SignalRService/WebPubSub',
-  'Microsoft.Sql/managedInstances',
-  'Microsoft.Sql/servers/databases',
-  'Microsoft.Sql/servers/elasticPools',
-  'Microsoft.Storage/storageAccounts',
-  'Microsoft.Storage/storageAccounts/blobServices',
-  'Microsoft.Storage/storageAccounts/fileServices',
-  'Microsoft.Storage/storageAccounts/queueServices',
-  'Microsoft.Storage/storageAccounts/tableServices',
-  'Microsoft.StorageCache/caches',
-  'microsoft.storagesync/storageSyncServices',
-  'Microsoft.StreamAnalytics/streamingjobs',
-  'Microsoft.Synapse/workspaces',
-  'Microsoft.Synapse/workspaces/bigDataPools',
-  'Microsoft.Synapse/workspaces/kustoPools',
-  'Microsoft.Synapse/workspaces/sqlPools',
-  'Microsoft.TimeSeriesInsights/environments',
-  'Microsoft.TimeSeriesInsights/environments/eventsources',
-  'Microsoft.VMwareCloudSimple/virtualMachines',
-  'Microsoft.Web/connections',
-  'Microsoft.Web/containerapps',
-  'Microsoft.Web/hostingEnvironments',
-  'Microsoft.Web/hostingEnvironments/multiRolePools',
-  'Microsoft.Web/hostingEnvironments/workerPools',
-  'Microsoft.Web/serverfarms',
-  'Microsoft.Web/sites',
-  'Microsoft.Web/sites/slots',
-  'Microsoft.Web/staticSites',
-  'Wandisco.Fusion/migrators',
-];

public/app/plugins/datasource/grafana-azure-monitor-datasource/components/MetricsQueryEditor/MetricsQueryEditor.test.tsx

@@ -31,17 +31,21 @@ const variableOptionGroup = {
 };
 
 export function createMockResourcePickerData() {
-  const mockDatasource = new ResourcePickerData(createMockInstanceSetttings());
+  const mockDatasource = createMockDatasource();
+  const mockResourcePicker = new ResourcePickerData(
+    createMockInstanceSetttings(),
+    mockDatasource.azureMonitorDatasource
+  );
 
-  mockDatasource.getSubscriptions = jest.fn().mockResolvedValue(createMockSubscriptions());
-  mockDatasource.getResourceGroupsBySubscriptionId = jest
+  mockResourcePicker.getSubscriptions = jest.fn().mockResolvedValue(createMockSubscriptions());
+  mockResourcePicker.getResourceGroupsBySubscriptionId = jest
     .fn()
     .mockResolvedValue(createMockResourceGroupsBySubscription());
-  mockDatasource.getResourcesForResourceGroup = jest.fn().mockResolvedValue(mockResourcesByResourceGroup());
-  mockDatasource.getResourceURIFromWorkspace = jest.fn().mockReturnValue('');
-  mockDatasource.getResourceURIDisplayProperties = jest.fn().mockResolvedValue({});
+  mockResourcePicker.getResourcesForResourceGroup = jest.fn().mockResolvedValue(mockResourcesByResourceGroup());
+  mockResourcePicker.getResourceURIFromWorkspace = jest.fn().mockReturnValue('');
+  mockResourcePicker.getResourceURIDisplayProperties = jest.fn().mockResolvedValue({});
 
-  return mockDatasource;
+  return mockResourcePicker;
 }
 
 describe('MetricsQueryEditor', () => {

public/app/plugins/datasource/grafana-azure-monitor-datasource/components/ResourcePicker/ResourcePicker.test.tsx

@@ -2,6 +2,7 @@ import { act, render, screen } from '@testing-library/react';
 import userEvent from '@testing-library/user-event';
 import React from 'react';
 
+import createMockDatasource from '../../__mocks__/datasource';
 import { createMockInstanceSetttings } from '../../__mocks__/instanceSettings';
 import {
   createMockResourceGroupsBySubscription,
@@ -23,7 +24,11 @@ const singleResourceSelectionURI =
 
 const noop: any = () => {};
 function createMockResourcePickerData() {
-  const mockResourcePicker = new ResourcePickerData(createMockInstanceSetttings());
+  const mockDatasource = createMockDatasource();
+  const mockResourcePicker = new ResourcePickerData(
+    createMockInstanceSetttings(),
+    mockDatasource.azureMonitorDatasource
+  );
 
   mockResourcePicker.getSubscriptions = jest.fn().mockResolvedValue(createMockSubscriptions());
   mockResourcePicker.getResourceGroupsBySubscriptionId = jest

public/app/plugins/datasource/grafana-azure-monitor-datasource/datasource.ts

@@ -46,7 +46,7 @@ export default class Datasource extends DataSourceWithBackend<AzureMonitorQuery,
     this.azureMonitorDatasource = new AzureMonitorDatasource(instanceSettings);
     this.azureLogAnalyticsDatasource = new AzureLogAnalyticsDatasource(instanceSettings);
     this.azureResourceGraphDatasource = new AzureResourceGraphDatasource(instanceSettings);
-    this.resourcePickerData = new ResourcePickerData(instanceSettings);
+    this.resourcePickerData = new ResourcePickerData(instanceSettings, this.azureMonitorDatasource);
 
     this.pseudoDatasource = {
       [AzureQueryType.AzureMonitor]: this.azureMonitorDatasource,

public/app/plugins/datasource/grafana-azure-monitor-datasource/resourcePicker/resourcePickerData.test.ts

@@ -3,6 +3,7 @@ import {
   createMockARGResourceGroupsResponse,
   createMockARGSubscriptionResponse,
 } from '../__mocks__/argResourcePickerResponse';
+import createMockDatasource from '../__mocks__/datasource';
 import { createMockInstanceSetttings } from '../__mocks__/instanceSettings';
 import { AzureGraphResponse } from '../types';
 
@@ -10,8 +11,11 @@ import ResourcePickerData from './resourcePickerData';
 
 const createResourcePickerData = (responses: AzureGraphResponse[]) => {
   const instanceSettings = createMockInstanceSetttings();
-  const resourcePickerData = new ResourcePickerData(instanceSettings);
-
+  const mockDatasource = createMockDatasource();
+  mockDatasource.azureMonitorDatasource.getMetricNamespaces = jest
+    .fn()
+    .mockResolvedValueOnce([{ text: 'Microsoft.Storage/storageAccounts', value: 'Microsoft.Storage/storageAccounts' }]);
+  const resourcePickerData = new ResourcePickerData(instanceSettings, mockDatasource.azureMonitorDatasource);
   const postResource = jest.fn();
   responses.forEach((res) => {
     postResource.mockResolvedValueOnce(res);
@@ -192,14 +196,18 @@ describe('AzureMonitor resourcePickerData', () => {
     });
 
     it('filters by metric specific resources', async () => {
-      const mockResponse = createMockARGResourceGroupsResponse();
-      const { resourcePickerData, postResource } = createResourcePickerData([mockResponse]);
+      const mockSubscriptionsResponse = createMockARGSubscriptionResponse();
+      const mockResourceGroupsResponse = createMockARGResourceGroupsResponse();
+      const { resourcePickerData, postResource } = createResourcePickerData([
+        mockSubscriptionsResponse,
+        mockResourceGroupsResponse,
+      ]);
       await resourcePickerData.getResourceGroupsBySubscriptionId('123', 'metrics');
 
-      expect(postResource).toBeCalledTimes(1);
-      const firstCall = postResource.mock.calls[0];
-      const [_, postBody] = firstCall;
-      expect(postBody.query).toContain('wandisco.fusion/migrators');
+      expect(postResource).toBeCalledTimes(2);
+      const secondCall = postResource.mock.calls[1];
+      const [_, postBody] = secondCall;
+      expect(postBody.query).toContain('microsoft.storage/storageaccounts');
     });
   });
 
@@ -262,19 +270,25 @@ describe('AzureMonitor resourcePickerData', () => {
     });
 
     it('should filter metrics resources', async () => {
-      const mockResponse = createARGResourcesResponse();
-      const { resourcePickerData, postResource } = createResourcePickerData([mockResponse]);
+      const mockSubscriptionsResponse = createMockARGSubscriptionResponse();
+      const mockResourcesResponse = createARGResourcesResponse();
+      const { resourcePickerData, postResource } = createResourcePickerData([
+        mockSubscriptionsResponse,
+        mockResourcesResponse,
+      ]);
       await resourcePickerData.getResourcesForResourceGroup('dev', 'metrics');
 
-      expect(postResource).toBeCalledTimes(1);
-      const firstCall = postResource.mock.calls[0];
-      const [_, postBody] = firstCall;
-      expect(postBody.query).toContain('wandisco.fusion/migrators');
+      expect(postResource).toBeCalledTimes(2);
+      const secondCall = postResource.mock.calls[1];
+      const [_, postBody] = secondCall;
+      expect(postBody.query).toContain('microsoft.storage/storageaccounts');
     });
   });
 
   describe('search', () => {
     it('makes requests for metrics searches', async () => {
+      const mockSubscriptionsResponse = createMockARGSubscriptionResponse();
+
       const mockResponse = {
         data: [
           {
@@ -287,11 +301,11 @@ describe('AzureMonitor resourcePickerData', () => {
           },
         ],
       };
-      const { resourcePickerData, postResource } = createResourcePickerData([mockResponse]);
+      const { resourcePickerData, postResource } = createResourcePickerData([mockSubscriptionsResponse, mockResponse]);
       const formattedResults = await resourcePickerData.search('vmname', 'metrics');
-      expect(postResource).toBeCalledTimes(1);
-      const firstCall = postResource.mock.calls[0];
-      const [_, postBody] = firstCall;
+      expect(postResource).toBeCalledTimes(2);
+      const secondCall = postResource.mock.calls[1];
+      const [_, postBody] = secondCall;
       expect(postBody.query).not.toContain('union resourcecontainers');
       expect(postBody.query).toContain('where id contains "vmname"');
 

public/app/plugins/datasource/grafana-azure-monitor-datasource/resourcePicker/resourcePickerData.ts

@@ -1,3 +1,5 @@
+import { uniq } from 'lodash';
+
 import { DataSourceInstanceSettings } from '@grafana/data';
 import { DataSourceWithBackend } from '@grafana/runtime';
 
@@ -6,8 +8,8 @@ import {
   logsSupportedLocationsKusto,
   logsResourceTypes,
   resourceTypeDisplayNames,
-  supportedMetricNamespaces,
 } from '../azureMetadata';
+import AzureMonitorDatasource from '../azure_monitor/azure_monitor_datasource';
 import { ResourceRow, ResourceRowGroup, ResourceRowType } from '../components/ResourcePicker/types';
 import { addResources, parseResourceDetails, parseResourceURI } from '../components/ResourcePicker/utils';
 import {
@@ -26,17 +28,22 @@ import { routeNames } from '../utils/common';
 const RESOURCE_GRAPH_URL = '/providers/Microsoft.ResourceGraph/resources?api-version=2021-03-01';
 
 const logsSupportedResourceTypesKusto = logsResourceTypes.map((v) => `"${v}"`).join(',');
-const supportedMetricNamespacesKusto = supportedMetricNamespaces.map((v) => `"${v.toLocaleLowerCase()}"`).join(',');
 
 export type ResourcePickerQueryType = 'logs' | 'metrics';
 
 export default class ResourcePickerData extends DataSourceWithBackend<AzureMonitorQuery, AzureDataSourceJsonData> {
   private resourcePath: string;
   resultLimit = 200;
+  azureMonitorDatasource;
+  supportedMetricNamespaces = '';
 
-  constructor(instanceSettings: DataSourceInstanceSettings<AzureDataSourceJsonData>) {
+  constructor(
+    instanceSettings: DataSourceInstanceSettings<AzureDataSourceJsonData>,
+    azureMonitorDatasource: AzureMonitorDatasource
+  ) {
     super(instanceSettings);
     this.resourcePath = `${routeNames.resourceGraph}`;
+    this.azureMonitorDatasource = azureMonitorDatasource;
   }
 
   async fetchInitialRows(
@@ -86,7 +93,7 @@ export default class ResourcePickerData extends DataSourceWithBackend<AzureMonit
     }
     searchQuery += `
         | where id contains "${searchPhrase}"
-        ${this.filterByType(searchType)}
+        ${await this.filterByType(searchType)}
         | order by tolower(name) asc
         | limit ${this.resultLimit}
       `;
@@ -173,7 +180,7 @@ export default class ResourcePickerData extends DataSourceWithBackend<AzureMonit
        | project resourceGroupURI=id, resourceGroupName=name, resourceGroup, subscriptionId
      ) on resourceGroup, subscriptionId
 
-     ${this.filterByType(type)}
+     ${await this.filterByType(type)}
      | where subscriptionId == '${subscriptionId}'
      | summarize count() by resourceGroupName, resourceGroupURI
      | order by resourceGroupURI asc`;
@@ -218,7 +225,7 @@ export default class ResourcePickerData extends DataSourceWithBackend<AzureMonit
     const { data: response } = await this.makeResourceGraphRequest<RawAzureResourceItem[]>(`
       resources
       | where id hasprefix "${resourceGroupId}"
-      ${this.filterByType(type)} and location in (${logsSupportedLocationsKusto})
+      ${await this.filterByType(type)} and location in (${logsSupportedLocationsKusto})
     `);
 
     return response.map((item) => {
@@ -326,9 +333,30 @@ export default class ResourcePickerData extends DataSourceWithBackend<AzureMonit
     }
   }
 
-  private filterByType = (t: ResourcePickerQueryType) => {
+  private filterByType = async (t: ResourcePickerQueryType) => {
+    if (this.supportedMetricNamespaces === '' && t !== 'logs') {
+      await this.fetchAllNamespaces();
+    }
     return t === 'logs'
       ? `| where type in (${logsSupportedResourceTypesKusto})`
-      : `| where type in (${supportedMetricNamespacesKusto})`;
+      : `| where type in (${this.supportedMetricNamespaces})`;
   };
+
+  private async fetchAllNamespaces() {
+    const subscriptions = await this.getSubscriptions();
+    let supportedMetricNamespaces: string[] = [];
+    for await (const subscription of subscriptions) {
+      const namespaces = await this.azureMonitorDatasource.getMetricNamespaces(
+        {
+          resourceUri: `/subscriptions/${subscription.id}`,
+        },
+        false
+      );
+      if (namespaces) {
+        const namespaceVals = namespaces.map((namespace) => `"${namespace.value.toLocaleLowerCase()}"`);
+        supportedMetricNamespaces = supportedMetricNamespaces.concat(namespaceVals);
+      }
+    }
+    this.supportedMetricNamespaces = uniq(supportedMetricNamespaces).join(',');
+  }
 }