{
  "__inputs": [],
  "__requires": [
    { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "7.3.0-pre" },
    { "type": "panel", "id": "graph", "name": "Graph", "version": "" },
    { "type": "datasource", "id": "stackdriver", "name": "Google Cloud Monitoring", "version": "1.0.0" }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "gnetId": null,
  "graphTooltip": 0,
  "id": null,
  "links": [],
  "panels": [
    {
      "datasource": null,
      "description": "",
      "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] },
      "gridPos": { "h": 3, "w": 24, "x": 0, "y": 0 },
      "id": 0,
      "options": {
        "mode": "markdown",
        "content": "This dashboard demonstrates 6 different charts in a single dashboard that provides a single view for the data processing pipeline [metrics](https://cloud.google.com/monitoring/api/metrics_gcp). Read the blog post [Use SRE principles to monitor pipelines with Cloud Monitoring dashboards](https://cloud.google.com/blog/products/management-tools/the-right-metrics-to-monitor-cloud-data-pipelines) for additional details."
      },
      "pluginVersion": "7.4.0-pre",
      "timeFrom": null,
      "timeShift": null,
      "title": "Data Processing Monitoring",
      "type": "text"
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$datasource",
      "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
      "hiddenSeries": false,
      "id": 1,
      "legend": {
        "alignAsTable": false,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": false,
        "show": true,
        "total": false,
        "values": false,
        "sideWidth": 220
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": { "alertThreshold": true },
      "percentage": false,
      "pluginVersion": "7.4.0-pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Dataflow (traffic)",
      "tooltip": { "shared": true, "sort": 0, "value_type": "individual" },
      "type": "graph",
      "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] },
      "yaxes": [
        { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true },
        { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }
      ],
      "yaxis": { "align": false, "alignLevel": null },
      "targets": [
        {
          "queryType": "metrics",
          "refId": "A",
          "metricQuery": {
            "aliasBy": "",
            "alignmentPeriod": "$alignmentPeriod",
            "crossSeriesReducer": "REDUCE_SUM",
            "perSeriesAligner": "ALIGN_RATE",
            "filters": ["resource.type", "=", "dataflow_job"],
            "groupBys": ["resource.label.job_name"],
            "metricKind": "DELTA",
            "metricType": "dataflow.googleapis.com/job/elements_produced_count",
            "projectName": "$project",
            "unit": "1",
            "valueType": "INT64"
          }
        }
      ]
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$datasource",
      "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
      "hiddenSeries": false,
      "id": 2,
      "legend": {
        "alignAsTable": false,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": false,
        "show": true,
        "total": false,
        "values": false,
        "sideWidth": 220
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": { "alertThreshold": true },
      "percentage": false,
      "pluginVersion": "7.4.0-pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Pubsub (traffic)",
      "tooltip": { "shared": true, "sort": 0, "value_type": "individual" },
      "type": "graph",
      "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] },
      "yaxes": [
        { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true },
        { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }
      ],
      "yaxis": { "align": false, "alignLevel": null },
      "targets": [
        {
          "queryType": "metrics",
          "refId": "A",
          "metricQuery": {
            "aliasBy": "",
            "alignmentPeriod": "$alignmentPeriod",
            "crossSeriesReducer": "REDUCE_SUM",
            "perSeriesAligner": "ALIGN_RATE",
            "filters": ["resource.type", "=", "pubsub_topic"],
            "groupBys": ["metric.label.response_code"],
            "metricKind": "DELTA",
            "metricType": "pubsub.googleapis.com/topic/send_request_count",
            "projectName": "$project",
            "unit": "1",
            "valueType": "INT64"
          }
        }
      ]
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$datasource",
      "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
      "hiddenSeries": false,
      "id": 3,
      "legend": {
        "alignAsTable": false,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": false,
        "show": true,
        "total": false,
        "values": false,
        "sideWidth": 220
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": { "alertThreshold": true },
      "percentage": false,
      "pluginVersion": "7.4.0-pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "BigQuery (traffic)",
      "tooltip": { "shared": true, "sort": 0, "value_type": "individual" },
      "type": "graph",
      "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] },
      "yaxes": [
        { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true },
        { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }
      ],
      "yaxis": { "align": false, "alignLevel": null },
      "targets": [
        {
          "queryType": "metrics",
          "refId": "A",
          "metricQuery": {
            "aliasBy": "",
            "alignmentPeriod": "$alignmentPeriod",
            "perSeriesAligner": "ALIGN_RATE",
            "filters": ["resource.type", "=", "bigquery_dataset"],
            "groupBys": [],
            "metricKind": "DELTA",
            "metricType": "bigquery.googleapis.com/storage/uploaded_bytes",
            "projectName": "$project",
            "unit": "By",
            "valueType": "INT64"
          }
        }
      ]
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$datasource",
      "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
      "hiddenSeries": false,
      "id": 4,
      "legend": {
        "alignAsTable": false,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": false,
        "show": true,
        "total": false,
        "values": false,
        "sideWidth": 220
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": { "alertThreshold": true },
      "percentage": false,
      "pluginVersion": "7.4.0-pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "System lag (latency)",
      "tooltip": { "shared": true, "sort": 0, "value_type": "individual" },
      "type": "graph",
      "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] },
      "yaxes": [
        { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true },
        { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }
      ],
      "yaxis": { "align": false, "alignLevel": null },
      "targets": [
        {
          "queryType": "metrics",
          "refId": "A",
          "metricQuery": {
            "aliasBy": "",
            "alignmentPeriod": "$alignmentPeriod",
            "perSeriesAligner": "ALIGN_MEAN",
            "filters": ["resource.type", "=", "dataflow_job"],
            "groupBys": [],
            "metricKind": "GAUGE",
            "metricType": "dataflow.googleapis.com/job/system_lag",
            "projectName": "$project",
            "unit": "s",
            "valueType": "INT64"
          }
        },
        {
          "queryType": "metrics",
          "refId": "B",
          "metricQuery": {
            "aliasBy": "",
            "alignmentPeriod": "$alignmentPeriod",
            "perSeriesAligner": "ALIGN_MEAN",
            "filters": ["resource.type", "=", "dataflow_job"],
            "groupBys": [],
            "metricKind": "GAUGE",
            "metricType": "dataflow.googleapis.com/job/data_watermark_age",
            "projectName": "$project",
            "unit": "s",
            "valueType": "INT64"
          }
        }
      ]
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$datasource",
      "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 },
      "hiddenSeries": false,
      "id": 5,
      "legend": {
        "alignAsTable": false,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": false,
        "show": true,
        "total": false,
        "values": false,
        "sideWidth": 220
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": { "alertThreshold": true },
      "percentage": false,
      "pluginVersion": "7.4.0-pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Dataflow processing volume (throughput)",
      "tooltip": { "shared": true, "sort": 0, "value_type": "individual" },
      "type": "graph",
      "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] },
      "yaxes": [
        { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true },
        { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }
      ],
      "yaxis": { "align": false, "alignLevel": null },
      "targets": [
        {
          "queryType": "metrics",
          "refId": "A",
          "metricQuery": {
            "aliasBy": "",
            "alignmentPeriod": "$alignmentPeriod",
            "crossSeriesReducer": "REDUCE_SUM",
            "perSeriesAligner": "ALIGN_MEAN",
            "filters": ["resource.type", "=", "dataflow_job"],
            "groupBys": ["resource.label.job_name"],
            "metricKind": "GAUGE",
            "metricType": "dataflow.googleapis.com/job/element_count",
            "projectName": "$project",
            "unit": "1",
            "valueType": "INT64"
          }
        }
      ]
    },
    {
      "aliasColors": {},
      "bars": false,
      "dashLength": 10,
      "dashes": false,
      "datasource": "$datasource",
      "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] },
      "fill": 1,
      "fillGradient": 0,
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 },
      "hiddenSeries": false,
      "id": 6,
      "legend": {
        "alignAsTable": false,
        "avg": false,
        "current": false,
        "max": false,
        "min": false,
        "rightSide": false,
        "show": true,
        "total": false,
        "values": false,
        "sideWidth": 220
      },
      "lines": true,
      "linewidth": 1,
      "nullPointMode": "null",
      "options": { "alertThreshold": true },
      "percentage": false,
      "pluginVersion": "7.4.0-pre",
      "pointradius": 2,
      "points": false,
      "renderer": "flot",
      "seriesOverrides": [],
      "spaceLength": 10,
      "stack": false,
      "steppedLine": false,
      "thresholds": [],
      "timeFrom": null,
      "timeRegions": [],
      "timeShift": null,
      "title": "Pipeline logging (errors)",
      "tooltip": { "shared": true, "sort": 0, "value_type": "individual" },
      "type": "graph",
      "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] },
      "yaxes": [
        { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true },
        { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }
      ],
      "yaxis": { "align": false, "alignLevel": null },
      "targets": [
        {
          "queryType": "metrics",
          "refId": "A",
          "metricQuery": {
            "aliasBy": "",
            "alignmentPeriod": "$alignmentPeriod",
            "perSeriesAligner": "ALIGN_RATE",
            "filters": ["resource.type", "=", "bigquery_dataset", "AND", "metric.label.severity", "=", "ERROR"],
            "groupBys": [],
            "metricKind": "DELTA",
            "metricType": "logging.googleapis.com/log_entry_count",
            "projectName": "$project",
            "unit": "1",
            "valueType": "INT64"
          }
        },
        {
          "queryType": "metrics",
          "refId": "B",
          "metricQuery": {
            "aliasBy": "",
            "alignmentPeriod": "$alignmentPeriod",
            "perSeriesAligner": "ALIGN_RATE",
            "filters": ["resource.type", "=", "pubsub_topic", "AND", "metric.label.severity", "=", "ERROR"],
            "groupBys": [],
            "metricKind": "DELTA",
            "metricType": "logging.googleapis.com/log_entry_count",
            "projectName": "$project",
            "unit": "1",
            "valueType": "INT64"
          }
        },
        {
          "queryType": "metrics",
          "refId": "C",
          "metricQuery": {
            "aliasBy": "",
            "alignmentPeriod": "$alignmentPeriod",
            "perSeriesAligner": "ALIGN_RATE",
            "filters": ["resource.type", "=", "dataflow_job", "AND", "metric.label.severity", "=", "ERROR"],
            "groupBys": [],
            "metricKind": "DELTA",
            "metricType": "logging.googleapis.com/log_entry_count",
            "projectName": "$project",
            "unit": "1",
            "valueType": "INT64"
          }
        }
      ]
    }
  ],
  "schemaVersion": 26,
  "style": "dark",
  "tags": ["Bigdata", "Cloud Monitoring", "GCP"],
  "templating": {
    "list": [
      {
        "current": {},
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": false,
        "label": "Datasource",
        "multi": false,
        "name": "datasource",
        "options": [],
        "query": "stackdriver",
        "queryValue": "",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "type": "datasource"
      },
      {
        "allValue": null,
        "current": {},
        "datasource": "$datasource",
        "definition": "Google Cloud Monitoring - Projects",
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": false,
        "label": "Project",
        "multi": false,
        "name": "project",
        "options": [],
        "query": {
          "labelKey": "",
          "loading": false,
          "projectName": "$project",
          "projects": [],
          "selectedMetricType": "actions.googleapis.com/smarthome_action/num_active_users",
          "selectedQueryType": "projects",
          "selectedSLOService": "",
          "selectedService": "actions.googleapis.com",
          "sloServices": []
        },
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      },
      {
        "allValue": null,
        "current": { "selected": false, "text": "grafana auto", "value": "grafana-auto" },
        "datasource": "${datasource}",
        "definition": "",
        "description": null,
        "error": null,
        "hide": 0,
        "includeAll": false,
        "label": "Alignment Period",
        "multi": false,
        "name": "alignmentPeriod",
        "options": [
          { "selected": true, "text": "grafana auto", "value": "grafana-auto" },
          { "selected": false, "text": "stackdriver auto", "value": "stackdriver-auto" },
          { "selected": false, "text": "cloud monitoring auto", "value": "cloud-monitoring-auto" },
          { "selected": false, "text": "1m", "value": "+60s" },
          { "selected": false, "text": "2m", "value": "+120s" },
          { "selected": false, "text": "5m", "value": "+300s" },
          { "selected": false, "text": "10m", "value": "+600s" },
          { "selected": false, "text": "30m", "value": "+1800s" },
          { "selected": false, "text": "1h", "value": "+3600s" },
          { "selected": false, "text": "3h", "value": "+7200s" },
          { "selected": false, "text": "6h", "value": "+21600s" },
          { "selected": false, "text": "1d", "value": "+86400s" },
          { "selected": false, "text": "3d", "value": "+259200s" },
          { "selected": false, "text": "1w", "value": "+604800s" }
        ],
        "query": {
          "labelKey": "",
          "loading": false,
          "projectName": "$project",
          "projects": [
            { "name": "project-1", "value": "project-1" },
            { "name": "project-2", "value": "project-2" }
          ],
          "refId": "CloudMonitoringVariableQueryEditor-VariableQuery",
          "selectedMetricType": "actions.googleapis.com/smarthome_action/num_active_users",
          "selectedQueryType": "alignmentPeriods",
          "selectedSLOService": "",
          "selectedService": "actions.googleapis.com",
          "sloServices": []
        },
        "refresh": 0,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "tagValuesQuery": "",
        "tags": [],
        "tagsQuery": "",
        "type": "query",
        "useTags": false
      }
    ]
  },
  "time": { "from": "now-24h", "to": "now" },
  "timepicker": {},
  "timezone": "",
  "title": "Data Processing Monitoring",
  "uid": "",
  "version": 5
}