Drone: Add Windows PR pipeline (#26589)

* Drone: Add Windows PR pipeline
* CI: Add Dockerfile for WiX image

.drone.yml

@@ -26,7 +26,7 @@ steps:
   - yarn install --frozen-lockfile --no-progress
   environment:
     DOCKERIZE_VERSION: 0.6.1
-    GRABPL_VERSION: 0.5.0
+    GRABPL_VERSION: 0.5.1
 
 - name: lint-backend
   image: grafana/build-container:1.2.24
@@ -253,7 +253,7 @@ steps:
   - yarn install --frozen-lockfile --no-progress
   environment:
     DOCKERIZE_VERSION: 0.6.1
-    GRABPL_VERSION: 0.5.0
+    GRABPL_VERSION: 0.5.1
 
 - name: lint-backend
   image: grafana/build-container:1.2.24
@@ -452,13 +452,6 @@ steps:
   - test-backend
   - test-frontend
 
-- name: build-windows-installer
-  image: grafana/wix-toolset-ci:v3
-  commands:
-  - echo ./scripts/build/ci-msi-build/ci-msi-build-oss.sh
-  depends_on:
-  - package
-
 - name: release-next-npm-packages
   image: grafana/build-container:1.2.24
   commands:
@@ -503,4 +496,49 @@ trigger:
   event:
   - push
 
+---
+kind: pipeline
+type: docker
+name: windows-installer-master
+
+platform:
+  os: windows
+  arch: amd64
+  version: 1809
+
+steps:
+- name: identify-runner
+  image: mcr.microsoft.com/windows:1809
+  commands:
+  - echo $Env:DRONE_RUNNER_NAME
+
+- name: build-windows-installer
+  image: grafana/ci-wix:0.1.1
+  commands:
+  - $$gcpKey = $$env:GCP_KEY
+  - "[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json"
+  - dos2unix gcpkey.json
+  - gcloud auth activate-service-account --key-file=gcpkey.json
+  - rm gcpkey.json
+  - $$ProgressPreference = "SilentlyContinue"
+  - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v0.5.1/windows/grabpl.exe -OutFile grabpl.exe
+  - Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/oss/master/grafana-7.2.0-9fffe273pre.windows-amd64.zip -OutFile grafana.zip
+  - cp C:\App\nssm-2.24.zip .
+  - ./grabpl.exe windows-installer --edition oss grafana.zip
+  - $$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]
+  - echo "gsutil cp $$fname gs://grafana-downloads/oss/master/"
+  - echo "gsutil cp $$fname.sha256 gs://grafana-downloads/oss/master/"
+  environment:
+    GCP_KEY:
+      from_secret: gcp_key
+
+trigger:
+  branch:
+  - master
+  event:
+  - push
+
+depends_on:
+- test-master
+
 ...

go.mod

@@ -50,6 +50,7 @@ require (
 	github.com/klauspost/cpuid v1.2.0 // indirect
 	github.com/lib/pq v1.2.0
 	github.com/linkedin/goavro/v2 v2.9.7
+	github.com/magefile/mage v1.9.0
 	github.com/mattn/go-colorable v0.1.7 // indirect
 	github.com/mattn/go-isatty v0.0.12
 	github.com/mattn/go-sqlite3 v1.11.0

go.sum

@@ -288,6 +288,7 @@ github.com/linkedin/goavro/v2 v2.9.7 h1:Vd++Rb/RKcmNJjM0HP/JJFMEWa21eUBVKPYlKehO
 github.com/linkedin/goavro/v2 v2.9.7/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
 github.com/lunny/log v0.0.0-20160921050905-7887c61bf0de/go.mod h1:3q8WtuPQsoRbatJuy3nvq/hRSvuBJrHHr+ybPPiNvHQ=
 github.com/lunny/nodb v0.0.0-20160621015157-fc1ef06ad4af/go.mod h1:Cqz6pqow14VObJ7peltM+2n3PWOz7yTrfUuGbVFkzN0=
+github.com/magefile/mage v1.9.0 h1:t3AU2wNwehMCW97vuqQLtw6puppWXHO+O2MHo5a50XE=
 github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ=
 github.com/mattetti/filebuffer v1.0.0 h1:ixTvQ0JjBTwWbdpDZ98lLrydo7KRi8xNRIi5RFszsbY=

scripts/build/ci-wix/Dockerfile

@@ -0,0 +1,18 @@
+# This has to correspond to the version the Drone runners have
+FROM mcr.microsoft.com/windows:1809
+
+WORKDIR C:\\App
+
+RUN powershell Set-ExecutionPolicy RemoteSigned -Scope CurrentUser -Force
+RUN powershell Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')
+RUN scoop install wixtoolset@3.11.2
+RUN powershell (New-Object Net.WebClient).DownloadFile(\
+  \"https://grafana-downloads.storage.googleapis.com/ci-dependencies/nssm-2.24.zip\", \
+  \"nssm-2.24.zip\")
+RUN scoop install git@2.28.0.windows.1
+RUN scoop bucket add extras
+RUN scoop install gcloud@305.0.0
+# Installing dos2unix fails if not under PowerShell
+RUN powershell scoop install dos2unix
+
+ENTRYPOINT ["powershell"]

scripts/build/ci-wix/Magefile.go

@@ -0,0 +1,27 @@
+//+build mage
+
+package main
+
+import (
+	"github.com/magefile/mage/mg"
+	"github.com/magefile/mage/sh"
+)
+
+const imageName = "grafana/ci-wix:0.1.1"
+
+// Build builds the Docker image.
+func Build() error {
+	if err := sh.RunV("docker", "build", "-t", imageName, "."); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Publish publishes the Docker image.
+func Publish() error {
+	mg.Deps(Build)
+	return sh.RunV("docker", "push", imageName)
+}
+
+var Default = Build

scripts/build/ci-wix/README.md

@@ -0,0 +1,15 @@
+# WiX Docker Image
+This directory contains a Dockerfile for building a Windows based image containing
+the WiX toolkit, that we use to build Windows installers.
+
+To build the Docker image:
+
+```
+mage
+```
+
+To publish the Docker image:
+
+```
+mage publish
+```

scripts/lib.star

@@ -2,6 +2,8 @@ build_image = 'grafana/build-container:1.2.24'
 publish_image = 'grafana/grafana-ci-deploy:1.2.5'
 grafana_docker_image = 'grafana/drone-grafana-docker:0.2.0'
 alpine_image = 'alpine:3.12'
+windows_image = 'mcr.microsoft.com/windows:1809'
+grabpl_version = '0.5.1'
 
 def pr_pipelines(edition):
     services = [
@@ -95,47 +97,79 @@ def master_pipelines(edition):
         build_docker_images_step(edition=edition, ubuntu=True),
         postgres_integration_tests_step(),
         mysql_integration_tests_step(),
-        build_windows_installer_step(),
         release_next_npm_packages_step(edition),
         publish_packages_step(edition),
         deploy_to_kubernetes_step(edition),
     ]
-    return [
+    windows_steps = [
-        pipeline(
+        windows_installer_step(edition),
-            name='test-master', edition=edition, trigger={
+    ]
+    trigger = {
         'event': ['push',],
         'branch': 'master',
-            }, services=services, steps=steps
+    }
+    return [
+        pipeline(
+            name='test-master', edition=edition, trigger=trigger, services=services, steps=steps
+        ),
+        pipeline(
+            name='windows-installer-master', edition=edition, trigger=trigger,
+            steps=windows_steps, platform='windows', depends_on=['test-master'],
         ),
     ]
 
-def pipeline(name, edition, trigger, steps, services=[]):
+def pipeline(name, edition, trigger, steps, services=[], platform='linux', depends_on=[]):
+    if platform != 'windows':
+        platform_conf = {
+            'os': 'linux',
+            'arch': 'amd64',
+        }
+    else:
+        platform_conf = {
+            'os': 'windows',
+            'arch': 'amd64',
+            'version': '1809',
+        }
+
     pipeline = {
         'kind': 'pipeline',
         'type': 'docker',
+        'platform': platform_conf,
         'name': name,
         'trigger': trigger,
         'services': services,
-        'steps': init_steps(edition) + steps,
+        'steps': init_steps(edition, platform) + steps,
+        'depends_on': depends_on,
     }
+
     if edition == 'enterprise':
         # We have a custom clone step for enterprise
         pipeline['clone'] = {
             'disable': True,
         }
 
-    pipeline['steps'].insert(0, {
+    return pipeline
+
+def init_steps(edition, platform):
+    if platform == 'windows':
+        return [
+            {
+                'name': 'identify-runner',
+                'image': windows_image,
+                'commands': [
+                    'echo $Env:DRONE_RUNNER_NAME',
+                ],
+            },
+        ]
+
+    identify_runner_step = {
         'name': 'identify-runner',
         'image': alpine_image,
         'commands': [
             'echo $DRONE_RUNNER_NAME',
         ],
-    })
+    }
 
-    return pipeline
-
-def init_steps(edition):
-    grabpl_version = '0.5.0'
     common_cmds = [
         'curl -fLO https://github.com/jwilder/dockerize/releases/download/v$${DOCKERIZE_VERSION}/dockerize-linux-amd64-v$${DOCKERIZE_VERSION}.tar.gz',
         'tar -C bin -xzvf dockerize-linux-amd64-v$${DOCKERIZE_VERSION}.tar.gz',
@@ -144,6 +178,7 @@ def init_steps(edition):
     ]
     if edition == 'enterprise':
         return [
+            identify_runner_step,
             {
                 'name': 'clone',
                 'image': 'alpine/git:v2.26.2',
@@ -181,6 +216,7 @@ def init_steps(edition):
         ]
 
     return [
+        identify_runner_step,
         {
             'name': 'initialize',
             'image': build_image,
@@ -568,8 +604,6 @@ def publish_packages_step(edition):
         'image': publish_image,
         'depends_on': [
             'package',
-            # TODO
-            # 'build-windows-installer',
             'end-to-end-tests',
             'mysql-integration-tests',
             'postgres-integration-tests',
@@ -580,17 +614,33 @@ def publish_packages_step(edition):
         ],
     }
 
-def build_windows_installer_step():
+def windows_installer_step(edition):
-    # TODO: Build Windows installer, waiting on Brian to fix the build image
+    sfx = ''
+    if edition == 'enterprise':
+        sfx = '-enterprise'
     return {
         'name': 'build-windows-installer',
-        # TODO: Need new image that can execute as root
+        'image': 'grafana/ci-wix:0.1.1',
-        'image': 'grafana/wix-toolset-ci:v3',
+        'environment': {
-        'depends_on': [
+            'GCP_KEY': {
-            'package',
+                'from_secret': 'gcp_key',
-        ],
+            },
+        },
         'commands': [
-          # TODO: Enable. Waiting on Brian to fix image.
+            '$$gcpKey = $$env:GCP_KEY',
-          'echo ./scripts/build/ci-msi-build/ci-msi-build-oss.sh',
+            '[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$gcpKey)) > gcpkey.json',
+            # gcloud fails to read the file unless converted with dos2unix
+            'dos2unix gcpkey.json',
+            'gcloud auth activate-service-account --key-file=gcpkey.json',
+            'rm gcpkey.json',
+            '$$ProgressPreference = "SilentlyContinue"',
+            'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/grafana-build-pipeline/v{}/windows/grabpl.exe -OutFile grabpl.exe'.format(grabpl_version),
+            # TODO: Infer correct Grafana version
+            'Invoke-WebRequest https://grafana-downloads.storage.googleapis.com/{}/master/grafana{}-7.2.0-9fffe273pre.windows-amd64.zip -OutFile grafana.zip'.format(edition, sfx),
+            'cp C:\\App\\nssm-2.24.zip .',
+            './grabpl.exe windows-installer --edition {} grafana.zip'.format(edition),
+            '$$fname = ((Get-Childitem grafana*.msi -name) -split "`n")[0]',
+            'echo "gsutil cp $$fname gs://grafana-downloads/{}/master/"'.format(edition),
+            'echo "gsutil cp $$fname.sha256 gs://grafana-downloads/{}/master/"'.format(edition),
         ],
     }