build: crossplatform build with packages.

Big thanks to @fg2it who created a POC as
well as the build container that this work
is based on.

.circleci/config.yml

@@ -70,10 +70,13 @@ jobs:
 
   build:
     docker:
-     - image: grafana/build-container:v0.1
+     - image: grafana/build-container:crosscompile
     working_directory: /go/src/github.com/grafana/grafana
     steps:
       - checkout
+      - run:
+          name: prepare build tools
+          command: '/tmp/bootstrap.sh'
       - run:
           name: build and package grafana
           command: './scripts/build/build.sh'

Gruntfile.js

@@ -12,6 +12,10 @@ module.exports = function (grunt) {
     platform: process.platform.replace('win32', 'windows'),
   };
 
+  if (grunt.option('platform')) {
+    config.platform = grunt.option('platform');
+  }
+
   if (grunt.option('arch')) {
     config.arch = grunt.option('arch');
   } else {

build.go

@@ -27,8 +27,7 @@ var (
 	goarch  string
 	goos    string
 	gocc    string
-	gocxx   string
+	cgo     bool
-	cgo     string
 	pkgArch string
 	version string = "v1"
 	// deb & rpm does not support semver so have to handle their version a little differently
@@ -53,8 +52,7 @@ func main() {
 	flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
 	flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
 	flag.StringVar(&gocc, "cc", "", "CC")
-	flag.StringVar(&gocxx, "cxx", "", "CXX")
+	flag.BoolVar(&cgo, "cgo-enabled", cgo, "Enable cgo")
-	flag.StringVar(&cgo, "cgo-enabled", "", "CGO_ENABLED")
 	flag.StringVar(&pkgArch, "pkg-arch", "", "PKG ARCH")
 	flag.StringVar(&phjsToRelease, "phjs", "", "PhantomJS binary")
 	flag.BoolVar(&race, "race", race, "Use race detector")
@@ -93,20 +91,24 @@ func main() {
 			build("grafana-server", "./pkg/cmd/grafana-server", []string{})
 
 		case "build":
-			clean()
+			//clean()
 			for _, binary := range binaries {
 				build(binary, "./pkg/cmd/"+binary, []string{})
 			}
 
+		case "build-frontend":
+			grunt(gruntBuildArg("build")...)
+
 		case "test":
 			test("./pkg/...")
 			grunt("test")
 
 		case "package":
-			grunt(gruntBuildArg("release")...)
+			grunt(gruntBuildArg("build")...)
-			if runtime.GOOS != "windows" {
+			packageGrafana()
-				createLinuxPackages()
+
-			}
+		case "package-only":
+			packageGrafana()
 
 		case "pkg-rpm":
 			grunt(gruntBuildArg("release")...)
@@ -131,6 +133,22 @@ func main() {
 	}
 }
 
+func packageGrafana() {
+	platformArg := fmt.Sprintf("--platform=%v", goos)
+	previousPkgArch := pkgArch
+	if pkgArch == "" {
+		pkgArch = goarch
+	}
+	postProcessArgs := gruntBuildArg("package")
+	postProcessArgs = append(postProcessArgs, platformArg)
+	grunt(postProcessArgs...)
+	pkgArch = previousPkgArch
+
+	if goos == "linux" && goarch == "amd64"{
+		createLinuxPackages()
+	}
+}
+
 func makeLatestDistCopies() {
 	files, err := ioutil.ReadDir("dist")
 	if err != nil {
@@ -138,9 +156,9 @@ func makeLatestDistCopies() {
 	}
 
 	latestMapping := map[string]string{
-		".deb":    "dist/grafana_latest_amd64.deb",
+		"_amd64.deb":    "dist/grafana_latest_amd64.deb",
-		".rpm":    "dist/grafana-latest-1.x86_64.rpm",
+		".x86_64.rpm":    "dist/grafana-latest-1.x86_64.rpm",
-		".tar.gz": "dist/grafana-latest.linux-x64.tar.gz",
+		".linux-amd64.tar.gz": "dist/grafana-latest.linux-x64.tar.gz",
 	}
 
 	for _, file := range files {
@@ -386,7 +404,8 @@ func test(pkg string) {
 }
 
 func build(binaryName, pkg string, tags []string) {
-	binary := "./bin/" + binaryName
+	binary := fmt.Sprintf("./bin/%s-%s/%s", goos, goarch, binaryName)
+
 	if goos == "windows" {
 		binary += ".exe"
 	}
@@ -408,6 +427,7 @@ func build(binaryName, pkg string, tags []string) {
 	if !isDev {
 		setBuildEnv()
 		runPrint("go", "version")
+		fmt.Printf("Targeting %s/%s\n", goos, goarch)
 	}
 
 	runPrint("go", args...)
@@ -451,6 +471,14 @@ func clean() {
 
 func setBuildEnv() {
 	os.Setenv("GOOS", goos)
+	if goos == "windows" {
+		// require windows >=7
+		os.Setenv("CGO_CFLAGS", "-D_WIN32_WINNT=0x0601")
+	}
+	if goarch != "amd64" || goos != "linux" {
+		// needed for all other archs
+		cgo = true
+	}
 	if strings.HasPrefix(goarch, "armv") {
 		os.Setenv("GOARCH", "arm")
 		os.Setenv("GOARM", goarch[4:])
@@ -460,15 +488,12 @@ func setBuildEnv() {
 	if goarch == "386" {
 		os.Setenv("GO386", "387")
 	}
-	if cgo != "" {
+	if cgo {
-		os.Setenv("CGO_ENABLED", cgo)
+		os.Setenv("CGO_ENABLED", "1")
 	}
 	if gocc != "" {
 		os.Setenv("CC", gocc)
 	}
-	if gocxx != "" {
-		os.Setenv("CXX", gocxx)
-	}
 }
 
 func getGitSha() string {

scripts/build/build.sh

@@ -4,6 +4,12 @@
 #   This script is executed from within the container.
 #
 
+CCARMV7=arm-linux-gnueabihf-gcc
+CCARM64=aarch64-linux-gnu-gcc
+CCOSX64=/tmp/osxcross/target/bin/o64-clang
+CCWIN64=x86_64-w64-mingw32-gcc
+CCX64=/tmp/x86_64-centos6-linux-gnu/bin/x86_64-centos6-linux-gnu-gcc
+
 GOPATH=/go
 REPO_PATH=$GOPATH/src/github.com/grafana/grafana
 
@@ -11,23 +17,45 @@ cd /go/src/github.com/grafana/grafana
 echo "current dir: $(pwd)"
 
 if [ "$CIRCLE_TAG" != "" ]; then
-  echo "Building a release from tag $CIRCLE_TAG"
+  echo "Building releases from tag $CIRCLE_TAG"
-  go run build.go -buildNumber=${CIRCLE_BUILD_NUM} -includeBuildNumber=false build
+  go run build.go -goarch armv7 -cc ${CCARMV7} -includeBuildNumber=false build
+  go run build.go -goarch arm64 -cc ${CCARM64} -includeBuildNumber=false build
+  go run build.go -goos darwin -cc ${CCOSX64} -includeBuildNumber=false build
+  go run build.go -goos windows -cc ${CCWIN64} -includeBuildNumber=false build
+  CC=${CCX64} go run build.go -includeBuildNumber=false build
 else
   echo "Building incremental build for $CIRCLE_BRANCH"
-  go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build
+  go run build.go -goarch armv7 -cc ${CCARMV7} -buildNumber=${CIRCLE_BUILD_NUM} build
+  go run build.go -goarch arm64 -cc ${CCARM64} -buildNumber=${CIRCLE_BUILD_NUM} build
+  go run build.go -goos darwin -cc ${CCOSX64} -buildNumber=${CIRCLE_BUILD_NUM} build
+  go run build.go -goos windows -cc ${CCWIN64} -buildNumber=${CIRCLE_BUILD_NUM} build
+  CC=${CCX64} go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build
 fi
 
 yarn install --pure-lockfile --no-progress
 
-source /etc/profile.d/rvm.sh
-
 echo "current dir: $(pwd)"
 
-if [ "$CIRCLE_TAG" != "" ]; then
+if [ -d "dist" ]; then
-  echo "Packaging a release from tag $CIRCLE_TAG"
+  rm -rf dist
-  go run build.go -buildNumber=${CIRCLE_BUILD_NUM} -includeBuildNumber=false package latest
+fi
-else
+
-  echo "Packaging incremental build for $CIRCLE_BRANCH"
+if [ "$CIRCLE_TAG" != "" ]; then
-  go run build.go -buildNumber=${CIRCLE_BUILD_NUM} package latest
+  echo "Building frontend from tag $CIRCLE_TAG"
+  go run build.go -includeBuildNumber=false build-frontend
+  echo "Packaging a release from tag $CIRCLE_TAG"
+  go run build.go -goos linux -pkg-arch amd64 -includeBuildNumber=false package-only latest
+  go run build.go -goos linux -pkg-arch armv7 -includeBuildNumber=false package-only
+  go run build.go -goos linux -pkg-arch arm64 -includeBuildNumber=false package-only
+  go run build.go -goos darwin -pkg-arch amd64 -includeBuildNumber=false package-only
+  go run build.go -goos windows -pkg-arch amd64 -includeBuildNumber=false package-only
+else
+  echo "Building frontend for $CIRCLE_BRANCH"
+  go run build.go -buildNumber=${CIRCLE_BUILD_NUM} build-frontend
+  echo "Packaging incremental build for $CIRCLE_BRANCH"
+  go run build.go -goos linux -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only latest
+  go run build.go -goos linux -pkg-arch armv7 -buildNumber=${CIRCLE_BUILD_NUM} package-only
+  go run build.go -goos linux -pkg-arch arm64 -buildNumber=${CIRCLE_BUILD_NUM} package-only
+  go run build.go -goos darwin -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only
+  go run build.go -goos windows -pkg-arch amd64 -buildNumber=${CIRCLE_BUILD_NUM} package-only
 fi

scripts/grunt/release_task.js

@@ -3,13 +3,20 @@ var path = require('path');
 module.exports = function(grunt) {
   "use strict";
 
-  // build, then zip and upload to s3
+  // build then zip
   grunt.registerTask('release', [
     'build',
     'build-post-process',
     'compress:release'
   ]);
 
+  // package into archives
+  grunt.registerTask('package', [
+    'clean:temp',
+    'build-post-process',
+    'compress:release'
+  ]);
+
   grunt.registerTask('build-post-process', function() {
     grunt.config('copy.public_to_temp', {
       expand: true,
@@ -18,7 +25,7 @@ module.exports = function(grunt) {
       dest: '<%= tempDir %>/public/',
     });
     grunt.config('copy.backend_bin', {
-      cwd: 'bin',
+      cwd: 'bin/<%= platform %>-<%= arch %>',
       expand: true,
       src: ['*'],
       options: { mode: true},