Chore: Try a different way to run integration tests (#66653)

* try a different way to run integration tests * fix formatting * clean test cache again * use previous command * playing around with random commands * dont run tests in parallel yet * use parallel option instead of gomaxprocs * use same package set for redis/memcached; use p=1
2025-02-25 18:55:37 -06:00 · 2023-04-17 18:33:43 +02:00 · 2023-04-17 18:33:43 +02:00 · 391a192310
commit 391a192310
parent 8ffd58a855
3 changed files with 67 additions and 42 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -336,7 +336,9 @@ steps:
  image: grafana/build-container:1.7.3
  name: test-backend
 - commands:
-  - go test -run Integration -covermode=atomic -timeout=5m ./pkg/...
+  - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
  - wire-install
  image: grafana/build-container:1.7.3
@ -872,8 +874,9 @@ steps:
  - dockerize -wait tcp://postgres:5432 -timeout 120s
  - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
  - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
  - wire-install
  environment:
@ -889,8 +892,9 @@ steps:
  - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root
    -prootpass
  - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
  - wire-install
  environment:
@ -1291,7 +1295,9 @@ steps:
  image: grafana/build-container:1.7.3
  name: test-backend
 - commands:
-  - go test -run Integration -covermode=atomic -timeout=5m ./pkg/...
+  - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
  - wire-install
  image: grafana/build-container:1.7.3
@ -1876,8 +1882,9 @@ steps:
  - dockerize -wait tcp://postgres:5432 -timeout 120s
  - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
  - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
  - wire-install
  environment:
@ -1893,8 +1900,9 @@ steps:
  - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root
    -prootpass
  - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
  - wire-install
  environment:
@ -2545,7 +2553,9 @@ steps:
  image: grafana/build-container:1.7.3
  name: test-backend
 - commands:
-  - go test -run Integration -covermode=atomic -timeout=5m ./pkg/...
+  - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
  - wire-install
  image: grafana/build-container:1.7.3
@ -3107,7 +3117,9 @@ steps:
  image: grafana/build-container:1.7.3
  name: test-backend
 - commands:
-  - go test -run Integration -covermode=atomic -timeout=5m ./pkg/...
+  - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
  - wire-install
  image: grafana/build-container:1.7.3
@ -4992,7 +5004,9 @@ steps:
  image: grafana/build-container:1.7.3
  name: test-backend
 - commands:
-  - go test -run Integration -covermode=atomic -timeout=5m ./pkg/...
+  - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
  - wire-install
  image: grafana/build-container:1.7.3
@ -5087,8 +5101,9 @@ steps:
  - dockerize -wait tcp://postgres:5432 -timeout 120s
  - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
  - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
  - wire-install
  environment:
@ -5104,8 +5119,9 @@ steps:
  - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root
    -prootpass
  - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
  - wire-install
  environment:
@ -5678,7 +5694,9 @@ steps:
  image: grafana/build-container:1.7.3
  name: test-backend
 - commands:
-  - go test -run Integration -covermode=atomic -timeout=5m ./pkg/...
+  - go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
  - wire-install
  image: grafana/build-container:1.7.3
@ -5800,8 +5818,9 @@ steps:
  - dockerize -wait tcp://postgres:5432 -timeout 120s
  - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
  - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
  - wire-install
  environment:
@ -5817,8 +5836,9 @@ steps:
  - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root
    -prootpass
  - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
  - wire-install
  environment:
@ -6267,8 +6287,9 @@ steps:
  - dockerize -wait tcp://postgres:5432 -timeout 120s
  - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
  - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
  - wire-install
  environment:
@ -6284,8 +6305,9 @@ steps:
  - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root
    -prootpass
  - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
  - wire-install
  environment:
@ -6438,8 +6460,9 @@ steps:
  - dockerize -wait tcp://postgres:5432 -timeout 120s
  - psql -p 5432 -h postgres -U grafanatest -d grafanatest -f devenv/docker/blocks/postgres_tests/setup.sql
  - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
  - wire-install
  environment:
@ -6455,8 +6478,9 @@ steps:
  - cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root
    -prootpass
  - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find
+    ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+'
+    | grep -o '\(.*\)/' | sort -u)
  depends_on:
  - wire-install
  environment:
@ -6786,6 +6810,6 @@ kind: secret
 name: enterprise2_security_prefix
 ---
 kind: signature
-hmac: da82099e984231d3b292921e031655723eae03f45efac84af1a5a2104601fbfe
+hmac: d3567c954d0ac9f1f47d55bc85eb6594379c888fe591428f2dcd0146f2bd3ca2

 ...
--- a/15
+++ b/15
@ -18,6 +18,8 @@ GO_BUILD_FLAGS += $(if $(GO_BUILD_TAGS),-build-tags=$(GO_BUILD_TAGS))

 targets := $(shell echo '$(sources)' | tr "," " ")

+GO_INTEGRATION_TESTS := $(shell find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\(.*\)/' | sort -u)
+
 all: deps build

 ##@ Dependencies
@ -129,31 +131,32 @@ test-go-unit: ## Run unit tests for backend with flags.
 .PHONY: test-go-integration
 test-go-integration: ## Run integration tests for backend with flags.
 	@echo "test backend integration tests"
-	$(GO) test -run Integration -covermode=atomic -timeout=30m ./pkg/...
+	$(GO) test -count=1 -run "^TestIntegration" -covermode=atomic -timeout=5m $(GO_INTEGRATION_TESTS)

 .PHONY: test-go-integration-postgres
 test-go-integration-postgres: devenv-postgres ## Run integration tests for postgres backend with flags.
 	@echo "test backend integration postgres tests"
 	$(GO) clean -testcache
-	$(GO) list './pkg/...' | xargs -I {} sh -c 'GRAFANA_TEST_DB=postgres go test -run Integration -covermode=atomic -timeout=2m {}'
+	GRAFANA_TEST_DB=postgres \
+	$(GO) test -p=1 -count=1 -run "^TestIntegration" -covermode=atomic -timeout=10m $(GO_INTEGRATION_TESTS)

 .PHONY: test-go-integration-mysql
 test-go-integration-mysql: devenv-mysql ## Run integration tests for mysql backend with flags.
 	@echo "test backend integration mysql tests"
-	$(GO) clean -testcache
-	$(GO) list './pkg/...' | xargs -I {} sh -c 'GRAFANA_TEST_DB=mysql go test -run Integration -covermode=atomic -timeout=2m {}'
+	GRAFANA_TEST_DB=mysql \
+	$(GO) test -p=1 -count=1 -run "^TestIntegration" -covermode=atomic -timeout=10m $(GO_INTEGRATION_TESTS)

 .PHONY: test-go-integration-redis
 test-go-integration-redis: ## Run integration tests for redis cache.
 	@echo "test backend integration redis tests"
 	$(GO) clean -testcache
-	REDIS_URL=localhost:6379 $(GO) test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...
+	REDIS_URL=localhost:6379 $(GO) test -run IntegrationRedis -covermode=atomic -timeout=2m $(GO_INTEGRATION_TESTS)

 .PHONY: test-go-integration-memcached
 test-go-integration-memcached: ## Run integration tests for memcached cache.
 	@echo "test backend integration memcached tests"
 	$(GO) clean -testcache
-	MEMCACHED_HOSTS=localhost:11211 $(GO) test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...
+	MEMCACHED_HOSTS=localhost:11211 $(GO) test -run IntegrationMemcached -covermode=atomic -timeout=2m $(GO_INTEGRATION_TESTS)

 test-js: ## Run tests for frontend.
 	@echo "test frontend"
--- a/scripts/drone/steps/lib.star
+++ b/scripts/drone/steps/lib.star
@ -623,7 +623,7 @@ def test_backend_integration_step():
            "wire-install",
        ],
        "commands": [
-            "go test -run Integration -covermode=atomic -timeout=5m ./pkg/...",
+            "go test -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\\(.*\\)/' | sort -u)",
        ],
    }

@ -1096,9 +1096,8 @@ def postgres_integration_tests_step():
        "dockerize -wait tcp://postgres:5432 -timeout 120s",
        "psql -p 5432 -h postgres -U grafanatest -d grafanatest -f " +
        "devenv/docker/blocks/postgres_tests/setup.sql",
-        # Make sure that we don't use cached results for another database
        "go clean -testcache",
-        "go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=5m {}'",
+        "go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\\(.*\\)/' | sort -u)",
    ]
    return {
        "name": "postgres-integration-tests",
@ -1118,9 +1117,8 @@ def mysql_integration_tests_step():
        "apt-get install -yq default-mysql-client",
        "dockerize -wait tcp://mysql:3306 -timeout 120s",
        "cat devenv/docker/blocks/mysql_tests/setup.sql | mysql -h mysql -P 3306 -u root -prootpass",
-        # Make sure that we don't use cached results for another database
        "go clean -testcache",
-        "go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=5m {}'",
+        "go test -p=1 -count=1 -covermode=atomic -timeout=5m -run '^TestIntegration' $(find ./pkg -type f -name '*_test.go' -exec grep -l '^func TestIntegration' '{}' '+' | grep -o '\\(.*\\)/' | sort -u)",
    ]
    return {
        "name": "mysql-integration-tests",