Build: Fix Redis/Memcached integration tests (#64298)

* Build: Fix integration cache tests * Allow REDIS_URL with scheme * Reduce cache integration tests timeout to 5m * Apply suggestion from code review * Run redis/memcached integration tests in OSS pipelines * Change redis image
2025-02-25 18:55:37 -06:00 · 2023-04-05 11:55:55 +03:00 · 2023-04-05 11:55:55 +03:00 · caac9838d8
commit caac9838d8
parent f69304fd50
10 changed files with 215 additions and 57 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -797,6 +797,12 @@ services:
  volumes:
  - name: mysql
    path: /var/lib/mysql
+- environment: {}
+  image: redis:6.2.11-alpine
+  name: redis
+- environment: {}
+  image: memcached:1.6.9-alpine
+  name: memcached
 steps:
 - commands:
  - is_fork=$(curl "https://$GITHUB_TOKEN@api.github.com/repos/grafana/grafana/pulls/$DRONE_PULL_REQUEST"
@ -892,6 +898,26 @@ steps:
    MYSQL_HOST: mysql
  image: grafana/build-container:1.7.2
  name: mysql-integration-tests
+- commands:
+  - dockerize -wait tcp://redis:6379/0 -timeout 120s
+  - go clean -testcache
+  - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...
+  depends_on:
+  - wire-install
+  environment:
+    REDIS_URL: redis://redis:6379/0
+  image: grafana/build-container:1.7.2
+  name: redis-integration-tests
+- commands:
+  - dockerize -wait tcp://memcached:11211 -timeout 120s
+  - go clean -testcache
+  - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...
+  depends_on:
+  - wire-install
+  environment:
+    MEMCACHED_HOSTS: memcached:11211
+  image: grafana/build-container:1.7.2
+  name: memcached-integration-tests
 trigger:
  event:
  - pull_request
@ -1797,6 +1823,12 @@ services:
  volumes:
  - name: mysql
    path: /var/lib/mysql
+- environment: {}
+  image: redis:6.2.11-alpine
+  name: redis
+- environment: {}
+  image: memcached:1.6.9-alpine
+  name: memcached
 steps:
 - commands:
  - mkdir -p bin
@ -1870,6 +1902,26 @@ steps:
    MYSQL_HOST: mysql
  image: grafana/build-container:1.7.2
  name: mysql-integration-tests
+- commands:
+  - dockerize -wait tcp://redis:6379/0 -timeout 120s
+  - go clean -testcache
+  - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...
+  depends_on:
+  - wire-install
+  environment:
+    REDIS_URL: redis://redis:6379/0
+  image: grafana/build-container:1.7.2
+  name: redis-integration-tests
+- commands:
+  - dockerize -wait tcp://memcached:11211 -timeout 120s
+  - go clean -testcache
+  - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...
+  depends_on:
+  - wire-install
+  environment:
+    MEMCACHED_HOSTS: memcached:11211
+  image: grafana/build-container:1.7.2
+  name: memcached-integration-tests
 trigger:
  branch: main
  event:
@ -4956,6 +5008,12 @@ services:
  volumes:
  - name: mysql
    path: /var/lib/mysql
+- environment: {}
+  image: redis:6.2.11-alpine
+  name: redis
+- environment: {}
+  image: memcached:1.6.9-alpine
+  name: memcached
 steps:
 - commands:
  - mkdir -p bin
@ -5022,6 +5080,26 @@ steps:
    MYSQL_HOST: mysql
  image: grafana/build-container:1.7.2
  name: mysql-integration-tests
+- commands:
+  - dockerize -wait tcp://redis:6379/0 -timeout 120s
+  - go clean -testcache
+  - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...
+  depends_on:
+  - wire-install
+  environment:
+    REDIS_URL: redis://redis:6379/0
+  image: grafana/build-container:1.7.2
+  name: redis-integration-tests
+- commands:
+  - dockerize -wait tcp://memcached:11211 -timeout 120s
+  - go clean -testcache
+  - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...
+  depends_on:
+  - wire-install
+  environment:
+    MEMCACHED_HOSTS: memcached:11211
+  image: grafana/build-container:1.7.2
+  name: memcached-integration-tests
 trigger:
  ref:
  - refs/heads/v[0-9]*
@ -5617,7 +5695,7 @@ services:
  - name: mysql
    path: /var/lib/mysql
 - environment: {}
-  image: redis:6.2.1-alpine
+  image: redis:6.2.11-alpine
  name: redis
 - environment: {}
  image: memcached:1.6.9-alpine
@ -5718,8 +5796,7 @@ steps:
 - commands:
  - dockerize -wait tcp://redis:6379/0 -timeout 120s
  - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...
  depends_on:
  - wire-install
  environment:
@ -5729,8 +5806,7 @@ steps:
 - commands:
  - dockerize -wait tcp://memcached:11211 -timeout 120s
  - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...
  depends_on:
  - wire-install
  environment:
@ -6112,6 +6188,12 @@ services:
  volumes:
  - name: mysql
    path: /var/lib/mysql
+- environment: {}
+  image: redis:6.2.11-alpine
+  name: redis
+- environment: {}
+  image: memcached:1.6.9-alpine
+  name: memcached
 steps:
 - commands:
  - mkdir -p bin
@ -6178,6 +6260,26 @@ steps:
    MYSQL_HOST: mysql
  image: grafana/build-container:1.7.2
  name: mysql-integration-tests
+- commands:
+  - dockerize -wait tcp://redis:6379/0 -timeout 120s
+  - go clean -testcache
+  - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...
+  depends_on:
+  - wire-install
+  environment:
+    REDIS_URL: redis://redis:6379/0
+  image: grafana/build-container:1.7.2
+  name: redis-integration-tests
+- commands:
+  - dockerize -wait tcp://memcached:11211 -timeout 120s
+  - go clean -testcache
+  - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...
+  depends_on:
+  - wire-install
+  environment:
+    MEMCACHED_HOSTS: memcached:11211
+  image: grafana/build-container:1.7.2
+  name: memcached-integration-tests
 trigger:
  event:
  - promote
@ -6230,7 +6332,7 @@ services:
  - name: mysql
    path: /var/lib/mysql
 - environment: {}
-  image: redis:6.2.1-alpine
+  image: redis:6.2.11-alpine
  name: redis
 - environment: {}
  image: memcached:1.6.9-alpine
@ -6332,8 +6434,7 @@ steps:
 - commands:
  - dockerize -wait tcp://redis:6379/0 -timeout 120s
  - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...
  depends_on:
  - wire-install
  environment:
@ -6343,8 +6444,7 @@ steps:
 - commands:
  - dockerize -wait tcp://memcached:11211 -timeout 120s
  - go clean -testcache
-  - go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic
-    -timeout=5m {}'
+  - go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...
  depends_on:
  - wire-install
  environment:
@ -6653,6 +6753,6 @@ kind: secret
 name: enterprise2_security_prefix
 ---
 kind: signature
-hmac: 675c90070f9b68e15f349d079fbafa5288c3e5c30c345a4c9678d69afd1cc5a7
+hmac: 54bded4dc7c3ffbbf3859a4f8f6364f59f3b6e0696152a0b835211eca7119e50

 ...
--- a/12
+++ b/12
@ -143,6 +143,18 @@ test-go-integration-mysql: devenv-mysql ## Run integration tests for mysql backe
 	$(GO) clean -testcache
 	$(GO) list './pkg/...' | xargs -I {} sh -c 'GRAFANA_TEST_DB=mysql go test -run Integration -covermode=atomic -timeout=2m {}'

+.PHONY: test-go-integration-redis
+test-go-integration-redis: ## Run integration tests for redis cache.
+	@echo "test backend integration redis tests"
+	$(GO) clean -testcache
+	REDIS_URL=localhost:6379 $(GO) test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...
+
+.PHONY: test-go-integration-memcached
+test-go-integration-memcached: ## Run integration tests for memcached cache.
+	@echo "test backend integration memcached tests"
+	$(GO) clean -testcache
+	MEMCACHED_HOSTS=localhost:11211 $(GO) test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...
+
 test-js: ## Run tests for frontend.
 	@echo "test frontend"
 	yarn test
--- a/pkg/infra/remotecache/memcached_storage_integration_test.go
+++ b/pkg/infra/remotecache/memcached_storage_integration_test.go
@ -1,16 +1,23 @@
-//go:build memcached
-// +build memcached
-
 package remotecache

 import (
+	"os"
 	"testing"

 	"github.com/grafana/grafana/pkg/setting"
 )

-func TestMemcachedCacheStorage(t *testing.T) {
-	opts := &setting.RemoteCacheOptions{Name: memcachedCacheType, ConnStr: "localhost:11211"}
+func TestIntegrationMemcachedCacheStorage(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	u, ok := os.LookupEnv("MEMCACHED_HOSTS")
+	if !ok || u == "" {
+		t.Skip("No Memcached hosts provided")
+	}
+
+	opts := &setting.RemoteCacheOptions{Name: memcachedCacheType, ConnStr: u}
 	client := createTestClient(t, opts, nil)
 	runTestsForClient(t, client)
 	runCountTestsForClient(t, opts, nil)
--- a/pkg/infra/remotecache/redis_storage_integration_test.go
+++ b/pkg/infra/remotecache/redis_storage_integration_test.go
@ -1,17 +1,40 @@
-//go:build redis
-// +build redis
-
 package remotecache

 import (
+	"fmt"
+	"os"
+	"strings"
 	"testing"

+	"github.com/go-redis/redis/v8"
 	"github.com/grafana/grafana/pkg/setting"
 )

-func TestRedisCacheStorage(t *testing.T) {
+func TestIntegrationRedisCacheStorage(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}

-	opts := &setting.RemoteCacheOptions{Name: redisCacheType, ConnStr: "addr=localhost:6379"}
+	u, ok := os.LookupEnv("REDIS_URL")
+	if !ok || u == "" {
+		t.Skip("No redis URL supplied")
+	}
+
+	addr := u
+	db := 0
+	parsed, err := redis.ParseURL(u)
+	if err == nil {
+		addr = parsed.Addr
+		db = parsed.DB
+	}
+
+	b := strings.Builder{}
+	b.WriteString(fmt.Sprintf("addr=%s", addr))
+	if db != 0 {
+		b.WriteString(fmt.Sprintf(",db=%d", db))
+	}
+
+	opts := &setting.RemoteCacheOptions{Name: redisCacheType, ConnStr: b.String()}
 	client := createTestClient(t, opts, nil)
 	runTestsForClient(t, client)
 	runCountTestsForClient(t, opts, nil)
--- a/pkg/infra/remotecache/remotecache_test.go
+++ b/pkg/infra/remotecache/remotecache_test.go
@ -95,7 +95,8 @@ func canPutGetAndDeleteCachedObjects(t *testing.T, client CacheStorage) {
 	assert.Equal(t, err, nil)

 	_, err = client.Get(context.Background(), "key1")
-	assert.Equal(t, err, ErrCacheItemNotFound)
+	// redis client returns redis.Nil error when key does not exist.
+	assert.Error(t, err)
 }

 func canNotFetchExpiredItems(t *testing.T, client CacheStorage) {
@ -109,7 +110,8 @@ func canNotFetchExpiredItems(t *testing.T, client CacheStorage) {

 	// should not be able to read that value since its expired
 	_, err = client.Get(context.Background(), "key1")
-	assert.Equal(t, err, ErrCacheItemNotFound)
+	// redis client returns redis.Nil error when key does not exist.
+	assert.Error(t, err)
 }

 func TestCollectUsageStats(t *testing.T) {
--- a/pkg/services/live/managedstream/cache_redis_test.go
+++ b/pkg/services/live/managedstream/cache_redis_test.go
@ -1,18 +1,34 @@
-//go:build redis
-// +build redis
-
 package managedstream

 import (
+	"os"
 	"testing"

 	"github.com/go-redis/redis/v8"
 	"github.com/stretchr/testify/require"
 )

-func TestRedisCacheStorage(t *testing.T) {
+func TestIntegrationRedisCacheStorage(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	u, ok := os.LookupEnv("REDIS_URL")
+	if !ok || u == "" {
+		t.Skip("No redis URL supplied")
+	}
+
+	addr := u
+	db := 0
+	parsed, err := redis.ParseURL(u)
+	if err == nil {
+		addr = parsed.Addr
+		db = parsed.DB
+	}
+
 	redisClient := redis.NewClient(&redis.Options{
-		Addr: "localhost:6379",
+		Addr: addr,
+		DB:   db,
 	})
 	c := NewRedisFrameCache(redisClient)
 	require.NotNil(t, c)
--- a/scripts/drone/events/release.star
+++ b/scripts/drone/events/release.star
@ -141,7 +141,7 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):

    environment = {"EDITION": "oss"}

-    services = integration_test_services(edition = "oss")
+    services = integration_test_services()
    volumes = integration_test_services_volumes()

    init_steps = [
@ -203,6 +203,8 @@ def oss_pipelines(ver_mode = ver_mode, trigger = release_trigger):
    integration_test_steps = [
        postgres_integration_tests_step(),
        mysql_integration_tests_step(),
+        redis_integration_tests_step(),
+        memcached_integration_tests_step(),
    ]

    # We don't need to run integration tests at release time since they have
@ -282,7 +284,7 @@ def enterprise_pipelines(ver_mode = ver_mode, trigger = release_trigger):

    environment = {"EDITION": "enterprise"}

-    services = integration_test_services(edition = "enterprise")
+    services = integration_test_services()
    volumes = integration_test_services_volumes()

    init_steps = [
@ -694,11 +696,9 @@ def integration_test_pipelines():
    }
    pipelines = []
    volumes = integration_test_services_volumes()
-    oss_integration_test_steps = [
+    integration_test_steps = [
        postgres_integration_tests_step(),
        mysql_integration_tests_step(),
-    ]
-    enterprise_integration_test_steps = oss_integration_test_steps + [
        redis_integration_tests_step(),
        memcached_integration_tests_step(),
    ]
@ -708,7 +708,7 @@ def integration_test_pipelines():
        name = "integration-tests-oss",
        edition = "oss",
        trigger = trigger,
-        services = integration_test_services(edition = "oss"),
+        services = integration_test_services(),
        steps = [
                    download_grabpl_step(),
                    identify_runner_step(),
@ -716,7 +716,7 @@ def integration_test_pipelines():
                    verify_gen_jsonnet_step(),
                    wire_install_step(),
                ] +
-                oss_integration_test_steps,
+                integration_test_steps,
        environment = {"EDITION": "oss"},
        volumes = volumes,
    ))
@ -725,7 +725,7 @@ def integration_test_pipelines():
        name = "integration-tests-enterprise",
        edition = "enterprise",
        trigger = trigger,
-        services = integration_test_services(edition = "enterprise"),
+        services = integration_test_services(),
        steps = [
                    download_grabpl_step(),
                    identify_runner_step(),
@ -744,7 +744,7 @@ def integration_test_pipelines():
                [
                    wire_install_step(),
                ] +
-                enterprise_integration_test_steps,
+                integration_test_steps,
        environment = {"EDITION": "enterprise"},
        volumes = volumes,
    ))
--- a/scripts/drone/pipelines/integration_tests.star
+++ b/scripts/drone/pipelines/integration_tests.star
@ -8,8 +8,10 @@ load(
    "download_grabpl_step",
    "enterprise_setup_step",
    "identify_runner_step",
+    "memcached_integration_tests_step",
    "mysql_integration_tests_step",
    "postgres_integration_tests_step",
+    "redis_integration_tests_step",
    "verify_gen_cue_step",
    "verify_gen_jsonnet_step",
    "wire_install_step",
@ -37,7 +39,7 @@ def integration_tests(trigger, prefix, ver_mode = "pr"):
    """
    environment = {"EDITION": "oss"}

-    services = integration_test_services(edition = "oss")
+    services = integration_test_services()
    volumes = integration_test_services_volumes()

    init_steps = []
@ -66,6 +68,8 @@ def integration_tests(trigger, prefix, ver_mode = "pr"):
    test_steps = [
        postgres_integration_tests_step(),
        mysql_integration_tests_step(),
+        redis_integration_tests_step(),
+        memcached_integration_tests_step(),
    ]

    return pipeline(
--- a/scripts/drone/services/services.star
+++ b/scripts/drone/services/services.star
@ -8,7 +8,7 @@ def integration_test_services_volumes():
        {"name": "mysql", "temp": {"medium": "memory"}},
    ]

-def integration_test_services(edition):
+def integration_test_services():
    services = [
        {
            "name": "postgres",
@ -34,24 +34,18 @@ def integration_test_services(edition):
            },
            "volumes": [{"name": "mysql", "path": "/var/lib/mysql"}],
        },
+        {
+            "name": "redis",
+            "image": "redis:6.2.11-alpine",
+            "environment": {},
+        },
+        {
+            "name": "memcached",
+            "image": "memcached:1.6.9-alpine",
+            "environment": {},
+        },
    ]

-    if edition in ("enterprise", "enterprise2"):
-        services.extend(
-            [
-                {
-                    "name": "redis",
-                    "image": "redis:6.2.1-alpine",
-                    "environment": {},
-                },
-                {
-                    "name": "memcached",
-                    "image": "memcached:1.6.9-alpine",
-                    "environment": {},
-                },
-            ],
-        )
-
    return services

 def ldap_service():
--- a/scripts/drone/steps/lib.star
+++ b/scripts/drone/steps/lib.star
@ -1144,7 +1144,7 @@ def redis_integration_tests_step():
        "commands": [
            "dockerize -wait tcp://redis:6379/0 -timeout 120s",
            "go clean -testcache",
-            "go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=5m {}'",
+            "go test -run IntegrationRedis -covermode=atomic -timeout=2m ./pkg/...",
        ],
    }

@ -1159,7 +1159,7 @@ def memcached_integration_tests_step():
        "commands": [
            "dockerize -wait tcp://memcached:11211 -timeout 120s",
            "go clean -testcache",
-            "go list './pkg/...' | xargs -I {} sh -c 'go test -run Integration -covermode=atomic -timeout=5m {}'",
+            "go test -run IntegrationMemcached -covermode=atomic -timeout=2m ./pkg/...",
        ],
    }