From 1ed5d87342cb37d53a9a741fbfb0d5338cad2922 Mon Sep 17 00:00:00 2001 From: Nicolas Le Cam Date: Mon, 13 Nov 2023 14:30:24 +0100 Subject: [PATCH] Switch to docker compose v2 (#25048) --- .github/workflows/esrupgrade-common.yml | 24 +++++----- .github/workflows/migration.yml | 52 +++++++++++----------- .github/workflows/mmctl-test-template.yml | 10 ++--- .github/workflows/server-test-template.yml | 10 ++--- server/Makefile | 30 ++++++------- server/build/local-test-env.sh | 16 +++---- server/scripts/replica-mysql-config.sh | 2 +- 7 files changed, 72 insertions(+), 72 deletions(-) diff --git a/.github/workflows/esrupgrade-common.yml b/.github/workflows/esrupgrade-common.yml index a4062c82d0..b32d500ae8 100644 --- a/.github/workflows/esrupgrade-common.yml +++ b/.github/workflows/esrupgrade-common.yml @@ -18,7 +18,7 @@ env: DUMP_SERVER_NAME: esr.${{ inputs.initial-version }}-${{ inputs.final-version }}.dump.server.sql DUMP_SCRIPT_NAME: esr.${{ inputs.initial-version }}-${{ inputs.final-version }}.dump.script.sql MIGRATION_SCRIPT: esr.${{ inputs.initial-version }}-${{ inputs.final-version }}.mysql.up.sql - CLEANUP_SCRIPT: esr.${{ inputs.initial-version }}-${{ inputs.final-version }}.mysql.cleanup.sql + CLEANUP_SCRIPT: esr.${{ inputs.initial-version }}-${{ inputs.final-version }}.mysql.cleanup.sql PREPROCESS_SCRIPT: esr.common.mysql.preprocess.sql DIFF_NAME: esr.${{ inputs.initial-version }}-${{ inputs.final-version }}.diff jobs: @@ -31,10 +31,10 @@ jobs: - name: Run docker compose run: | cd server/build - docker-compose --no-ansi run --rm start_dependencies - cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; - docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test'; - docker-compose --no-ansi ps + docker compose --no-ansi run --rm start_dependencies + cat ../tests/test-data.ldif | docker compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; + docker compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test'; + docker compose --no-ansi ps - name: Wait for docker compose run: | until docker network inspect ghactions_mm-test; do echo "Waiting for Docker Compose Network..."; sleep 1; done; @@ -74,7 +74,7 @@ jobs: # otherwise, the name of the container gets written to the console, which is weird docker stop mmserver > /dev/null - name: Cleanup DB - run : | + run: | cd server/scripts/esrupgrades docker exec -i ghactions_mysql_1 mysql -AN $MYSQL_CONN_ARGS < $CLEANUP_SCRIPT - name: Dump upgraded database @@ -102,10 +102,10 @@ jobs: - name: Run docker compose run: | cd server/build - docker-compose --no-ansi run --rm start_dependencies - cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; - docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test'; - docker-compose --no-ansi ps + docker compose --no-ansi run --rm start_dependencies + cat ../tests/test-data.ldif | docker compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; + docker compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test'; + docker compose --no-ansi ps - name: Wait for docker compose run: | until docker network inspect ghactions_mm-test; do echo "Waiting for Docker Compose Network..."; sleep 1; done; @@ -119,11 +119,11 @@ jobs: cd server/scripts/esrupgrades docker exec -i ghactions_mysql_1 mysql -AN $MYSQL_CONN_ARGS < $PREPROCESS_SCRIPT - name: Run migration through script - run : | + run: | cd server/scripts/esrupgrades docker exec -i ghactions_mysql_1 mysql -AN $MYSQL_CONN_ARGS < $MIGRATION_SCRIPT - name: Cleanup DB - run : | + run: | cd server/scripts/esrupgrades docker exec -i ghactions_mysql_1 mysql -AN $MYSQL_CONN_ARGS < $CLEANUP_SCRIPT - name: Dump upgraded database diff --git a/.github/workflows/migration.yml b/.github/workflows/migration.yml index 6d572996b2..d6b950542b 100644 --- a/.github/workflows/migration.yml +++ b/.github/workflows/migration.yml @@ -2,24 +2,24 @@ name: Database Migration Test on: workflow_call: jobs: - test: - name: MySQL -> Postgres Migration - runs-on: ubuntu-22.04 - env: - COMPOSE_PROJECT_NAME: ghactions - BUILD_IMAGE: mattermost/mattermost-build-server:20230904_golang-1.20.7 - defaults: - run: - working-directory: server - steps: + test: + name: MySQL -> Postgres Migration + runs-on: ubuntu-22.04 + env: + COMPOSE_PROJECT_NAME: ghactions + BUILD_IMAGE: mattermost/mattermost-build-server:20230904_golang-1.20.7 + defaults: + run: + working-directory: server + steps: - name: Checkout mattermost project uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - name: Run docker compose run: | cd build - docker-compose --ansi never run --rm start_dependencies - docker-compose --ansi never exec -T minio sh -c 'mkdir -p /data/mattermost-test'; - docker-compose --ansi never ps + docker compose --ansi never run --rm start_dependencies + docker compose --ansi never exec -T minio sh -c 'mkdir -p /data/mattermost-test'; + docker compose --ansi never ps - name: Generate test-data run: | docker run --net ${COMPOSE_PROJECT_NAME}_mm-test \ @@ -33,22 +33,22 @@ jobs: make test-data - name: Migrate the DB and compare run: | - docker run --net ${COMPOSE_PROJECT_NAME}_mm-test \ - --ulimit nofile=8096:8096 \ - --env-file=build/dotenv/migration.env \ - -v $(go env GOCACHE):/go/cache \ - -e GOCACHE=/go/cache \ - -v $PWD:/mattermost \ - -w /mattermost \ - $BUILD_IMAGE \ - make test-migration + docker run --net ${COMPOSE_PROJECT_NAME}_mm-test \ + --ulimit nofile=8096:8096 \ + --env-file=build/dotenv/migration.env \ + -v $(go env GOCACHE):/go/cache \ + -e GOCACHE=/go/cache \ + -v $PWD:/mattermost \ + -w /mattermost \ + $BUILD_IMAGE \ + make test-migration - name: Upload artifacts uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 with: - name: Migration logs - path: server/migration.log - retention-days: 7 + name: Migration logs + path: server/migration.log + retention-days: 7 - name: Stop docker compose run: | cd build - docker-compose --ansi never stop + docker compose --ansi never stop diff --git a/.github/workflows/mmctl-test-template.yml b/.github/workflows/mmctl-test-template.yml index 73ed28feae..2c7247011c 100644 --- a/.github/workflows/mmctl-test-template.yml +++ b/.github/workflows/mmctl-test-template.yml @@ -45,10 +45,10 @@ jobs: - name: Run docker compose run: | cd server/build - docker-compose --ansi never run --rm start_dependencies - cat ../tests/test-data.ldif | docker-compose --ansi never exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; - docker-compose --ansi never exec -T minio sh -c 'mkdir -p /data/mattermost-test'; - docker-compose --ansi never ps + docker compose --ansi never run --rm start_dependencies + cat ../tests/test-data.ldif | docker compose --ansi never exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; + docker compose --ansi never exec -T minio sh -c 'mkdir -p /data/mattermost-test'; + docker compose --ansi never ps - name: Run mmctl Tests run: | if [[ ${{ github.ref_name }} == 'master' ]]; then @@ -70,7 +70,7 @@ jobs: - name: Stop docker compose run: | cd server/build - docker-compose --ansi never stop + docker compose --ansi never stop - name: Archive logs if: ${{ always() }} uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 diff --git a/.github/workflows/server-test-template.yml b/.github/workflows/server-test-template.yml index 70587af73c..a5464f8698 100644 --- a/.github/workflows/server-test-template.yml +++ b/.github/workflows/server-test-template.yml @@ -37,10 +37,10 @@ jobs: - name: Run docker compose run: | cd server/build - docker-compose --ansi never run --rm start_dependencies - cat ../tests/test-data.ldif | docker-compose --ansi never exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; - docker-compose --ansi never exec -T minio sh -c 'mkdir -p /data/mattermost-test'; - docker-compose --ansi never ps + docker compose --ansi never run --rm start_dependencies + cat ../tests/test-data.ldif | docker compose --ansi never exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; + docker compose --ansi never exec -T minio sh -c 'mkdir -p /data/mattermost-test'; + docker compose --ansi never ps - name: Run Tests run: | if [[ ${{ github.ref_name }} == 'master' ]]; then @@ -62,7 +62,7 @@ jobs: - name: Stop docker compose run: | cd server/build - docker-compose --ansi never stop + docker compose --ansi never stop - name: Archive logs if: ${{ always() }} uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 diff --git a/server/Makefile b/server/Makefile index ce3d8a80be..8fcf300196 100644 --- a/server/Makefile +++ b/server/Makefile @@ -230,10 +230,10 @@ else ifeq ($(MM_NO_DOCKER),true) else @echo Starting docker containers - docker-compose rm start_dependencies - $(GO) run ./build/docker-compose-generator/main.go $(ENABLED_DOCKER_SERVICES) | docker-compose -f docker-compose.makefile.yml -f /dev/stdin $(DOCKER_COMPOSE_OVERRIDE) run -T --rm start_dependencies + docker compose rm start_dependencies + $(GO) run ./build/docker-compose-generator/main.go $(ENABLED_DOCKER_SERVICES) | docker compose -f docker-compose.makefile.yml -f /dev/stdin $(DOCKER_COMPOSE_OVERRIDE) run -T --rm start_dependencies ifneq (,$(findstring openldap,$(ENABLED_DOCKER_SERVICES))) - cat tests/${LDAP_DATA}-data.ldif | docker-compose -f docker-compose.makefile.yml $(DOCKER_COMPOSE_OVERRIDE) exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest || true'; + cat tests/${LDAP_DATA}-data.ldif | docker compose -f docker-compose.makefile.yml $(DOCKER_COMPOSE_OVERRIDE) exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest || true'; endif ifneq (,$(findstring mysql-read-replica,$(ENABLED_DOCKER_SERVICES))) ./scripts/replica-mysql-config.sh @@ -243,18 +243,18 @@ endif update-docker: stop-docker ## Updates the docker containers for local development. @echo Updating docker containers - $(GO) run ./build/docker-compose-generator/main.go $(ENABLED_DOCKER_SERVICES) | docker-compose -f docker-compose.makefile.yml -f /dev/stdin $(DOCKER_COMPOSE_OVERRIDE) up --no-start + $(GO) run ./build/docker-compose-generator/main.go $(ENABLED_DOCKER_SERVICES) | docker compose -f docker-compose.makefile.yml -f /dev/stdin $(DOCKER_COMPOSE_OVERRIDE) up --no-start run-haserver: ifeq ($(BUILD_ENTERPRISE_READY),true) @echo Starting mattermost in an HA topology '(3 node cluster)' - docker-compose -f docker-compose.yaml $(DOCKER_COMPOSE_OVERRIDE) up --remove-orphans haproxy + docker compose -f docker-compose.yaml $(DOCKER_COMPOSE_OVERRIDE) up --remove-orphans haproxy endif stop-haserver: @echo Stopping docker containers for HA topology - docker-compose stop + docker compose stop stop-docker: ## Stops the docker containers for local development. ifeq ($(MM_NO_DOCKER),true) @@ -262,7 +262,7 @@ ifeq ($(MM_NO_DOCKER),true) else @echo Stopping docker containers - docker-compose stop + docker compose stop endif clean-docker: ## Deletes the docker containers for local development. @@ -271,8 +271,8 @@ ifeq ($(MM_NO_DOCKER),true) else @echo Removing docker containers - docker-compose down -v - docker-compose rm -v + docker compose down -v + docker compose rm -v endif plugin-checker: @@ -455,7 +455,7 @@ ifneq ($(IS_CI),true) ifneq ($(MM_NO_DOCKER),true) ifneq ($(TEMP_DOCKER_SERVICES),) @echo Stopping temporary docker services - docker-compose stop $(TEMP_DOCKER_SERVICES) + docker compose stop $(TEMP_DOCKER_SERVICES) endif endif endif @@ -470,7 +470,7 @@ ifneq ($(IS_CI),true) ifneq ($(MM_NO_DOCKER),true) ifneq ($(TEMP_DOCKER_SERVICES),) @echo Stopping temporary docker services - docker-compose stop $(TEMP_DOCKER_SERVICES) + docker compose stop $(TEMP_DOCKER_SERVICES) endif endif endif @@ -658,10 +658,10 @@ restart-server: | stop-server run-server ## Restarts the mattermost server to pi restart-haserver: @echo Restarting mattermost in an HA topology - docker-compose restart follower2 - docker-compose restart follower - docker-compose restart leader - docker-compose restart haproxy + docker compose restart follower2 + docker compose restart follower + docker compose restart leader + docker compose restart haproxy restart-client: | stop-client run-client ## Restarts the webapp. diff --git a/server/build/local-test-env.sh b/server/build/local-test-env.sh index e53844b17b..3650710dce 100755 --- a/server/build/local-test-env.sh +++ b/server/build/local-test-env.sh @@ -14,14 +14,14 @@ USAGE up() { - docker-compose run --rm start_dependencies + docker compose run --rm start_dependencies - docker-compose exec openldap bash -c 'echo -e "dn: ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: organizationalunit" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' - docker-compose exec openldap bash -c 'echo -e "dn: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test1\nmail: success+testone@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' - docker-compose exec openldap bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" -w mostest' - docker-compose exec openldap bash -c 'echo -e "dn: uid=test.two,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test2\nmail: success+testtwo@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' - docker-compose exec openldap bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.two,ou=testusers,dc=mm,dc=test,dc=com" -w mostest' - docker-compose exec openldap bash -c 'echo -e "dn: cn=tgroup,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: groupOfUniqueNames\nuniqueMember: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' + docker compose exec openldap bash -c 'echo -e "dn: ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: organizationalunit" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' + docker compose exec openldap bash -c 'echo -e "dn: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test1\nmail: success+testone@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' + docker compose exec openldap bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" -w mostest' + docker compose exec openldap bash -c 'echo -e "dn: uid=test.two,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test2\nmail: success+testtwo@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' + docker compose exec openldap bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.two,ou=testusers,dc=mm,dc=test,dc=com" -w mostest' + docker compose exec openldap bash -c 'echo -e "dn: cn=tgroup,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: groupOfUniqueNames\nuniqueMember: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' docker run -it -u root \ --privileged \ @@ -37,7 +37,7 @@ up() down() { - docker-compose down + docker compose down } # process arguments diff --git a/server/scripts/replica-mysql-config.sh b/server/scripts/replica-mysql-config.sh index 6728e44e3a..5ef09379ff 100755 --- a/server/scripts/replica-mysql-config.sh +++ b/server/scripts/replica-mysql-config.sh @@ -9,7 +9,7 @@ done priv_stmt='GRANT REPLICATION SLAVE ON *.* TO "mmuser"@"%" IDENTIFIED BY "mostest"; FLUSH PRIVILEGES;' docker exec mattermost-mysql sh -c "mysql -u root -pmostest -e '$priv_stmt'" -until docker-compose -f docker-compose.makefile.yml exec mysql-read-replica sh -c 'mysql -u root -pmostest -e ";"' +until docker compose -f docker-compose.makefile.yml exec mysql-read-replica sh -c 'mysql -u root -pmostest -e ";"' do echo "Waiting for mysql-read-replica database connection..." sleep 4