IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2025-02-25 18:55:24 -06:00
Code Issues Packages Projects Releases Wiki Activity

Migrate to CI pipeline to GitHub actions (#22392)

Browse Source 
* migrate to CI pipeline to github actions

* simplfy workflow

---------

Co-authored-by: Akis Maziotis <36594156+phoinixgrr@users.noreply.github.com>
This commit is contained in:
Ibrahim Serdar Acikgoz
2023-03-02 16:12:24 +03:00
committed by GitHub
parent 8c34e6c80c
commit 931142b640
3 changed files with 339 additions and 801 deletions
Download Patch File Download Diff File Expand all files Collapse all files

801
.circleci/config.yml
View File

@@ -1,801 +0,0 @@
version: 2.1
orbs:
aws-s3: circleci/aws-s3@3.0.0
executors:
ubuntu:
working_directory: ~/mattermost/
resource_class: 2xlarge
machine:
image: "ubuntu-2004:202201-02"
environment:
COMPOSE_PROJECT_NAME: "circleci"
ubuntu_xlarge:
working_directory: ~/mattermost/
resource_class: xlarge
machine:
image: "ubuntu-2004:202201-02"
environment:
COMPOSE_PROJECT_NAME: "circleci"
jobs:
setup-multi-product-repositories:
parameters:
target-branch:
type: string
default: '$(curl -s $(echo https://api.github.com/repos/${CIRCLE_PULL_REQUEST:19} | sed "s/\/pull\//\/pulls\//") | jq ".base.ref" | tr -d "\042" )'
working_directory: /mnt/ramdisk/mattermost-server
docker:
- image: cimg/go:1.18
resource_class: large
steps:
- checkout
- run: |
cd ..
git clone --depth=1 --no-single-branch https://github.com/mattermost/focalboard.git
cd focalboard
git checkout $CIRCLE_BRANCH || git checkout <<parameters.target-branch>> || git checkout rolling-stable
echo $(git rev-parse HEAD)
cd ../mattermost-server
make setup-go-work
cd ../focalboard
EXCLUDE_ENTERPRISE=TRUE make templates-archive
- persist_to_workspace:
root: /mnt/ramdisk
paths:
- mattermost-server
- focalboard
setup-mattermost-webapp:
working_directory: /mnt/ramdisk/
docker:
- image: mattermost/mattermost-build-webapp:20220802_node-16.10.0@sha256:3272aa759f10c2ef1719ed08cc82ddb07224bec5be86f09800c72f5e2a623c3d
resource_class: xlarge
steps:
- run: |
git clone --depth=1 --no-single-branch https://github.com/mattermost/mattermost-webapp.git
cd mattermost-webapp
git checkout $CIRCLE_BRANCH || git checkout master
export WEBAPP_GIT_COMMIT=$(git rev-parse HEAD)
echo "$WEBAPP_GIT_COMMIT"
FILE_DIST=dist.tar.gz
curl --version
runtime="2 minute"
endtime=$(date -ud "$runtime" +%s)
while [[ $(date -u +%s) -le $endtime ]]; do
if curl -s --max-time 30 -f -o $FILE_DIST https://pr-builds.mattermost.com/mattermost-webapp/commit/$WEBAPP_GIT_COMMIT/mattermost-webapp.tar.gz; then
break
fi
echo "Waiting for webapp git commit $WEBAPP_GIT_COMMIT with sleep 5: `date +%H:%M:%S`"
sleep 5
done
if [[ -f "$FILE_DIST" ]]; then
echo "Precompiled version of web app found"
mkdir dist && tar -xf $FILE_DIST -C dist --strip-components=1
else
echo "Building web app from source"
make dist
fi
- persist_to_workspace:
root: /mnt/ramdisk
paths:
- mattermost-webapp
setup-focalboard-product:
working_directory: /mnt/ramdisk/
docker:
- image: cimg/go:1.18-node
resource_class: large
steps:
- attach_workspace:
at: /mnt/ramdisk
- run: |
cd focalboard
echo "Building Boards product for web app"
# make prebuild build-product # TODO figure out how to get this to run without bypassing the Makefile
make prebuild
cd mattermost-plugin/webapp
npm run build:product
cd ../..
- persist_to_workspace:
root: /mnt/ramdisk
paths:
- focalboard/mattermost-plugin/webapp/dist
check-app-layers:
docker:
- image: cimg/go:1.18
resource_class: medium
working_directory: /mnt/ramdisk
steps:
- attach_workspace:
at: /mnt/ramdisk
- run:
command: |
cd mattermost-server
make app-layers
if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi
check-generate-worktemplates:
docker:
- image: cimg/go:1.18
resource_class: medium
working_directory: /mnt/ramdisk
steps:
- attach_workspace:
at: /mnt/ramdisk
- run:
command: |
cd mattermost-server
make generate-worktemplates
if [[ -n $(git status --porcelain) ]]; then echo "Please update the worktemplates using make generate-worktemplates"; exit 1; fi
check-go-mod-tidy:
docker:
- image: cimg/go:1.18
resource_class: medium+
working_directory: /mnt/ramdisk
steps:
- attach_workspace:
at: /mnt/ramdisk
- run:
command: |
cd mattermost-server
make modules-tidy
if [[ -n $(git status --porcelain) ]]; then echo "Please tidy up the Go modules using make modules-tidy"; git diff; exit 1; fi
check-store-layers:
docker:
- image: cimg/go:1.18
resource_class: medium
working_directory: /mnt/ramdisk
steps:
- attach_workspace:
at: /mnt/ramdisk
- run:
command: |
cd mattermost-server
make store-layers
if [[ -n $(git status --porcelain) ]]; then echo "Please update the store layers using make store-layers"; exit 1; fi
check-mocks:
docker:
- image: cimg/go:1.18
resource_class: xlarge
working_directory: /mnt/ramdisk
steps:
- attach_workspace:
at: /mnt/ramdisk
- run:
command: |
cd mattermost-server
make mocks
if [[ -n $(git status --porcelain) ]]; then echo "Please update the mocks using `make mocks`"; exit 1; fi
check-email-templates:
docker:
- image: cimg/go:1.18-node
resource_class: medium+
working_directory: /mnt/ramdisk
steps:
- attach_workspace:
at: /mnt/ramdisk
- run:
command: |
cd mattermost-server
sudo npm install -g mjml@4.9.0
make build-templates
if [[ -n $(git status --porcelain) ]]; then echo "Please update the email templates using `make build-templates`"; exit 1; fi
check-gen-serialized:
docker:
- image: cimg/go:1.18
resource_class: small
working_directory: /mnt/ramdisk
steps:
- attach_workspace:
at: /mnt/ramdisk
- run:
command: |
cd mattermost-server
make gen-serialized
if [[ -n $(git status --porcelain) ]]; then echo "Please update the serialized files using 'make gen-serialized'"; exit 1; fi
check-migrations:
docker:
- image: cimg/go:1.18
resource_class: small
working_directory: /mnt/ramdisk
steps:
- attach_workspace:
at: /mnt/ramdisk
- run:
command: |
cd mattermost-server
make migrations-extract
if [[ -n $(git status --porcelain) ]]; then echo "Please update the migrations using make migrations-extract"; exit 1; fi
# Dedicated job for golangci-lint it does not need anything only the server code for that
# and to make more clear when the job fails
check-golangci-lint:
docker:
# Keep the version in sync with the command in Makefile
- image: golangci/golangci-lint:v1.50.1
resource_class: xlarge
working_directory: /mnt/ramdisk
steps:
- attach_workspace:
at: /mnt/ramdisk
- run:
command: |
cd mattermost-server
go vet ./...
make golangci-lint
# Dedicate job for mattermost-vet to make more clear when the job fails
check-mattermost-vet:
docker:
- image: mattermost/mattermost-build-server:20230118_golang-1.19.5
resource_class: xlarge
working_directory: /mnt/ramdisk
steps:
- attach_workspace:
at: /mnt/ramdisk
- run:
command: |
export GOBIN=${PWD}/mattermost-server/bin
export PATH=$PATH:$GOBIN
cd mattermost-server
make config-reset
make plugin-checker
make vet BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}' MM_NO_ENTERPRISE_LINT=true MM_VET_OPENSPEC_PATH='${PWD}/../mattermost-api-reference/v4/html/static/mattermost-openapi-v4.yaml'
build-api-spec:
docker:
- image: cimg/node:lts
working_directory: /mnt/ramdisk
steps:
- attach_workspace:
at: /mnt/ramdisk
- run:
command: |
git clone --depth=1 --no-single-branch https://github.com/mattermost/mattermost-api-reference.git
cd mattermost-api-reference
echo "Trying to checkout the same branch on mattermost-api-reference as mattermost-server"
git checkout ${CIRCLE_BRANCH} || true
make build
build-focalboard:
docker:
- image: mattermost/mattermost-build-server:20230118_golang-1.19.5
resource_class: medium
working_directory: /mnt/ramdisk
steps:
- attach_workspace:
at: /mnt/ramdisk
- run:
command: |
cd focalboard
make server-linux
build-mattermost-server:
docker:
- image: mattermost/mattermost-build-server:20230118_golang-1.19.5
resource_class: xlarge
working_directory: /mnt/ramdisk
steps:
- attach_workspace:
at: /mnt/ramdisk
- run:
command: |
cd mattermost-server
make config-reset
make build-cmd BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}'
make package BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}'
- store_artifacts:
path: /mnt/ramdisk/mattermost-server/dist/mattermost-team-linux-amd64.tar.gz
- store_artifacts:
path: /mnt/ramdisk/mattermost-server/dist/mattermost-team-osx-amd64.tar.gz
- store_artifacts:
path: /mnt/ramdisk/mattermost-server/dist/mattermost-team-linux-arm64.tar.gz
- store_artifacts:
path: /mnt/ramdisk/mattermost-server/dist/mattermost-team-osx-arm64.tar.gz
- store_artifacts:
path: /mnt/ramdisk/mattermost-server/dist/mattermost-team-windows-amd64.zip
- persist_to_workspace:
root: /mnt/ramdisk
paths:
- mattermost-server/dist
upload-artifacts-to-s3:
docker:
- image: cimg/python:3.10
working_directory: /mnt/ramdisk/
steps:
- attach_workspace:
at: /mnt/ramdisk/
- run:
command: |
cd /mnt/ramdisk/mattermost-server/dist/
rm -rf mattermost
- aws-s3/copy:
from: /mnt/ramdisk/mattermost-server/dist/
to: "s3://pr-builds.mattermost.com/mattermost-server/commit/${CIRCLE_SHA1}/"
arguments: --acl public-read --cache-control "no-cache" --recursive
- aws-s3/copy:
from: /mnt/ramdisk/mattermost-server/dist/
to: s3://pr-builds.mattermost.com/mattermost-server/$(echo "${CIRCLE_BRANCH}" | sed 's/pull\//PR-/g')/
arguments: --acl public-read --cache-control "no-cache" --recursive
build-docker:
working_directory: /mnt/ramdisk/
docker:
- image: cimg/base:stable
steps:
- attach_workspace:
at: /mnt/ramdisk/
- setup_remote_docker:
version: 20.10.7
- run:
command: |
export TAG="${CIRCLE_SHA1:0:7}"
cd mattermost-server/build
export DOCKER_CLI_EXPERIMENTAL=enabled
echo $DOCKER_PASSWORD | docker login --username $DOCKER_USERNAME --password-stdin
export MM_PACKAGE=https://pr-builds.mattermost.com/mattermost-server/commit/${CIRCLE_SHA1}/mattermost-team-linux-amd64.tar.gz
docker buildx build --push --build-arg MM_PACKAGE=$MM_PACKAGE -t mattermost/mm-te-test:${TAG} .
# Dedicate job for sentry it does not need anything only the server code for that
# and to make more clear when the job fails
sentry:
docker:
- image: getsentry/sentry-cli:1.64.1
working_directory: /mnt/ramdisk/
steps:
- checkout
- run:
command: apk add git
- run:
command: |
sentry-cli --log-level=debug releases new --finalize -p mattermost-server `git rev-parse HEAD`
sentry-cli --log-level=debug releases set-commits --auto `git rev-parse HEAD`
test:
executor:
name: ubuntu
parameters:
dbdriver:
type: string
dbsource:
type: string
racemode:
type: string
steps:
- attach_workspace:
at: ~/mattermost
- run:
name: Run Docker compose
command: |
cd mattermost-server/build
docker-compose --no-ansi run --rm start_dependencies
cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test';
docker-compose --no-ansi ps
background: true
- run:
name: Wait for docker compose
command: |
until docker network inspect circleci_mm-test; do echo "Waiting for Docker Compose Network..."; sleep 1; done;
docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;"
docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
- run:
name: Run Tests
command: |
mkdir -p mattermost-server/client/plugins
cd mattermost-server/build
docker run -it --net circleci_mm-test \
--env-file=dotenv/test.env \
--env MM_SQLSETTINGS_DATASOURCE="<<parameters.dbsource>>" \
--env MM_SQLSETTINGS_DRIVERNAME=<<parameters.dbdriver>> \
--env CIRCLECI=true \
-v ~/mattermost:/mattermost \
-w /mattermost/mattermost-server \
mattermost/mattermost-build-server:20230118_golang-1.19.5 \
bash -c "ulimit -n 8096; make test-server<< parameters.racemode >> BUILD_NUMBER=$CIRCLE_BRANCH-$CIRCLE_PREVIOUS_BUILD_NUM TESTFLAGS= TESTFLAGSEE=" \
bash -c scripts/diff-email-templates.sh
no_output_timeout: 2h
- run:
name: Capture docker logs
when: always
command: |
cd mattermost-server/build
# Capture docker logs
mkdir -p logs
docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs
docker ps -a --no-trunc > logs/docker_ps
docker stats -a --no-stream > logs/docker_stats
tar -czvf logs/docker_logs.tar.gz logs/docker-compose_logs logs/docker_ps logs/docker_stats
- store_artifacts:
path: ~/mattermost/mattermost-server/build/logs
- run:
when: always
command: |
cd mattermost-server
mkdir -p test-results
cp report.xml test-results
- store_test_results:
path: ~/mattermost/mattermost-server/test-results/
- store_artifacts:
path: ~/mattermost/mattermost-server/test-results/
test-schema:
executor:
name: ubuntu_xlarge
steps:
- attach_workspace:
at: ~/mattermost
- run:
name: Run Docker compose
command: |
cd mattermost-server/build
docker-compose --no-ansi run --rm start_dependencies
cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test';
docker-compose --no-ansi ps
background: true
- run:
name: Wait for docker compose
command: |
until docker network inspect circleci_mm-test; do echo "Waiting for Docker Compose Network..."; sleep 1; done;
docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;"
docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
- run:
name: Postgres schema migration validation
command: |
cd mattermost-server/build
mkdir -p mattermost-server/client/plugins
echo "Creating databases"
docker-compose --no-ansi exec -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test'
echo "Importing postgres dump from version 6.0.0"
docker-compose --no-ansi exec -T postgres psql -U mmuser -d migrated < ../scripts/mattermost-postgresql-6.0.0.sql
docker-compose --no-ansi exec -T postgres psql -U mmuser -d migrated -c "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')"
docker run -it --net circleci_mm-test \
--env-file="dotenv/test-schema-validation.env" \
--env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10" \
--env MM_SQLSETTINGS_DRIVERNAME=postgres \
-v ~/mattermost:/mattermost \
-w /mattermost/mattermost-server \
mattermost/mattermost-build-server:20230118_golang-1.19.5 \
bash -c "ulimit -n 8096; make ARGS='db migrate' run-cli && make MM_SQLSETTINGS_DATASOURCE='postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10' ARGS='db migrate' run-cli"
echo "Generating dump"
docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql
docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql
echo "Removing databases created for db comparison"
docker-compose --no-ansi exec -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test'
echo "Generating diff"
diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
no_output_timeout: 1h
- run:
name: MySQL schema migration validation
command: |
cd mattermost-server/build
mkdir -p mattermost-server/client/plugins
echo "Creating databases"
docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e "CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser"
echo "Importing mysql dump from version 6.0.0"
docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest < ../scripts/mattermost-mysql-6.0.0.sql
docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest -e "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')"
docker run -it --net circleci_mm-test \
--env-file="dotenv/test-schema-validation.env" \
--env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" \
--env MM_SQLSETTINGS_DRIVERNAME=mysql \
-v ~/mattermost:/mattermost \
-w /mattermost/mattermost-server \
mattermost/mattermost-build-server:20230118_golang-1.19.5 \
bash -c "ulimit -n 8096; make ARGS='db migrate' run-cli && make MM_SQLSETTINGS_DATASOURCE='mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s' ARGS='db migrate' run-cli"
echo "Generating dump"
docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql
docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql
echo "Removing databases created for db comparison"
docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest'
echo "Generating diff"
diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
no_output_timeout: 1h
workflows:
version: 2
untagged-build:
jobs:
- setup-multi-product-repositories:
context:
- matterbuild-github-token
filters:
branches:
ignore:
- master
- /^release-.*/
- setup-mattermost-webapp:
filters:
branches:
ignore:
- master
- /^release-.*/
- setup-focalboard-product:
filters:
branches:
ignore:
- master
- /^release-.*/
requires:
- setup-multi-product-repositories
- check-go-mod-tidy:
requires:
- setup-multi-product-repositories
- check-golangci-lint:
requires:
- setup-multi-product-repositories
- check-app-layers:
requires:
- setup-multi-product-repositories
- check-generate-worktemplates:
requires:
- setup-multi-product-repositories
- check-store-layers:
requires:
- setup-multi-product-repositories
- check-mocks:
requires:
- setup-multi-product-repositories
- check-email-templates:
requires:
- setup-multi-product-repositories
- check-gen-serialized:
requires:
- setup-multi-product-repositories
- check-mattermost-vet:
requires:
- setup-multi-product-repositories
- check-migrations:
requires:
- setup-multi-product-repositories
- build-api-spec:
requires:
- setup-multi-product-repositories
- build-mattermost-server:
requires:
- check-go-mod-tidy
- check-golangci-lint
- check-app-layers
- check-generate-worktemplates
- check-store-layers
- check-mocks
- check-email-templates
- check-gen-serialized
- check-mattermost-vet
- check-migrations
- build-api-spec
- setup-mattermost-webapp
- setup-focalboard-product
- build-focalboard:
requires:
- check-go-mod-tidy
- check-golangci-lint
- check-app-layers
- check-generate-worktemplates
- check-store-layers
- check-mocks
- check-email-templates
- check-gen-serialized
- check-mattermost-vet
- check-migrations
- build-api-spec
- setup-mattermost-webapp
- test:
name: test-mysql
dbdriver: mysql
dbsource: mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true
racemode: ""
requires:
- check-go-mod-tidy
- check-golangci-lint
- check-app-layers
- check-generate-worktemplates
- check-store-layers
- check-mocks
- check-email-templates
- check-gen-serialized
- check-mattermost-vet
- check-migrations
- build-api-spec
- setup-mattermost-webapp
- setup-focalboard-product
- test:
name: test-postgres-normal
dbdriver: postgres
dbsource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10
racemode: ""
requires:
- check-go-mod-tidy
- check-golangci-lint
- check-app-layers
- check-generate-worktemplates
- check-store-layers
- check-mocks
- check-email-templates
- check-gen-serialized
- check-mattermost-vet
- check-migrations
- build-api-spec
- setup-mattermost-webapp
- setup-focalboard-product
- test:
name: test-postgres-binary
dbdriver: postgres
dbsource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes
racemode: ""
requires:
- check-go-mod-tidy
- check-golangci-lint
- check-app-layers
- check-generate-worktemplates
- check-store-layers
- check-mocks
- check-email-templates
- check-gen-serialized
- check-mattermost-vet
- check-migrations
- build-api-spec
- setup-mattermost-webapp
- setup-focalboard-product
- test-schema:
requires:
- check-go-mod-tidy
- check-golangci-lint
- check-app-layers
- check-generate-worktemplates
- check-store-layers
- check-mocks
- check-email-templates
- check-gen-serialized
- check-mattermost-vet
- check-migrations
- build-api-spec
- setup-mattermost-webapp
- setup-focalboard-product
- upload-artifacts-to-s3:
context: mattermost-ci-pr-builds-s3
requires:
- build-mattermost-server
- build-focalboard
- build-docker:
context: matterbuild-docker
requires:
- upload-artifacts-to-s3
release-build:
jobs:
- setup-multi-product-repositories:
context:
- matterbuild-github-token
filters:
branches:
only:
- master
- /^release-.*/
- cloud
- setup-mattermost-webapp:
filters:
branches:
only:
- master
- /^release-.*/
- cloud
- setup-focalboard-product:
filters:
branches:
only:
- master
- /^release-.*/
- cloud
requires:
- setup-multi-product-repositories
- check-go-mod-tidy:
requires:
- setup-multi-product-repositories
- check-golangci-lint:
requires:
- setup-multi-product-repositories
- check-app-layers:
requires:
- setup-multi-product-repositories
- check-generate-worktemplates:
requires:
- setup-multi-product-repositories
- check-store-layers:
requires:
- setup-multi-product-repositories
- check-mocks:
requires:
- setup-multi-product-repositories
- check-email-templates:
requires:
- setup-multi-product-repositories
- check-gen-serialized:
requires:
- setup-multi-product-repositories
- check-mattermost-vet:
requires:
- setup-multi-product-repositories
- check-migrations:
requires:
- setup-multi-product-repositories
- build-api-spec:
requires:
- setup-multi-product-repositories
- sentry:
context: matterbuild-sentry
requires:
- setup-multi-product-repositories
- build-mattermost-server:
requires:
- check-go-mod-tidy
- check-golangci-lint
- check-app-layers
- check-generate-worktemplates
- check-store-layers
- check-mocks
- check-email-templates
- check-gen-serialized
- check-mattermost-vet
- check-migrations
- build-api-spec
- setup-mattermost-webapp
- setup-focalboard-product
- build-focalboard:
requires:
- check-go-mod-tidy
- check-golangci-lint
- check-app-layers
- check-generate-worktemplates
- check-store-layers
- check-mocks
- check-email-templates
- check-gen-serialized
- check-mattermost-vet
- check-migrations
- build-api-spec
- setup-mattermost-webapp
- test:
name: test-mysql
dbdriver: mysql
dbsource: mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true
racemode: "-race"
requires:
- check-go-mod-tidy
- check-golangci-lint
- check-app-layers
- check-generate-worktemplates
- check-store-layers
- check-mocks
- check-email-templates
- check-gen-serialized
- check-mattermost-vet
- check-migrations
- build-api-spec
- test:
name: test-postgres-normal
dbdriver: postgres
dbsource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10
racemode: "-race"
requires:
- check-go-mod-tidy
- check-golangci-lint
- check-app-layers
- check-generate-worktemplates
- check-store-layers
- check-mocks
- check-email-templates
- check-gen-serialized
- check-mattermost-vet
- check-migrations
- build-api-spec
- test:
name: test-postgres-binary
dbdriver: postgres
dbsource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes
racemode: ""
requires:
- check-go-mod-tidy
- check-golangci-lint
- check-app-layers
- check-generate-worktemplates
- check-store-layers
- check-mocks
- check-email-templates
- check-gen-serialized
- check-mattermost-vet
- check-migrations
- build-api-spec

291
.github/workflows/ci.yml vendored Normal file
View File

@@ -0,0 +1,291 @@
name: mattermost-build
on:
pull_request:
push:
branches:
- master
- cloud
- release-*
jobs:
check-mocks:
name: Check mocks
runs-on: ubuntu-22.04
steps:
- name: Checkout mattermost-server
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: Generate mocks
run: make mocks
- name: Check mocks
run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the mocks using `make mocks`"; exit 1; fi
check-go-mod-tidy:
name: Check go mod tidy
runs-on: ubuntu-22.04
steps:
- name: Checkout mattermost-server
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: Run go mod tidy
run: make modules-tidy
- name: Check modules
run: if [[ -n $(git status --porcelain) ]]; then echo "Please tidy up the Go modules using make modules-tidy"; git diff; exit 1; fi
check-gen-serialized:
name: Check serialization methods for hot structs
runs-on: ubuntu-22.04
steps:
- name: Checkout mattermost-server
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: Run make-gen-serialized
run: make gen-serialized
- name: Check serialized
run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the serialized files using 'make gen-serialized'"; exit 1; fi
check-mattermost-vet:
name: Check style
runs-on: ubuntu-latest-8-cores
steps:
- name: Checkout mattermost-server
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: Checkout focalboard
run: |
cd ..
git clone --depth=1 --no-single-branch https://github.com/mattermost/focalboard.git
cd focalboard
git checkout $GITHUB_HEAD_REF || git checkout $GITHUB_BASE_REF || git checkout rolling-stable
echo $(git rev-parse HEAD)
cd ../mattermost-server
make setup-go-work
- name: Reset config
run: make config-reset
- name: Run plugin-checker
run: make plugin-checker
- name: Run mattermost-vet
run: make vet BUILD_NUMBER='${GITHUB_HEAD_REF}' MM_NO_ENTERPRISE_LINT=true MM_VET_OPENSPEC_PATH='${PWD}/../mattermost-api-reference/v4/html/static/mattermost-openapi-v4.yaml'
check-migrations:
name: Check migration files
runs-on: ubuntu-22.04
steps:
- name: Checkout mattermost-server
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: Extract migrations files
run: make migrations-extract
- name: Check migration files
run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the migrations using make migrations-extract"; exit 1; fi
build-api-spec:
name: Build API specification
runs-on: ubuntu-22.04
steps:
- name: Checkout mattermost-server
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: Checkout mattermost-api-reference
run: |
cd ..
git clone --depth=1 --no-single-branch https://github.com/mattermost/mattermost-api-reference.git
cd mattermost-api-reference
echo "Trying to checkout the same branch on mattermost-api-reference as mattermost-server"
git checkout $GITHUB_HEAD_REF || git checkout $GITHUB_BASE_REF || true
make build
cd ../mattermost-server
check-generate-work-templates:
name: Generate work templates
runs-on: ubuntu-22.04
steps:
- name: Checkout mattermost-server
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: Generate work templates
run: make generate-worktemplates
- name: Check generated work templates
run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the worktemplates using make generate-worktemplates"; exit 1; fi
check-email-templates:
name: Generate email templates
runs-on: ubuntu-22.04
steps:
- name: Checkout mattermost-server
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: Generate email templates
run: |
sudo npm install -g mjml@4.9.0
make build-templates
- name: Check generated email templates
run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the email templates using `make build-templates`"; exit 1; fi
check-store-layers:
name: Check store layers
runs-on: ubuntu-22.04
steps:
- name: Checkout mattermost-server
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: Generate store layers
run: make store-layers
- name: Check generated code
run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the store layers using make store-layers"; exit 1; fi
check-app-layers:
name: Check app layers
runs-on: ubuntu-22.04
steps:
- name: Checkout mattermost-server
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: Generate app layers
run: make app-layers
- name: Check generated code
run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi
test-postgres-binary:
name: Run tests on postgres with binary parameters
needs: check-mattermost-vet
uses: ./.github/workflows/test.yml
with:
datasource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes
drivername: postgres
test-postgres-normal:
name: Run tests on postgres
needs: check-mattermost-vet
uses: ./.github/workflows/test.yml
with:
datasource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10
drivername: postgres
test-mysql:
name: Run tests on mysql
needs: check-mattermost-vet
uses: ./.github/workflows/test.yml
with:
datasource: mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true
drivername: mysql
build-mattermost-server:
name: Build mattermost-server
runs-on: ubuntu-latest-8-cores
needs: check-mattermost-vet
steps:
- name: Checkout mattermost-server
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: Checkout mattermost-webapp
run: |
cd ..
git clone --depth=1 --no-single-branch https://github.com/mattermost/mattermost-webapp.git
cd mattermost-webapp
git checkout $GITHUB_HEAD_REF || git checkout master
export WEBAPP_GIT_COMMIT=$(git rev-parse HEAD)
echo "$WEBAPP_GIT_COMMIT"
FILE_DIST=dist.tar.gz
runtime="2 minute"
endtime=$(date -ud "$runtime" +%s)
while [[ $(date -u +%s) -le $endtime ]]; do
if curl -s --max-time 30 -f -o $FILE_DIST https://pr-builds.mattermost.com/mattermost-webapp/commit/$WEBAPP_GIT_COMMIT/mattermost-webapp.tar.gz; then
break
fi
echo "Waiting for webapp git commit $WEBAPP_GIT_COMMIT with sleep 5: `date +%H:%M:%S`"
sleep 5
done
if [[ -f "$FILE_DIST" ]]; then
echo "Precompiled version of web app found"
mkdir dist && tar -xf $FILE_DIST -C dist --strip-components=1
else
echo "Building web app from source"
make dist
fi
cd ../mattermost-server
- name: Checkout and build focalboard
run: |
cd ..
git clone --depth=1 --no-single-branch https://github.com/mattermost/focalboard.git
cd focalboard
git checkout $GITHUB_HEAD_REF || git checkout $GITHUB_BASE_REF || git checkout rolling-stable
echo $(git rev-parse HEAD)
make server-linux
echo "Building Boards product for web app"
# make prebuild build-product # TODO figure out how to get this to run without bypassing the Makefile
make prebuild
cd mattermost-plugin/webapp
npm run build:product
cd ../../../mattermost-server
make setup-go-work
- name: Build
run: |
make config-reset
make build-cmd BUILD_NUMBER='${GITHUB_HEAD_REF}-${GITHUB_RUN_ID}'
make package BUILD_NUMBER='${GITHUB_HEAD_REF}-${GITHUB_RUN_ID}'
- name: Persist dist artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
with:
name: server-dist-artifact
path: dist/
retention-days: 14
- name: Persist build artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
with:
name: server-build-artifact
path: build/
retention-days: 14
upload-s3:
name: Upload to S3 bucket
runs-on: ubuntu-22.04
needs:
- build-mattermost-server
- test-mysql
- test-postgres-binary
- test-postgres-normal
env:
REPO_NAME: ${{ github.event.repository.name }}
steps:
- name: Download dist artifacts
uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b # v3.0.2
with:
name: server-dist-artifact
path: dist/
- name: Configure AWS
uses: aws-actions/configure-aws-credentials@07c2f971bac433df982ccc261983ae443861db49 # v1-node16
with:
aws-region: us-east-1
aws-access-key-id: ${{ secrets.MM_SERVER_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.MM_SERVER_AWS_SECRET_ACCESS_KEY }}
# We need to sanitize the branch name before using it
- name: ci/sanitize-branch-name
id: branch
uses: transferwise/sanitize-branch-name@b10b4d524ac5a7b645b43a3527db3a6cca017b9d # v1
# Search for the string "pull" and replace it with "PR" in branch-name
- name: ci/sanitize-branch-name-replace-pull-with-PR-
run: echo "BRANCH_NAME=$(echo ${{ steps.branch.outputs.sanitized-branch-name }} | sed 's/^pull\//PR-/g')" >> $GITHUB_ENV
- name: ci/artifact-upload
run: |
aws s3 cp dist/ s3://pr-builds.mattermost.com/$REPO_NAME/$BRANCH_NAME/ --acl public-read --cache-control "no-cache" --recursive
aws s3 cp dist/ s3://pr-builds.mattermost.com/$REPO_NAME/commit/${{ github.sha }}/ --acl public-read --cache-control "no-cache" --recursive
build-docker:
name: Build docker image
runs-on: ubuntu-22.04
needs: upload-s3
steps:
- name: Download build artifacts
uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b # v3.0.2
with:
name: server-build-artifact
path: build/
- name: Login to Docker Hub
uses: docker/login-action@3da7dc6e2b31f99ef2cb9fb4c50fb0971e0d0139 # v2.1.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Setup Docker Buildx
uses: docker/setup-buildx-action@11e8a2e2910826a92412015c515187a2d6750279 # v2.4
- name: Docker build and push
env:
DOCKER_CLI_EXPERIMENTAL: enabled
run: |
export TAG=$(echo "${{ github.event.pull_request.head.sha }}" | cut -c1-7)
cd build
export DOCKER_CLI_EXPERIMENTAL=enabled
export MM_PACKAGE=https://pr-builds.mattermost.com/mattermost-server/commit/${GITHUB_SHA}/mattermost-team-linux-amd64.tar.gz
docker buildx build --push --build-arg MM_PACKAGE=$MM_PACKAGE -t mattermost/mm-te-test:${TAG} .
sentry:
name: Send build info to sentry
runs-on: ubuntu-22.04
needs:
- test-postgres-binary
- test-postgres-normal
- test-mysql
if: ${{ github.event_name == 'push' }}
env:
SENTRY_CLI_VERSION: 1.64.1
steps:
- name: Checkout mattermost-server
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: Install sentry-cli
run: curl -sL https://sentry.io/get-cli/ | sh
- name: Run sentry-cli
run: |
sentry-cli --log-level=debug releases new --finalize -p mattermost-server `git rev-parse HEAD`
sentry-cli --log-level=debug releases set-commits --auto `git rev-parse HEAD`

48
.github/workflows/test.yml vendored Normal file
View File

@@ -0,0 +1,48 @@
name: Test
on:
workflow_call:
inputs:
datasource:
required: true
type: string
drivername:
required: true
type: string
jobs:
run-tests:
runs-on: ubuntu-latest-8-cores
env:
COMPOSE_PROJECT_NAME: ghactions
BUILD_IMAGE: mattermost/mattermost-build-server:20230118_golang-1.19.5
steps:
- name: Checkout mattermost-server
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: Run docker compose
run: |
cd build
docker-compose --no-ansi run --rm start_dependencies
cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test';
docker-compose --no-ansi ps
cd ..
- name: Wait for docker compose
run: |
until docker network inspect ghactions_mm-test; do echo "Waiting for Docker Compose Network..."; sleep 1; done;
docker run --net ghactions_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;"
docker run --net ghactions_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
- name: Run Tests
run: |
if [[ ${{ github.event_name }} == 'push' ]]; then
export RACE_MODE="-race"
fi
mkdir -p client/plugins
cd build
docker run --net ghactions_mm-test \
--ulimit nofile=8096:8096 \
--env-file=dotenv/test.env \
--env MM_SQLSETTINGS_DRIVERNAME="${{ inputs.drivername }}" \
--env MM_SQLSETTINGS_DATASOURCE="${{ inputs.datasource }}" \
-v ~/work/mattermost-server:/mattermost-server \
-w /mattermost-server/mattermost-server \
$BUILD_IMAGE \
make test-server$RACE_MODE BUILD_NUMBER=$GITHUB_HEAD_REF-$GITHUB_RUN_ID TESTFLAGS= TESTFLAGSEE=