mirror of
https://github.com/mattermost/mattermost.git
synced 2025-02-25 18:55:24 -06:00
5d45aa81e0
* MM-16888: fix missing indexes As part of https://mattermost.atlassian.net/browse/MM-16888, we discovered and fixed a number of column and index mismatches between the canonical (i.e. created from scratch) and migrated schemas (i.e migrated from 5.0 through 5.16). Unfortunately, the migration to fix same was added to `UpgradeDatabaseToVersion514` but never cherry picked to the pending v5.14 release at the time. Customers who upgraded to v5.14 or v5.15 and then get this code as part of v5.16 will never run that migration. Copy it to the UpgradeDatabaseToVersion516 accordingly. * avoid fixing ChannelMembers.SchemeGuest on MySQL * synchronize .circleci/config.yml with scripts/mysql-migration-test.sh * fix circleci invocation * additional logging on diff * update build/Jenkinsfile.pr too!
426 lines
16 KiB
Groovy
426 lines
16 KiB
Groovy
#!/usr/bin/env groovy
|
|
|
|
@Library('shared-pipelines') _
|
|
import org.mattermost.Utils
|
|
import java.util.*;
|
|
def utils = new org.mattermost.Utils()
|
|
def rnd = UUID.randomUUID().toString()
|
|
|
|
pipeline {
|
|
agent {
|
|
label 'default-mm-builder'
|
|
}
|
|
|
|
options {
|
|
buildDiscarder(logRotator(numToKeepStr: '3'))
|
|
timeout(time: 1, unit: 'HOURS')
|
|
}
|
|
|
|
environment {
|
|
COMPOSE_PROJECT_NAME="${rnd}-${env.BUILD_NUMBER}"
|
|
}
|
|
|
|
stages {
|
|
stage('Setup') {
|
|
steps {
|
|
script {
|
|
utils.stopOldBuilds()
|
|
}
|
|
cleanWs notFailBuild: true
|
|
sh """
|
|
mkdir -p src/github.com/mattermost/mattermost-server
|
|
mkdir -p src/github.com/mattermost/mattermost-webapp
|
|
mkdir -p src/github.com/mattermost/enterprise
|
|
"""
|
|
dir('src/github.com/mattermost/mattermost-server') {
|
|
checkout scm
|
|
}
|
|
dir('src/github.com/mattermost/mattermost-webapp') {
|
|
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '310159d3-f7c5-4f5d-bfa1-151e3ef2db57', url: 'https://github.com/mattermost/mattermost-webapp.git']]]
|
|
}
|
|
dir('src/github.com/mattermost/enterprise') {
|
|
checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '310159d3-f7c5-4f5d-bfa1-151e3ef2db57', url: 'https://github.com/mattermost/enterprise.git']]]
|
|
}
|
|
dir('src/github.com/mattermost/enterprise') {
|
|
ansiColor('xterm') {
|
|
sh """
|
|
#!/bin/bash -ex
|
|
git checkout $env.BRANCH_NAME || git checkout $env.CHANGE_BRANCH || git checkout $env.GIT_BRANCH || git checkout $env.CHANGE_TARGET || git checkout master || echo 1
|
|
export EE_GIT_COMMIT=\$(git rev-parse HEAD)
|
|
|
|
echo EE Commit: \${EE_GIT_COMMIT}
|
|
"""
|
|
}
|
|
}
|
|
dir('src/github.com/mattermost/mattermost-webapp') {
|
|
withDockerContainer(args: '', image: 'mattermost/mattermost-build-webapp:oct-2-2018') {
|
|
ansiColor('xterm') {
|
|
sh """
|
|
#!/bin/bash -ex
|
|
git checkout $env.BRANCH_NAME || git checkout $env.CHANGE_BRANCH || git checkout $env.GIT_BRANCH || git checkout $env.CHANGE_TARGET || git checkout master
|
|
rm -rf ./dist
|
|
export WEBAPP_GIT_COMMIT=\$(git rev-parse HEAD)
|
|
|
|
echo Webapp Commit: \${WEBAPP_GIT_COMMIT}
|
|
|
|
curl -f -o ./dist.tar.gz https://releases.mattermost.com/mattermost-webapp/commit/\${WEBAPP_GIT_COMMIT}/mattermost-webapp.tar.gz && mkdir ./dist && tar -xvf ./dist.tar.gz -C ./dist --strip-components=1 || make node_modules test build
|
|
"""
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Check i18n') {
|
|
environment {
|
|
GOPATH = "/go"
|
|
}
|
|
|
|
steps {
|
|
withDockerContainer(args: '-u root --privileged -v ${WORKSPACE}/src:/go/src/', image: 'mattermost/mattermost-build-server:sep-17-2019') {
|
|
ansiColor('xterm') {
|
|
sh """
|
|
cd /go/src/github.com/mattermost/mattermost-server
|
|
cp i18n/en.json /tmp/en.json
|
|
make i18n-extract
|
|
diff /tmp/en.json i18n/en.json
|
|
"""
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Build') {
|
|
environment {
|
|
GOPATH = "/go"
|
|
}
|
|
|
|
steps {
|
|
withDockerContainer(args: '-u root --privileged -v ${WORKSPACE}/src:/go/src/', image: 'mattermost/mattermost-build-server:sep-17-2019') {
|
|
ansiColor('xterm') {
|
|
sh """
|
|
cd /go/src/github.com/mattermost/mattermost-server
|
|
make config-reset
|
|
make check-style BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}'
|
|
make build BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}'
|
|
make package BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}'
|
|
"""
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Push to S3') {
|
|
stages {
|
|
stage('Pull request') {
|
|
when {
|
|
allOf {
|
|
expression { env.CHANGE_ID != null }
|
|
expression { env.CHANGE_TARGET != null }
|
|
}
|
|
}
|
|
steps {
|
|
dir('src/github.com/mattermost/mattermost-server/dist') {
|
|
step([$class: 'S3BucketPublisher', dontWaitForConcurrentBuildCompletion: false, entries: [[
|
|
bucket: "releases.mattermost.com/mattermost-platform-pr/${CHANGE_ID}",
|
|
excludedFile: '',
|
|
flatten: true,
|
|
gzipFiles: false,
|
|
keepForever: false,
|
|
managedArtifacts: false,
|
|
noUploadOnFailure: true,
|
|
selectedRegion: 'us-east-1',
|
|
showDirectlyInBrowser: false,
|
|
sourceFile: '*.tar.gz',
|
|
storageClass: 'STANDARD',
|
|
uploadFromSlave: false,
|
|
useServerSideEncryption: false,
|
|
userMetadata: [[key: 'Cache-Control', value: 'no-cache']]
|
|
], [
|
|
bucket: "releases.mattermost.com/mattermost-platform-pr/${CHANGE_BRANCH}",
|
|
excludedFile: '',
|
|
flatten: true,
|
|
gzipFiles: false,
|
|
keepForever: false,
|
|
managedArtifacts: false,
|
|
noUploadOnFailure: true,
|
|
selectedRegion: 'us-east-1',
|
|
showDirectlyInBrowser: false,
|
|
sourceFile: '*.tar.gz',
|
|
storageClass: 'STANDARD',
|
|
uploadFromSlave: false,
|
|
useServerSideEncryption: false,
|
|
userMetadata: [[key: 'Cache-Control', value: 'no-cache']]
|
|
]], profileName: 'Releases', userMetadAta: []])
|
|
}
|
|
}
|
|
}
|
|
stage('Branch') {
|
|
when {
|
|
expression { env.CHANGE_ID == null }
|
|
}
|
|
steps {
|
|
dir('src/github.com/mattermost/mattermost-server/dist') {
|
|
step([$class: 'S3BucketPublisher', dontWaitForConcurrentBuildCompletion: false, entries: [[
|
|
bucket: "releases.mattermost.com/mattermost-platform-pr/${BRANCH_NAME}",
|
|
excludedFile: '',
|
|
flatten: true,
|
|
gzipFiles: false,
|
|
keepForever: false,
|
|
managedArtifacts: false,
|
|
noUploadOnFailure: true,
|
|
selectedRegion: 'us-east-1',
|
|
showDirectlyInBrowser: false,
|
|
sourceFile: '*.tar.gz',
|
|
storageClass: 'STANDARD',
|
|
uploadFromSlave: false,
|
|
useServerSideEncryption: false,
|
|
userMetadata: [[key: 'Cache-Control', value: 'no-cache']]
|
|
]], profileName: 'Releases', userMetadAta: []])
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Clean checkout') {
|
|
when {
|
|
expression { env.CHANGE_ID != null }
|
|
}
|
|
// We need to perform a clean checkout here to ge the original git commit hash from the PR
|
|
// Jenkins now merges master in top of the PR and this generate a new git hash
|
|
// We need to do that to build the docker image based on the original git commit and then this will be used by
|
|
// mattermod to update the test server.
|
|
steps {
|
|
sh """
|
|
mkdir -p /tmp/mattermost-server
|
|
"""
|
|
dir('/tmp/mattermost-server') {
|
|
checkout([$class: 'GitSCM', branches: [[name: 'FETCH_HEAD']],
|
|
doGenerateSubmoduleConfigurations: false, extensions: [],
|
|
submoduleCfg: [], userRemoteConfigs: [
|
|
[refspec: "+refs/pull/${CHANGE_ID}/head:refs/remotes/origin/PR-${CHANGE_ID}",
|
|
credentialsId: "310159d3-f7c5-4f5d-bfa1-151e3ef2db57",url: "https://github.com/mattermost/mattermost-server.git"]]])
|
|
sh 'git rev-parse --short HEAD'
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Build Docker Image') {
|
|
environment {
|
|
GIT_COMMIT_SHORT = sh(
|
|
script: "cd /tmp/mattermost-server && printf \$(git rev-parse --short HEAD)",
|
|
returnStdout: true
|
|
)
|
|
}
|
|
when {
|
|
expression { env.CHANGE_ID != null }
|
|
}
|
|
steps {
|
|
dir('src/github.com/mattermost/mattermost-server') {
|
|
withCredentials([usernamePassword(credentialsId: 'matterbuild-docker-hub', usernameVariable: 'DOCKER_USER', passwordVariable: 'DOCKER_PASS')]) {
|
|
sh 'docker login --username ${DOCKER_USER} --password ${DOCKER_PASS}'
|
|
sh """
|
|
docker build --no-cache --build-arg MM_PACKAGE=https://releases.mattermost.com/mattermost-platform-pr/${CHANGE_ID}/mattermost-enterprise-linux-amd64.tar.gz -t mattermost/mattermost-enterprise-edition:${GIT_COMMIT_SHORT} build
|
|
docker push mattermost/mattermost-enterprise-edition:${GIT_COMMIT_SHORT}
|
|
docker logout
|
|
"""
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
stage('Test') {
|
|
environment {
|
|
GOPATH = "/go"
|
|
TEST_DATABASE_MYSQL_DSN = "mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s"
|
|
TEST_DATABASE_POSTGRESQL_DSN = "postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10"
|
|
TEST_DATABASE_MYSQL_ROOT_PASSWD = "mostest"
|
|
CI_INBUCKET_HOST = "inbucket"
|
|
CI_MINIO_HOST = "minio"
|
|
CI_INBUCKET_PORT = "10080"
|
|
CI_MINIO_PORT = "9000"
|
|
CI_INBUCKET_SMTP_PORT = "10025"
|
|
CI_LDAP_HOST = "openldap"
|
|
IS_CI = true
|
|
MM_SQLSETTINGS_DATASOURCE = "mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8"
|
|
MM_EMAILSETTINGS_SMTPSERVER = "inbucket"
|
|
MM_EMAILSETTINGS_SMTPPORT = "10025"
|
|
MM_ELASTICSEARCHSETTINGS_CONNECTIONURL = "http://elasticsearch:9200"
|
|
LDAP_DATA = "test"
|
|
}
|
|
|
|
steps {
|
|
dir('src/github.com/mattermost/mattermost-server/build') {
|
|
ansiColor('xterm') {
|
|
sh """
|
|
/usr/local/bin/docker-compose --no-ansi run --rm start_dependencies
|
|
/usr/local/bin/docker-compose --no-ansi ps
|
|
cat ../tests/${LDAP_DATA}-data.ldif | /usr/local/bin/docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
|
|
"""
|
|
}
|
|
}
|
|
|
|
dir('src/github.com/mattermost/mattermost-server') {
|
|
ansiColor('xterm') {
|
|
sh """
|
|
echo "Creating databases"
|
|
/usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test'
|
|
echo "Importing postgres dump from version 5.0"
|
|
/usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres psql -U mmuser -d migrated < \$(pwd)/scripts/mattermost-postgresql-5.0.sql
|
|
"""
|
|
}
|
|
}
|
|
|
|
withDockerContainer(args: "-u root --privileged --net ${COMPOSE_PROJECT_NAME}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:sep-17-2019') {
|
|
ansiColor('xterm') {
|
|
sh """
|
|
cd /go/src/github.com/mattermost/mattermost-server
|
|
echo "Setting up config for db migration"
|
|
export MM_SQLSETTINGS_DATASOURCE=\"postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10\"
|
|
export MM_SQLSETTINGS_DRIVERNAME=\"postgres\"
|
|
make ARGS="config get SqlSettings.DataSource" run-cli
|
|
echo "Running the migration"
|
|
make ARGS="version" run-cli
|
|
|
|
echo "Setting up config for fresh db setup"
|
|
export MM_SQLSETTINGS_DATASOURCE=\"postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10\"
|
|
make ARGS="config get SqlSettings.DataSource" run-cli
|
|
|
|
echo "Setting up fresh db"
|
|
make ARGS="version" run-cli
|
|
"""
|
|
}
|
|
}
|
|
|
|
|
|
dir('src/github.com/mattermost/mattermost-server') {
|
|
ansiColor('xterm') {
|
|
sh """
|
|
echo "Generating dump"
|
|
/usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql
|
|
/usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql
|
|
|
|
echo "Removing databases created for db comparison"
|
|
/usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres sh -c \'exec echo \"DROP DATABASE migrated; DROP DATABASE latest;\" | exec psql -U mmuser mattermost_test\'
|
|
|
|
echo "Generating diff"
|
|
set +e
|
|
diff migrated.sql latest.sql > diff.txt
|
|
set -e
|
|
export diffErrorCode=\$?
|
|
|
|
if [ \$diffErrorCode -eq 0 ]; then echo \"Both schemas are same\";else cat diff.txt; fi
|
|
|
|
exit \$diffErrorCode
|
|
"""
|
|
}
|
|
}
|
|
|
|
dir('src/github.com/mattermost/mattermost-server') {
|
|
ansiColor('xterm') {
|
|
sh """
|
|
echo "Creating databases"
|
|
/usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysql -uroot -pmostest -e \"CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser\"
|
|
echo "Importing mysql dump from version 5.0"
|
|
/usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysql -D migrated -uroot -pmostest < \$(pwd)/scripts/mattermost-mysql-5.0.sql
|
|
"""
|
|
}
|
|
}
|
|
|
|
withDockerContainer(args: "-u root --privileged --net ${COMPOSE_PROJECT_NAME}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:sep-17-2019') {
|
|
ansiColor('xterm') {
|
|
sh """
|
|
cd /go/src/github.com/mattermost/mattermost-server
|
|
echo "Setting up config for db migration"
|
|
export MM_SQLSETTINGS_DATASOURCE=\"mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s\"
|
|
export MM_SQLSETTINGS_DRIVERNAME=\"mysql\"
|
|
make ARGS="config get SqlSettings.DataSource" run-cli
|
|
echo "Running the migration"
|
|
make ARGS="version" run-cli
|
|
|
|
echo "Setting up config for fresh db setup"
|
|
export MM_SQLSETTINGS_DATASOURCE=\"mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s\"
|
|
make ARGS="config get SqlSettings.DataSource" run-cli
|
|
|
|
echo "Setting up fresh db"
|
|
make ARGS="version" run-cli
|
|
"""
|
|
}
|
|
}
|
|
|
|
|
|
dir('src/github.com/mattermost/mattermost-server') {
|
|
ansiColor('xterm') {
|
|
sh """
|
|
echo "Ignoring known MySQL mismatch: ChannelMembers.SchemeGuest"
|
|
/usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;"
|
|
/usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;"
|
|
|
|
echo "Generating dump"
|
|
/usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql
|
|
/usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql
|
|
|
|
echo "Removing databases created for db comparison"
|
|
/usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysql -uroot -pmostest -e \"DROP DATABASE migrated; DROP DATABASE latest\"
|
|
|
|
echo "Generating diff"
|
|
diff migrated.sql latest.sql > diff.txt
|
|
export diffErrorCode=\$?
|
|
|
|
if [ \$diffErrorCode -eq 0 ]; then echo \"Both schemas are same\";else cat diff.txt; fi
|
|
|
|
exit \$diffErrorCode
|
|
"""
|
|
}
|
|
}
|
|
|
|
withDockerContainer(args: "-u root --privileged --net ${COMPOSE_PROJECT_NAME}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:sep-17-2019') {
|
|
ansiColor('xterm') {
|
|
sh """
|
|
cd /go/src/github.com/mattermost/mattermost-server
|
|
|
|
mkdir -p client/plugins
|
|
cat config/config.json
|
|
|
|
make test-server BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' TESTFLAGS= TESTFLAGSEE=
|
|
"""
|
|
}
|
|
withCredentials([string(credentialsId: 'CODECOV_TOKEN', variable: 'CODECOV')]) {
|
|
sh '''
|
|
cd /go/src/github.com/mattermost/mattermost-server
|
|
curl -s https://codecov.io/bash | bash -s - -t $CODECOV || echo 'Codecov failed to upload'
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
post {
|
|
always {
|
|
dir('src/github.com/mattermost/mattermost-server') {
|
|
junit allowEmptyResults: true, healthScaleFactor: 0.0, testResults: 'report*.xml'
|
|
archiveArtifacts 'report*.xml'
|
|
}
|
|
dir('src/github.com/mattermost/mattermost-server/build') {
|
|
ansiColor('xterm') {
|
|
sh """
|
|
# Capture docker logs
|
|
docker-compose logs --tail="all" -t --no-color > docker-compose_logs
|
|
docker ps -a --no-trunc > docker_ps
|
|
docker stats -a --no-stream > docker_stats
|
|
tar -czvf docker_logs.tar.gz docker-compose_logs docker_ps docker_stats
|
|
|
|
docker-compose --no-ansi down -v
|
|
"""
|
|
}
|
|
archiveArtifacts 'docker_logs.tar.gz'
|
|
}
|
|
}
|
|
cleanup {
|
|
cleanWs notFailBuild: true
|
|
}
|
|
}
|
|
}
|