IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2025-02-25 18:55:24 -06:00
Code Issues Packages Projects Releases Wiki Activity

Speed up build (#9993)

Browse Source 
* configure MySQL to use tmpfs

* simplify use of COMPOSE_PROJECT_NAME, and set envs vs. overriding config.json, fix error output, --no-ansi

* cmd/mattermost/commands/*_test.go: ignore MM_SQLSETTINGS_DATASOURCE from the env

* clean up exit 1 semantics
This commit is contained in:
Jesse Hallam
2018-12-13 16:29:47 -05:00
committed by GitHub
parent 1f197d7c55
commit 86fa3866c1
5 changed files with 100 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
build/Jenkinsfile.pr
View File

@@ -14,7 +14,7 @@ pipeline {
options { buildDiscarder(logRotator(numToKeepStr: '3')) }
environment {
DCNAME = "${rnd}"
COMPOSE_PROJECT_NAME="${rnd}-${env.BUILD_NUMBER}"
}
stages {
@@ -54,13 +54,13 @@ pipeline {
ansiColor('xterm') {
sh """
#!/bin/bash -ex
git checkout $env.BRANCH_NAME || git checkout $env.CHANGE_BRANCH || git checkout $env.GIT_BRANCH || git checkout master || echo 1
git checkout $env.BRANCH_NAME || git checkout $env.CHANGE_BRANCH || git checkout $env.GIT_BRANCH || git checkout master
rm -rf ./dist
export WEBAPP_GIT_COMMIT=\$(git rev-parse HEAD)
echo Webapp Commit: \${WEBAPP_GIT_COMMIT}
curl -f -o ./dist.tar.gz https://releases.mattermost.com/mattermost-webapp/commit/\${WEBAPP_GIT_COMMIT}/mattermost-webapp.tar.gz && mkdir ./dist && tar -xvf ./dist.tar.gz -C ./dist --strip-components=1 || make node_modules test build || exit 1
curl -f -o ./dist.tar.gz https://releases.mattermost.com/mattermost-webapp/commit/\${WEBAPP_GIT_COMMIT}/mattermost-webapp.tar.gz && mkdir ./dist && tar -xvf ./dist.tar.gz -C ./dist --strip-components=1 || make node_modules test build
"""
}
}
@@ -79,9 +79,9 @@ pipeline {
sh """
cd /go/src/github.com/mattermost/mattermost-server
mv config/default.json config/config.json || echo ""
make check-style BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' || exit 1
make build BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' || exit 1
make package BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' || exit 1
make check-style BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}'
make build BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}'
make package BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}'
"""
}
}
@@ -101,41 +101,38 @@ pipeline {
CI_INBUCKET_SMTP_PORT = "10025"
CI_LDAP_HOST = "openldap"
IS_CI = true
MM_SQLSETTINGS_DATASOURCE = "mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8"
MM_EMAILSETTINGS_SMTPSERVER = "inbucket"
MM_EMAILSETTINGS_SMTPPORT = "10025"
MM_ELASTICSEARCHSETTINGS_CONNECTIONURL = "http://elasticsearch:9200"
}
steps {
dir('src/github.com/mattermost/mattermost-server/build') {
ansiColor('xterm') {
sh """
COMPOSE_PROJECT_NAME=${DCNAME}-${env.BUILD_NUMBER} docker-compose run --rm start_dependencies
COMPOSE_PROJECT_NAME=${DCNAME}-${env.BUILD_NUMBER} docker-compose up -d mysql postgres minio inbucket openldap elasticsearch redis
docker-compose --no-ansi run --rm start_dependencies
docker-compose --no-ansi ps
docker ps
docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'echo -e "dn: ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: organizationalunit" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'echo -e "dn: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test1\nmail: success+testone@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" -w mostest';
docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'echo -e "dn: uid=test.two,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test2\nmail: success+testtwo@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.two,ou=testusers,dc=mm,dc=test,dc=com" -w mostest';
docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'echo -e "dn: cn=tgroup,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: groupOfUniqueNames\nuniqueMember: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
docker-compose --no-ansi exec -T openldap bash -c 'echo -e "dn: ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: organizationalunit" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
docker-compose --no-ansi exec -T openldap bash -c 'echo -e "dn: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test1\nmail: success+testone@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
docker-compose --no-ansi exec -T openldap bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" -w mostest';
docker-compose --no-ansi exec -T openldap bash -c 'echo -e "dn: uid=test.two,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test2\nmail: success+testtwo@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
docker-compose --no-ansi exec -T openldap bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.two,ou=testusers,dc=mm,dc=test,dc=com" -w mostest';
docker-compose --no-ansi exec -T openldap bash -c 'echo -e "dn: cn=tgroup,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: groupOfUniqueNames\nuniqueMember: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
"""
}
}
withDockerContainer(args: "-u root --privileged --net ${DCNAME}-${BUILD_NUMBER}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:dec-7-2018') {
withDockerContainer(args: "-u root --privileged --net ${COMPOSE_PROJECT_NAME}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:dec-7-2018') {
ansiColor('xterm') {
sh """
cd /go/src/github.com/mattermost/mattermost-server
sed -i'.bak' 's|mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8|mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8|g' config/config.json;
sed -i'.bak' 's|"SMTPServer": "dockerhost",|"SMTPServer": "inbucket",|g' config/config.json;
sed -i'.bak' 's|"SMTPPort": "2500",|"SMTPPort": "10025",|g' config/config.json;
sed -i'.bak' 's|"ConnectionUrl": "http://dockerhost:9200",|"ConnectionUrl": "http://elasticsearch:9200",|g' config/config.json;
mkdir -p client/plugins
cat config/config.json
make test-server BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' TESTFLAGS= TESTFLAGSEE= || exit 1
make test-server BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' TESTFLAGS= TESTFLAGSEE=
"""
}
}
@@ -234,7 +231,7 @@ pipeline {
dir('src/github.com/mattermost/mattermost-server/build') {
ansiColor('xterm') {
sh """
COMPOSE_PROJECT_NAME=${DCNAME}-${env.BUILD_NUMBER} docker-compose down -v
docker-compose --no-ansi down -v
"""
}
}

11
build/docker-compose.yml
View File

@@ -3,7 +3,6 @@ services:
mysql:
image: "mysql:5.7"
restart: always
container_name: mysql-$DCNAME-$BUILD_NUMBER
networks:
- mm-test
environment:
@@ -12,20 +11,22 @@ services:
MYSQL_PASSWORD: mostest
MYSQL_USER: mmuser
MYSQL_DATABASE: mattermost_test
tmpfs: /var/lib/mysql
volumes:
- "./docker/mysql.conf.d:/etc/mysql/conf.d"
postgres:
image: "postgres:9.4"
restart: always
container_name: postgres-$DCNAME-$BUILD_NUMBER
networks:
- mm-test
environment:
POSTGRES_USER: mmuser
POSTGRES_PASSWORD: mostest
POSTGRES_DB: mattermost_test
tmpfs: /var/lib/postgresql/data
minio:
image: "minio/minio:RELEASE.2018-05-25T19-49-13Z"
command: "server /data"
container_name: minio-$DCNAME-$BUILD_NUMBER
networks:
- mm-test
environment:
@@ -34,13 +35,11 @@ services:
inbucket:
image: "jhillyerd/inbucket:release-1.2.0"
restart: always
container_name: inbucket-$DCNAME-$BUILD_NUMBER
networks:
- mm-test
openldap:
image: "osixia/openldap:1.1.6"
restart: always
container_name: openldap-$DCNAME-$BUILD_NUMBER
networks:
- mm-test
environment:
@@ -50,7 +49,6 @@ services:
LDAP_ADMIN_PASSWORD: "mostest"
elasticsearch:
image: "mattermost/mattermost-elasticsearch-docker:6.5.1"
container_name: elasticsearch-$DCNAME-$BUILD_NUMBER
networks:
- mm-test
environment:
@@ -61,7 +59,6 @@ services:
image: redis
networks:
- mm-test
container_name: redis-$DCNAME-$BUILD_NUMBER
start_dependencies:
image: mattermost/mattermost-wait-for-dep:latest

19
build/docker/mysql.conf.d/custom.cnf Normal file
View File

@@ -0,0 +1,19 @@
[mysqld]
bind-address = 0.0.0.0
log-output = NONE
slow-query-log = 0
innodb_flush_log_at_trx_commit = 2
innodb_flush_method = nosync
innodb_lock_wait_timeout = 50
innodb_log_buffer_size = 3M
innodb_buffer_pool_size = 180M
max_connect_errors = 1000000
max_connections = 900
character-set-server = utf8
sql_mode = ""
innodb = FORCE
default-storage-engine = InnoDB
max_allowed_packet = 256M

75
build/local-test-env.sh
View File

@@ -1,7 +1,6 @@
#!/bin/bash
export DCNAME=localdev
export BUILD_NUMBER=null
export COMPOSE_PROJECT_NAME=localdev
local_cmdname=${0##*/}
usage()
@@ -15,50 +14,41 @@ USAGE
up()
{
COMPOSE_PROJECT_NAME=$DCNAME-$BUILD_NUMBER docker-compose run --rm start_dependencies
COMPOSE_PROJECT_NAME=$DCNAME-$BUILD_NUMBER docker-compose up -d mysql postgres minio inbucket openldap elasticsearch redis
docker-compose run --rm start_dependencies
docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'echo -e "dn: ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: organizationalunit" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'
docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'echo -e "dn: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test1\nmail: success+testone@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'
docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" -w mostest'
docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'echo -e "dn: uid=test.two,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test2\nmail: success+testtwo@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'
docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.two,ou=testusers,dc=mm,dc=test,dc=com" -w mostest'
docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'echo -e "dn: cn=tgroup,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: groupOfUniqueNames\nuniqueMember: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'
docker-compose exec openldap bash -c 'echo -e "dn: ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: organizationalunit" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'
docker-compose exec openldap bash -c 'echo -e "dn: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test1\nmail: success+testone@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'
docker-compose exec openldap bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" -w mostest'
docker-compose exec openldap bash -c 'echo -e "dn: uid=test.two,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test2\nmail: success+testtwo@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'
docker-compose exec openldap bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.two,ou=testusers,dc=mm,dc=test,dc=com" -w mostest'
docker-compose exec openldap bash -c 'echo -e "dn: cn=tgroup,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: groupOfUniqueNames\nuniqueMember: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'
echo "Changing the config.json"
sed -i'.bak' 's|mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8|mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json
sed -i'.bak' 's|"SMTPServer": "dockerhost",|"SMTPServer": "inbucket",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json
sed -i'.bak' 's|"SMTPPort": "2500",|"SMTPPort": "10025",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json
sed -i'.bak' 's|"ConnectionUrl": "http://dockerhost:9200",|"ConnectionUrl": "http://elasticsearch:9200",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json
docker run -it -u root \
--privileged \
-v $GOPATH:/go \
-w /go/src/github.com/mattermost/mattermost-server/ \
--net $DCNAME-$BUILD_NUMBER\_mm-test \
-e GOPATH="/go" \
-e TEST_DATABASE_MYSQL_DSN="mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s" \
-e TEST_DATABASE_POSTGRESQL_DSN="postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" \
-e TEST_DATABASE_MYSQL_ROOT_PASSWD="mostest" \
-e CI_INBUCKET_HOST="inbucket" \
-e CI_MINIO_HOST="minio" \
-e CI_INBUCKET_PORT="10080" \
-e CI_MINIO_PORT="9000" \
-e CI_INBUCKET_SMTP_PORT="10025" \
-e CI_LDAP_HOST="openldap" \
-e IS_CI=true \
mattermost/mattermost-build-server:dec-7-2018 /bin/bash
docker run -it -u root \
--privileged \
-v $GOPATH:/go \
-w /go/src/github.com/mattermost/mattermost-server/ \
--net ${COMPOSE_PROJECT_NAME}_mm-test \
-e GOPATH="/go" \
-e TEST_DATABASE_MYSQL_DSN="mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s" \
-e TEST_DATABASE_POSTGRESQL_DSN="postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" \
-e TEST_DATABASE_MYSQL_ROOT_PASSWD="mostest" \
-e CI_INBUCKET_HOST="inbucket" \
-e CI_MINIO_HOST="minio" \
-e CI_INBUCKET_PORT="10080" \
-e CI_MINIO_PORT="9000" \
-e CI_INBUCKET_SMTP_PORT="10025" \
-e CI_LDAP_HOST="openldap" \
-e IS_CI=true \
-e MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8" \
-e MM_EMAILSETTINGS_SMTPSERVER="inbucket" \
-e MM_EMAILSETTINGS_SMTPPORT="10025" \
-e MM_ELASTICSEARCHSETTINGS_CONNECTIONURL="http://elasticsearch:9200" \
mattermost/mattermost-build-server:dec-7-2018 /bin/bash
}
down()
{
COMPOSE_PROJECT_NAME=$DCNAME-$BUILD_NUMBER docker-compose down
echo "Reverting the changes in the config.json"
sed -i'.bak' 's|mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8|mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json
sed -i'.bak' 's|"SMTPServer": "inbucket",|"SMTPServer": "dockerhost",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json
sed -i'.bak' 's|"SMTPPort": "10025",|"SMTPPort": "2500",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json
sed -i'.bak' 's|"ConnectionUrl": "http://elasticsearch:9200",|"ConnectionUrl": "http://dockerhost:9200",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json
docker-compose down
}
# process arguments
@@ -70,16 +60,19 @@ do
up
break
;;
down)
echo "Stopping Containers"
down
break
;;
--help)
usage
;;
*)
echoerr "Unknown argument: $1"
echo "Unknown argument: $1" >&2
usage
;;
esac

33
cmd/mattermost/commands/cmdtestlib.go
View File

@@ -132,28 +132,39 @@ func (h *testHelper) execArgs(t *testing.T, args []string) []string {
return ret
}
// CheckCommand invokes the test binary, returning the output modified for assertion testing.
func (h *testHelper) CheckCommand(t *testing.T, args ...string) string {
func (h *testHelper) cmd(t *testing.T, args []string) *exec.Cmd {
path, err := os.Executable()
require.NoError(t, err)
output, err := exec.Command(path, h.execArgs(t, args)...).CombinedOutput()
cmd := exec.Command(path, h.execArgs(t, args)...)
cmd.Env = []string{}
for _, env := range os.Environ() {
// Ignore MM_SQLSETTINGS_DATASOURCE from the environment, since we override.
if strings.HasPrefix(env, "MM_SQLSETTINGS_DATASOURCE=") {
continue
}
cmd.Env = append(cmd.Env, env)
}
return cmd
}
// CheckCommand invokes the test binary, returning the output modified for assertion testing.
func (h *testHelper) CheckCommand(t *testing.T, args ...string) string {
output, err := h.cmd(t, args).CombinedOutput()
require.NoError(t, err, string(output))
return strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(string(output)), "PASS"))
}
// RunCommand invokes the test binary, returning only any error.
func (h *testHelper) RunCommand(t *testing.T, args ...string) error {
path, err := os.Executable()
require.NoError(t, err)
return exec.Command(path, h.execArgs(t, args)...).Run()
return h.cmd(t, args).Run()
}
// RunCommandWithOutput is a variant of RunCommand that returns the unmodified output and any error.
func (h *testHelper) RunCommandWithOutput(t *testing.T, args ...string) (string, error) {
path, err := os.Executable()
require.NoError(t, err)
cmd := exec.Command(path, h.execArgs(t, args)...)
cmd := h.cmd(t, args)
var buf bytes.Buffer
reader, writer := io.Pipe()
@@ -166,7 +177,7 @@ func (h *testHelper) RunCommandWithOutput(t *testing.T, args ...string) (string,
close(done)
}()
err = cmd.Run()
err := cmd.Run()
writer.Close()
<-done