diff --git a/build/Jenkinsfile.pr b/build/Jenkinsfile.pr index 144441613e..414db896ca 100644 --- a/build/Jenkinsfile.pr +++ b/build/Jenkinsfile.pr @@ -14,7 +14,7 @@ pipeline { options { buildDiscarder(logRotator(numToKeepStr: '3')) } environment { - DCNAME = "${rnd}" + COMPOSE_PROJECT_NAME="${rnd}-${env.BUILD_NUMBER}" } stages { @@ -54,13 +54,13 @@ pipeline { ansiColor('xterm') { sh """ #!/bin/bash -ex - git checkout $env.BRANCH_NAME || git checkout $env.CHANGE_BRANCH || git checkout $env.GIT_BRANCH || git checkout master || echo 1 + git checkout $env.BRANCH_NAME || git checkout $env.CHANGE_BRANCH || git checkout $env.GIT_BRANCH || git checkout master rm -rf ./dist export WEBAPP_GIT_COMMIT=\$(git rev-parse HEAD) echo Webapp Commit: \${WEBAPP_GIT_COMMIT} - curl -f -o ./dist.tar.gz https://releases.mattermost.com/mattermost-webapp/commit/\${WEBAPP_GIT_COMMIT}/mattermost-webapp.tar.gz && mkdir ./dist && tar -xvf ./dist.tar.gz -C ./dist --strip-components=1 || make node_modules test build || exit 1 + curl -f -o ./dist.tar.gz https://releases.mattermost.com/mattermost-webapp/commit/\${WEBAPP_GIT_COMMIT}/mattermost-webapp.tar.gz && mkdir ./dist && tar -xvf ./dist.tar.gz -C ./dist --strip-components=1 || make node_modules test build """ } } @@ -79,9 +79,9 @@ pipeline { sh """ cd /go/src/github.com/mattermost/mattermost-server mv config/default.json config/config.json || echo "" - make check-style BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' || exit 1 - make build BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' || exit 1 - make package BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' || exit 1 + make check-style BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' + make build BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' + make package BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' """ } } @@ -101,41 +101,38 @@ pipeline { CI_INBUCKET_SMTP_PORT = "10025" CI_LDAP_HOST = "openldap" IS_CI = true + MM_SQLSETTINGS_DATASOURCE = "mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8" + MM_EMAILSETTINGS_SMTPSERVER = "inbucket" + MM_EMAILSETTINGS_SMTPPORT = "10025" + MM_ELASTICSEARCHSETTINGS_CONNECTIONURL = "http://elasticsearch:9200" } steps { dir('src/github.com/mattermost/mattermost-server/build') { ansiColor('xterm') { sh """ - COMPOSE_PROJECT_NAME=${DCNAME}-${env.BUILD_NUMBER} docker-compose run --rm start_dependencies - COMPOSE_PROJECT_NAME=${DCNAME}-${env.BUILD_NUMBER} docker-compose up -d mysql postgres minio inbucket openldap elasticsearch redis + docker-compose --no-ansi run --rm start_dependencies + docker-compose --no-ansi ps - docker ps - - docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'echo -e "dn: ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: organizationalunit" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; - docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'echo -e "dn: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test1\nmail: success+testone@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; - docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" -w mostest'; - docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'echo -e "dn: uid=test.two,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test2\nmail: success+testtwo@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; - docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.two,ou=testusers,dc=mm,dc=test,dc=com" -w mostest'; - docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'echo -e "dn: cn=tgroup,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: groupOfUniqueNames\nuniqueMember: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; + docker-compose --no-ansi exec -T openldap bash -c 'echo -e "dn: ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: organizationalunit" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; + docker-compose --no-ansi exec -T openldap bash -c 'echo -e "dn: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test1\nmail: success+testone@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; + docker-compose --no-ansi exec -T openldap bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" -w mostest'; + docker-compose --no-ansi exec -T openldap bash -c 'echo -e "dn: uid=test.two,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test2\nmail: success+testtwo@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; + docker-compose --no-ansi exec -T openldap bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.two,ou=testusers,dc=mm,dc=test,dc=com" -w mostest'; + docker-compose --no-ansi exec -T openldap bash -c 'echo -e "dn: cn=tgroup,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: groupOfUniqueNames\nuniqueMember: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; """ } } - withDockerContainer(args: "-u root --privileged --net ${DCNAME}-${BUILD_NUMBER}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:dec-7-2018') { + withDockerContainer(args: "-u root --privileged --net ${COMPOSE_PROJECT_NAME}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:dec-7-2018') { ansiColor('xterm') { sh """ cd /go/src/github.com/mattermost/mattermost-server - sed -i'.bak' 's|mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8|mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8|g' config/config.json; - sed -i'.bak' 's|"SMTPServer": "dockerhost",|"SMTPServer": "inbucket",|g' config/config.json; - sed -i'.bak' 's|"SMTPPort": "2500",|"SMTPPort": "10025",|g' config/config.json; - sed -i'.bak' 's|"ConnectionUrl": "http://dockerhost:9200",|"ConnectionUrl": "http://elasticsearch:9200",|g' config/config.json; mkdir -p client/plugins - cat config/config.json - make test-server BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' TESTFLAGS= TESTFLAGSEE= || exit 1 + make test-server BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' TESTFLAGS= TESTFLAGSEE= """ } } @@ -234,7 +231,7 @@ pipeline { dir('src/github.com/mattermost/mattermost-server/build') { ansiColor('xterm') { sh """ - COMPOSE_PROJECT_NAME=${DCNAME}-${env.BUILD_NUMBER} docker-compose down -v + docker-compose --no-ansi down -v """ } } diff --git a/build/docker-compose.yml b/build/docker-compose.yml index 75483d5cc5..6aa47e1767 100644 --- a/build/docker-compose.yml +++ b/build/docker-compose.yml @@ -3,7 +3,6 @@ services: mysql: image: "mysql:5.7" restart: always - container_name: mysql-$DCNAME-$BUILD_NUMBER networks: - mm-test environment: @@ -12,20 +11,22 @@ services: MYSQL_PASSWORD: mostest MYSQL_USER: mmuser MYSQL_DATABASE: mattermost_test + tmpfs: /var/lib/mysql + volumes: + - "./docker/mysql.conf.d:/etc/mysql/conf.d" postgres: image: "postgres:9.4" restart: always - container_name: postgres-$DCNAME-$BUILD_NUMBER networks: - mm-test environment: POSTGRES_USER: mmuser POSTGRES_PASSWORD: mostest POSTGRES_DB: mattermost_test + tmpfs: /var/lib/postgresql/data minio: image: "minio/minio:RELEASE.2018-05-25T19-49-13Z" command: "server /data" - container_name: minio-$DCNAME-$BUILD_NUMBER networks: - mm-test environment: @@ -34,13 +35,11 @@ services: inbucket: image: "jhillyerd/inbucket:release-1.2.0" restart: always - container_name: inbucket-$DCNAME-$BUILD_NUMBER networks: - mm-test openldap: image: "osixia/openldap:1.1.6" restart: always - container_name: openldap-$DCNAME-$BUILD_NUMBER networks: - mm-test environment: @@ -50,7 +49,6 @@ services: LDAP_ADMIN_PASSWORD: "mostest" elasticsearch: image: "mattermost/mattermost-elasticsearch-docker:6.5.1" - container_name: elasticsearch-$DCNAME-$BUILD_NUMBER networks: - mm-test environment: @@ -61,7 +59,6 @@ services: image: redis networks: - mm-test - container_name: redis-$DCNAME-$BUILD_NUMBER start_dependencies: image: mattermost/mattermost-wait-for-dep:latest diff --git a/build/docker/mysql.conf.d/custom.cnf b/build/docker/mysql.conf.d/custom.cnf new file mode 100644 index 0000000000..27722b73d8 --- /dev/null +++ b/build/docker/mysql.conf.d/custom.cnf @@ -0,0 +1,19 @@ +[mysqld] +bind-address = 0.0.0.0 + +log-output = NONE +slow-query-log = 0 +innodb_flush_log_at_trx_commit = 2 +innodb_flush_method = nosync +innodb_lock_wait_timeout = 50 +innodb_log_buffer_size = 3M +innodb_buffer_pool_size = 180M + +max_connect_errors = 1000000 +max_connections = 900 + +character-set-server = utf8 +sql_mode = "" +innodb = FORCE +default-storage-engine = InnoDB +max_allowed_packet = 256M diff --git a/build/local-test-env.sh b/build/local-test-env.sh index 0cb14527b7..e895bf8788 100755 --- a/build/local-test-env.sh +++ b/build/local-test-env.sh @@ -1,7 +1,6 @@ #!/bin/bash -export DCNAME=localdev -export BUILD_NUMBER=null +export COMPOSE_PROJECT_NAME=localdev local_cmdname=${0##*/} usage() @@ -15,50 +14,41 @@ USAGE up() { - COMPOSE_PROJECT_NAME=$DCNAME-$BUILD_NUMBER docker-compose run --rm start_dependencies - COMPOSE_PROJECT_NAME=$DCNAME-$BUILD_NUMBER docker-compose up -d mysql postgres minio inbucket openldap elasticsearch redis + docker-compose run --rm start_dependencies - docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'echo -e "dn: ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: organizationalunit" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' - docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'echo -e "dn: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test1\nmail: success+testone@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' - docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" -w mostest' - docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'echo -e "dn: uid=test.two,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test2\nmail: success+testtwo@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' - docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.two,ou=testusers,dc=mm,dc=test,dc=com" -w mostest' - docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'echo -e "dn: cn=tgroup,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: groupOfUniqueNames\nuniqueMember: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' + docker-compose exec openldap bash -c 'echo -e "dn: ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: organizationalunit" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' + docker-compose exec openldap bash -c 'echo -e "dn: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test1\nmail: success+testone@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' + docker-compose exec openldap bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" -w mostest' + docker-compose exec openldap bash -c 'echo -e "dn: uid=test.two,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test2\nmail: success+testtwo@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' + docker-compose exec openldap bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.two,ou=testusers,dc=mm,dc=test,dc=com" -w mostest' + docker-compose exec openldap bash -c 'echo -e "dn: cn=tgroup,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: groupOfUniqueNames\nuniqueMember: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' - echo "Changing the config.json" - sed -i'.bak' 's|mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8|mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json - sed -i'.bak' 's|"SMTPServer": "dockerhost",|"SMTPServer": "inbucket",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json - sed -i'.bak' 's|"SMTPPort": "2500",|"SMTPPort": "10025",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json - sed -i'.bak' 's|"ConnectionUrl": "http://dockerhost:9200",|"ConnectionUrl": "http://elasticsearch:9200",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json - - docker run -it -u root \ - --privileged \ - -v $GOPATH:/go \ - -w /go/src/github.com/mattermost/mattermost-server/ \ - --net $DCNAME-$BUILD_NUMBER\_mm-test \ - -e GOPATH="/go" \ - -e TEST_DATABASE_MYSQL_DSN="mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s" \ - -e TEST_DATABASE_POSTGRESQL_DSN="postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" \ - -e TEST_DATABASE_MYSQL_ROOT_PASSWD="mostest" \ - -e CI_INBUCKET_HOST="inbucket" \ - -e CI_MINIO_HOST="minio" \ - -e CI_INBUCKET_PORT="10080" \ - -e CI_MINIO_PORT="9000" \ - -e CI_INBUCKET_SMTP_PORT="10025" \ - -e CI_LDAP_HOST="openldap" \ - -e IS_CI=true \ - mattermost/mattermost-build-server:dec-7-2018 /bin/bash + docker run -it -u root \ + --privileged \ + -v $GOPATH:/go \ + -w /go/src/github.com/mattermost/mattermost-server/ \ + --net ${COMPOSE_PROJECT_NAME}_mm-test \ + -e GOPATH="/go" \ + -e TEST_DATABASE_MYSQL_DSN="mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s" \ + -e TEST_DATABASE_POSTGRESQL_DSN="postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" \ + -e TEST_DATABASE_MYSQL_ROOT_PASSWD="mostest" \ + -e CI_INBUCKET_HOST="inbucket" \ + -e CI_MINIO_HOST="minio" \ + -e CI_INBUCKET_PORT="10080" \ + -e CI_MINIO_PORT="9000" \ + -e CI_INBUCKET_SMTP_PORT="10025" \ + -e CI_LDAP_HOST="openldap" \ + -e IS_CI=true \ + -e MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8" \ + -e MM_EMAILSETTINGS_SMTPSERVER="inbucket" \ + -e MM_EMAILSETTINGS_SMTPPORT="10025" \ + -e MM_ELASTICSEARCHSETTINGS_CONNECTIONURL="http://elasticsearch:9200" \ + mattermost/mattermost-build-server:dec-7-2018 /bin/bash } down() { - COMPOSE_PROJECT_NAME=$DCNAME-$BUILD_NUMBER docker-compose down - - echo "Reverting the changes in the config.json" - sed -i'.bak' 's|mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8|mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json - sed -i'.bak' 's|"SMTPServer": "inbucket",|"SMTPServer": "dockerhost",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json - sed -i'.bak' 's|"SMTPPort": "10025",|"SMTPPort": "2500",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json - sed -i'.bak' 's|"ConnectionUrl": "http://elasticsearch:9200",|"ConnectionUrl": "http://dockerhost:9200",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json + docker-compose down } # process arguments @@ -70,16 +60,19 @@ do up break ;; + down) echo "Stopping Containers" down break ;; + --help) usage ;; + *) - echoerr "Unknown argument: $1" + echo "Unknown argument: $1" >&2 usage ;; esac diff --git a/cmd/mattermost/commands/cmdtestlib.go b/cmd/mattermost/commands/cmdtestlib.go index 24ae39a5c4..69d04adfbe 100644 --- a/cmd/mattermost/commands/cmdtestlib.go +++ b/cmd/mattermost/commands/cmdtestlib.go @@ -132,28 +132,39 @@ func (h *testHelper) execArgs(t *testing.T, args []string) []string { return ret } -// CheckCommand invokes the test binary, returning the output modified for assertion testing. -func (h *testHelper) CheckCommand(t *testing.T, args ...string) string { +func (h *testHelper) cmd(t *testing.T, args []string) *exec.Cmd { path, err := os.Executable() require.NoError(t, err) - output, err := exec.Command(path, h.execArgs(t, args)...).CombinedOutput() + cmd := exec.Command(path, h.execArgs(t, args)...) + + cmd.Env = []string{} + for _, env := range os.Environ() { + // Ignore MM_SQLSETTINGS_DATASOURCE from the environment, since we override. + if strings.HasPrefix(env, "MM_SQLSETTINGS_DATASOURCE=") { + continue + } + + cmd.Env = append(cmd.Env, env) + } + + return cmd +} + +// CheckCommand invokes the test binary, returning the output modified for assertion testing. +func (h *testHelper) CheckCommand(t *testing.T, args ...string) string { + output, err := h.cmd(t, args).CombinedOutput() require.NoError(t, err, string(output)) return strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(string(output)), "PASS")) } // RunCommand invokes the test binary, returning only any error. func (h *testHelper) RunCommand(t *testing.T, args ...string) error { - path, err := os.Executable() - require.NoError(t, err) - return exec.Command(path, h.execArgs(t, args)...).Run() + return h.cmd(t, args).Run() } // RunCommandWithOutput is a variant of RunCommand that returns the unmodified output and any error. func (h *testHelper) RunCommandWithOutput(t *testing.T, args ...string) (string, error) { - path, err := os.Executable() - require.NoError(t, err) - - cmd := exec.Command(path, h.execArgs(t, args)...) + cmd := h.cmd(t, args) var buf bytes.Buffer reader, writer := io.Pipe() @@ -166,7 +177,7 @@ func (h *testHelper) RunCommandWithOutput(t *testing.T, args ...string) (string, close(done) }() - err = cmd.Run() + err := cmd.Run() writer.Close() <-done