From de2d87a6d958fa7e66c4d3a642d1c2f375cf7a22 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 11 Dec 2021 10:25:58 +0100 Subject: [PATCH 1/2] Unified jobs in ci pipeline as much as possible --- .github/workflows/ci.yml | 164 ++++-------------- bin/test/run-api-tests.sh | 1 + composer.json | 1 + config/autoload/entity-manager.local.php.dist | 2 +- config/test/test_config.global.php | 7 +- docker-compose.ci.yml | 2 +- docker-compose.override.yml.dist | 2 +- docker-compose.yml | 8 +- .../Config/SimplifiedConfigParserTest.php | 4 +- 9 files changed, 47 insertions(+), 144 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b24f9135..67b5b6b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,107 +27,17 @@ jobs: - run: composer install --no-interaction --prefer-dist - run: composer ${{ matrix.command }} - unit-tests: - runs-on: ubuntu-20.04 - strategy: - matrix: - php-version: ['8.0', '8.1'] - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Use PHP - uses: shivammathur/setup-php@v2 - with: - php-version: ${{ matrix.php-version }} - tools: composer - extensions: openswoole-4.8.1 - coverage: pcov - ini-values: pcov.directory=module - - run: composer install --no-interaction --prefer-dist - - run: composer test:unit:ci - - uses: actions/upload-artifact@v2 - if: ${{ matrix.php-version == '8.0' }} - with: - name: coverage-unit - path: | - build/coverage-unit - build/coverage-unit.cov - - db-tests-sqlite: - runs-on: ubuntu-20.04 - strategy: - matrix: - php-version: ['8.0', '8.1'] - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Use PHP - uses: shivammathur/setup-php@v2 - with: - php-version: ${{ matrix.php-version }} - tools: composer - extensions: openswoole-4.8.1 - coverage: pcov - ini-values: pcov.directory=module - - run: composer install --no-interaction --prefer-dist - - run: composer test:db:sqlite:ci - - uses: actions/upload-artifact@v2 - if: ${{ matrix.php-version == '8.0' }} - with: - name: coverage-db - path: | - build/coverage-db - build/coverage-db.cov - - db-tests-mysql: - runs-on: ubuntu-20.04 - strategy: - matrix: - php-version: ['8.0', '8.1'] - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Start database server - run: docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db - - name: Use PHP - uses: shivammathur/setup-php@v2 - with: - php-version: ${{ matrix.php-version }} - tools: composer - extensions: openswoole-4.8.1 - coverage: none - - run: composer install --no-interaction --prefer-dist - - run: composer test:db:mysql - - db-tests-maria: - runs-on: ubuntu-20.04 - strategy: - matrix: - php-version: ['8.0', '8.1'] - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Start database server - run: docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_maria - - name: Use PHP - uses: shivammathur/setup-php@v2 - with: - php-version: ${{ matrix.php-version }} - tools: composer - extensions: openswoole-4.8.1 - coverage: none - - run: composer install --no-interaction --prefer-dist - - run: composer test:db:maria - - db-tests-postgres: + tests: runs-on: ubuntu-20.04 strategy: matrix: php-version: ['8.0', '8.1'] + test-group: ['unit', 'api'] steps: - name: Checkout code uses: actions/checkout@v2 - name: Start database server + if: ${{ matrix.test-group == 'api' }} run: docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_postgres - name: Use PHP uses: shivammathur/setup-php@v2 @@ -135,46 +45,35 @@ jobs: php-version: ${{ matrix.php-version }} tools: composer extensions: openswoole-4.8.1 - coverage: none + coverage: pcov + ini-values: pcov.directory=module - run: composer install --no-interaction --prefer-dist - - run: composer test:db:postgres + - run: composer test:${{ matrix.test-group }}:ci + - uses: actions/upload-artifact@v2 + if: ${{ matrix.php-version == '8.0' }} + with: + name: coverage-${{ matrix.test-group }} + path: | + build/coverage-${{ matrix.test-group }} + build/coverage-${{ matrix.test-group }}.cov - db-tests-ms: + db-tests: runs-on: ubuntu-20.04 strategy: matrix: php-version: ['8.0', '8.1'] + platform: ['sqlite:ci', 'mysql', 'maria', 'postgres', 'ms'] env: LC_ALL: C steps: - name: Checkout code uses: actions/checkout@v2 - name: Install MSSQL ODBC + if: ${{ matrix.platform == 'ms' }} run: sudo ./data/infra/ci/install-ms-odbc.sh - name: Start database server - run: docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms - - name: Use PHP - uses: shivammathur/setup-php@v2 - with: - php-version: ${{ matrix.php-version }} - tools: composer - extensions: openswoole-4.8.1, pdo_sqlsrv-5.10.0beta2 - coverage: none - - run: composer install --no-interaction --prefer-dist - - name: Create test database - run: docker-compose exec -T shlink_db_ms /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'Passw0rd!' -Q "CREATE DATABASE shlink_test;" - - run: composer test:db:ms - - api-tests: - runs-on: ubuntu-20.04 - strategy: - matrix: - php-version: ['8.0', '8.1'] - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Start database server - run: docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_postgres + if: ${{ matrix.platform != 'sqlite:ci' }} + run: docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_${{ matrix.platform }} - name: Use PHP uses: shivammathur/setup-php@v2 with: @@ -184,20 +83,24 @@ jobs: coverage: pcov ini-values: pcov.directory=module - run: composer install --no-interaction --prefer-dist - - run: bin/test/run-api-tests.sh - - uses: actions/upload-artifact@v2 - if: ${{ matrix.php-version == '8.0' }} + - name: Create test database + if: ${{ matrix.platform == 'ms' }} + run: docker-compose exec -T shlink_db_ms /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'Passw0rd!' -Q "CREATE DATABASE shlink_test;" + - name: Run tests + run: composer test:db:${{ matrix.platform }} + - name: Upload code coverage + uses: actions/upload-artifact@v2 + if: ${{ matrix.php-version == '8.0' && matrix.platform == 'sqlite:ci' }} with: - name: coverage-api + name: coverage-db path: | - build/coverage-api - build/coverage-api.cov + build/coverage-db + build/coverage-db.cov mutation-tests: needs: - - unit-tests - - db-tests-sqlite - - api-tests + - tests + - db-tests runs-on: ubuntu-20.04 strategy: matrix: @@ -227,9 +130,8 @@ jobs: upload-coverage: needs: - - unit-tests - - db-tests-sqlite - - api-tests + - tests + - db-tests runs-on: ubuntu-20.04 strategy: matrix: diff --git a/bin/test/run-api-tests.sh b/bin/test/run-api-tests.sh index dbd87a84..3e8530b6 100755 --- a/bin/test/run-api-tests.sh +++ b/bin/test/run-api-tests.sh @@ -2,6 +2,7 @@ export APP_ENV=test export DB_DRIVER=postgres export TEST_ENV=api +export GENERATE_COVERAGE=${GENERATE_COVERAGE:-"no"} rm -rf data/log/api-tests diff --git a/composer.json b/composer.json index 615173c0..6364443a 100644 --- a/composer.json +++ b/composer.json @@ -139,6 +139,7 @@ "test:db:postgres": "DB_DRIVER=postgres composer test:db:sqlite", "test:db:ms": "DB_DRIVER=mssql composer test:db:sqlite", "test:api": "bin/test/run-api-tests.sh", + "test:api:ci": "GENERATE_COVERAGE=yes composer test:api", "infect:ci:base": "infection --threads=4 --log-verbosity=default --only-covered --only-covering-test-cases --skip-initial-tests", "infect:ci:unit": "@infect:ci:base --coverage=build/coverage-unit --min-msi=83", "infect:ci:db": "@infect:ci:base --coverage=build/coverage-db --min-msi=95 --configuration=infection-db.json", diff --git a/config/autoload/entity-manager.local.php.dist b/config/autoload/entity-manager.local.php.dist index c4d2b921..ef5cabf8 100644 --- a/config/autoload/entity-manager.local.php.dist +++ b/config/autoload/entity-manager.local.php.dist @@ -9,7 +9,7 @@ return [ 'user' => 'root', 'password' => 'root', 'driver' => 'pdo_mysql', - 'host' => 'shlink_db', + 'host' => 'shlink_db_mysql', 'dbname' => 'shlink', 'charset' => 'utf8', ], diff --git a/config/test/test_config.global.php b/config/test/test_config.global.php index ab7a910f..0898c732 100644 --- a/config/test/test_config.global.php +++ b/config/test/test_config.global.php @@ -30,7 +30,8 @@ use const ShlinkioTest\Shlink\SWOOLE_TESTING_HOST; use const ShlinkioTest\Shlink\SWOOLE_TESTING_PORT; $isApiTest = env('TEST_ENV') === 'api'; -if ($isApiTest) { +$generateCoverage = env('GENERATE_COVERAGE') === 'yes'; +if ($isApiTest && $generateCoverage) { $filter = new Filter(); $filter->includeDirectory(__DIR__ . '/../../module/Core/src'); $filter->includeDirectory(__DIR__ . '/../../module/Rest/src'); @@ -40,7 +41,6 @@ if ($isApiTest) { $buildDbConnection = static function (): array { $driver = env('DB_DRIVER', 'sqlite'); $isCi = env('CI', false); - $getMysqlHost = static fn (string $driver) => sprintf('shlink_db%s', $driver === 'mysql' ? '' : '_maria'); $getCiMysqlPort = static fn (string $driver) => $driver === 'mysql' ? '3307' : '3308'; return match ($driver) { @@ -66,7 +66,7 @@ $buildDbConnection = static function (): array { ], default => [ // mysql and maria 'driver' => 'pdo_mysql', - 'host' => $isCi ? '127.0.0.1' : $getMysqlHost($driver), + 'host' => $isCi ? '127.0.0.1' : sprintf('shlink_db_%s', $driver), 'port' => $isCi ? $getCiMysqlPort($driver) : '3306', 'user' => 'root', 'password' => 'root', @@ -124,7 +124,6 @@ return [ if ($coverage) { // @phpstan-ignore-line $basePath = __DIR__ . '/../../build/coverage-api'; - // TODO Generate these coverages dynamically based on CLI options (new PHP())->process($coverage, $basePath . '.cov'); (new Xml(Version::getVersionString()))->process($coverage, $basePath . '/coverage-xml'); (new Html())->process($coverage, $basePath . '/coverage-html'); diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 3783fef2..f4235dcb 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -1,7 +1,7 @@ version: '3' services: - shlink_db: + shlink_db_mysql: environment: MYSQL_DATABASE: shlink_test diff --git a/docker-compose.override.yml.dist b/docker-compose.override.yml.dist index ea0bee84..990d1b5d 100644 --- a/docker-compose.override.yml.dist +++ b/docker-compose.override.yml.dist @@ -13,7 +13,7 @@ services: - /etc/passwd:/etc/passwd:ro - /etc/group:/etc/group:ro - shlink_db: + shlink_db_mysql: user: 1000:1000 volumes: - /etc/passwd:/etc/passwd:ro diff --git a/docker-compose.yml b/docker-compose.yml index 8ea38d98..d9dd776c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,7 @@ services: - ./:/home/shlink/www - ./data/infra/php.ini:/usr/local/etc/php/php.ini links: - - shlink_db + - shlink_db_mysql - shlink_db_postgres - shlink_db_maria - shlink_db_ms @@ -57,7 +57,7 @@ services: - ./:/home/shlink - ./data/infra/php.ini:/usr/local/etc/php/php.ini links: - - shlink_db + - shlink_db_mysql - shlink_db_postgres - shlink_db_maria - shlink_db_ms @@ -69,8 +69,8 @@ services: extra_hosts: - 'host.docker.internal:host-gateway' - shlink_db: - container_name: shlink_db + shlink_db_mysql: + container_name: shlink_db_mysql image: mysql:5.7 ports: - "3307:3306" diff --git a/module/Core/test/Config/SimplifiedConfigParserTest.php b/module/Core/test/Config/SimplifiedConfigParserTest.php index f4e5c8f0..48d41c00 100644 --- a/module/Core/test/Config/SimplifiedConfigParserTest.php +++ b/module/Core/test/Config/SimplifiedConfigParserTest.php @@ -29,7 +29,7 @@ class SimplifiedConfigParserTest extends TestCase 'entity_manager' => [ 'connection' => [ 'driver' => 'mysql', - 'host' => 'shlink_db', + 'host' => 'shlink_db_mysql', 'port' => '3306', ], ], @@ -78,7 +78,7 @@ class SimplifiedConfigParserTest extends TestCase 'entity_manager' => [ 'connection' => [ 'driver' => 'mysql', - 'host' => 'shlink_db', + 'host' => 'shlink_db_mysql', 'dbname' => 'shlink', 'user' => 'foo', 'password' => 'bar', From 7d7c0011bbf14727f440e5917acca5ce6f131932 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 11 Dec 2021 10:33:00 +0100 Subject: [PATCH 2/2] Fixed references to test:api and test:api:ci inside composer.json and added missing driver for MS SQL --- .github/workflows/ci.yml | 2 +- composer.json | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67b5b6b5..051be99f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: with: php-version: ${{ matrix.php-version }} tools: composer - extensions: openswoole-4.8.1 + extensions: openswoole-4.8.1, pdo_sqlsrv-5.10.0beta2 coverage: pcov ini-values: pcov.directory=module - run: composer install --no-interaction --prefer-dist diff --git a/composer.json b/composer.json index 6364443a..b40d24ba 100644 --- a/composer.json +++ b/composer.json @@ -126,7 +126,7 @@ "test:ci": [ "@test:unit:ci", "@test:db", - "@test:api" + "@test:api:ci" ], "test:unit": "@php vendor/bin/phpunit --order-by=random --colors=always --coverage-php build/coverage-unit.cov --testdox", "test:unit:ci": "@test:unit --coverage-xml=build/coverage-unit/coverage-xml --log-junit=build/coverage-unit/junit.xml", @@ -146,7 +146,7 @@ "infect:ci:api": "@infect:ci:base --coverage=build/coverage-api --min-msi=80 --configuration=infection-api.json", "infect:ci": "@parallel infect:ci:unit infect:ci:db infect:ci:api", "infect:test": [ - "@parallel test:unit:ci test:db:sqlite:ci test:api", + "@parallel test:unit:ci test:db:sqlite:ci test:api:ci", "@infect:ci" ], "infect:test:unit": [ @@ -154,7 +154,7 @@ "@infect:ci:unit" ], "infect:test:api": [ - "@test:api", + "@test:api:ci", "@infect:ci:api" ], "swagger:validate": "php-openapi validate docs/swagger/swagger.json", @@ -171,6 +171,7 @@ "test:ci": "Runs all test suites, generating all needed reports and logs for CI envs", "test:unit": "Runs unit test suites", "test:unit:ci": "Runs unit test suites, generating all needed reports and logs for CI envs", + "test:unit:pretty": "Runs unit test suites and generates an HTML code coverage report", "test:db": "Runs database test suites on a SQLite, MySQL, MariaDB, PostgreSQL and MsSQL", "test:db:sqlite": "Runs database test suites on a SQLite database", "test:db:sqlite:ci": "Runs database test suites on a SQLite database, generating all needed reports and logs for CI envs", @@ -179,7 +180,7 @@ "test:db:postgres": "Runs database test suites on a PostgreSQL database", "test:db:ms": "Runs database test suites on a Miscrosoft SQL Server database", "test:api": "Runs API test suites", - "test:unit:pretty": "Runs unit test suites and generates an HTML code coverage report", + "test:api:ci": "Runs API test suites, and generates code coverage reports", "infect:ci": "Checks unit and db tests quality applying mutation testing with existing reports and logs", "infect:ci:unit": "Checks unit tests quality applying mutation testing with existing reports and logs", "infect:ci:db": "Checks db tests quality applying mutation testing with existing reports and logs",