Merge pull request #1215 from shlinkio/develop

Release 2.9.2

CHANGELOG.md

@@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com), and this project adheres to [Semantic Versioning](https://semver.org).
 
-## [Unreleased]
+## [2.9.2] - 2021-10-23
 ### Added
 * *Nothing*
 
@@ -18,7 +18,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this
 * *Nothing*
 
 ### Fixed
-* [#1210](https://github.com/shlinkio/shlink/issues/1210) Fixed real time updates not being notified.
+* [#1210](https://github.com/shlinkio/shlink/issues/1210) Fixed real time updates not being notified due to an incorrect handling of db transactions on multi-process tasks.
+* [#1211](https://github.com/shlinkio/shlink/issues/1211) Fixed `There is no active transaction` error when running migrations in MySQL/Mariadb after updating to doctrine-migrations 3.3.
+* [#1197](https://github.com/shlinkio/shlink/issues/1197) Fixed amount of task workers provided via config option or env var not being validated to ensure enough workers to process all parallel tasks.
 
 
 ## [2.9.1] - 2021-10-11

composer.json

@@ -18,7 +18,7 @@
         "akrabat/ip-address-middleware": "^2.0",
         "cakephp/chronos": "^2.2",
         "cocur/slugify": "^4.0",
-        "doctrine/migrations": "^3.2 <3.3",
+        "doctrine/migrations": "^3.3",
         "doctrine/orm": "^2.9",
         "endroid/qr-code": "^4.2",
         "geoip2/geoip2": "^2.11",
@@ -51,7 +51,7 @@
         "shlinkio/shlink-config": "^1.2",
         "shlinkio/shlink-event-dispatcher": "^2.1",
         "shlinkio/shlink-importer": "^2.3.1",
-        "shlinkio/shlink-installer": "^6.2",
+        "shlinkio/shlink-installer": "^6.2.1",
         "shlinkio/shlink-ip-geolocation": "^2.0",
         "symfony/console": "^5.3",
         "symfony/filesystem": "^5.3",
@@ -73,7 +73,7 @@
         "phpunit/phpunit": "^9.5",
         "roave/security-advisories": "dev-master",
         "shlinkio/php-coding-standard": "~2.2.0",
-        "shlinkio/shlink-test-utils": "^2.2",
+        "shlinkio/shlink-test-utils": "^2.3",
         "symfony/var-dumper": "^5.3",
         "veewee/composer-run-parallel": "^1.0"
     },

config/autoload/swoole.global.php

@@ -4,22 +4,28 @@ declare(strict_types=1);
 
 use function Shlinkio\Shlink\Common\env;
 
-return [
+use const Shlinkio\Shlink\MIN_TASK_WORKERS;
 
-    'mezzio-swoole' => [
-        // Setting this to true can have unexpected behaviors when running several concurrent slow DB queries
-        'enable_coroutine' => false,
+return (static function () {
+    $taskWorkers = (int) env('TASK_WORKER_NUM', 16);
 
-        'swoole-http-server' => [
-            'host' => '0.0.0.0',
-            'port' => (int) env('PORT', 8080),
-            'process-name' => 'shlink',
+    return [
 
-            'options' => [
-                'worker_num' => (int) env('WEB_WORKER_NUM', 16),
-                'task_worker_num' => (int) env('TASK_WORKER_NUM', 16),
+        'mezzio-swoole' => [
+            // Setting this to true can have unexpected behaviors when running several concurrent slow DB queries
+            'enable_coroutine' => false,
+
+            'swoole-http-server' => [
+                'host' => '0.0.0.0',
+                'port' => (int) env('PORT', 8080),
+                'process-name' => 'shlink',
+
+                'options' => [
+                    'worker_num' => (int) env('WEB_WORKER_NUM', 16),
+                    'task_worker_num' => $taskWorkers < MIN_TASK_WORKERS ? MIN_TASK_WORKERS : $taskWorkers,
+                ],
             ],
         ],
-    ],
 
-];
+    ];
+})();

config/constants.php

@@ -18,3 +18,4 @@ const DEFAULT_QR_CODE_SIZE = 300;
 const DEFAULT_QR_CODE_MARGIN = 0;
 const DEFAULT_QR_CODE_FORMAT = 'png';
 const DEFAULT_QR_CODE_ERROR_CORRECTION = 'l';
+const MIN_TASK_WORKERS = 4;

config/test/bootstrap_api_tests.php

@@ -29,6 +29,6 @@ register_shutdown_function(function () use ($httpClient): void {
     );
 });
 
-$testHelper->createTestDb();
+$testHelper->createTestDb(['bin/cli', 'db:create'], ['bin/cli', 'db:migrate']);
 ApiTest\ApiTestCase::setApiClient($httpClient);
 ApiTest\ApiTestCase::setSeedFixturesCallback(fn () => $testHelper->seedFixtures($em, $config['data_fixtures'] ?? []));

data/migrations/Version20160819142757.php

@@ -39,6 +39,11 @@ class Version20160819142757 extends AbstractMigration
      */
     public function down(Schema $schema): void
     {
-        $db = $this->connection->getDatabasePlatform()->getName();
+        $this->connection->getDatabasePlatform()->getName();
+    }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
     }
 }

data/migrations/Version20160820191203.php

@@ -73,4 +73,9 @@ class Version20160820191203 extends AbstractMigration
         $schema->dropTable('short_urls_in_tags');
         $schema->dropTable('tags');
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20171021093246.php

@@ -45,4 +45,9 @@ class Version20171021093246 extends AbstractMigration
         $shortUrls->dropColumn('valid_since');
         $shortUrls->dropColumn('valid_until');
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20171022064541.php

@@ -42,4 +42,9 @@ class Version20171022064541 extends AbstractMigration
 
         $shortUrls->dropColumn('max_visits');
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20180801183328.php

@@ -39,4 +39,9 @@ final class Version20180801183328 extends AbstractMigration
     {
         $schema->getTable('short_urls')->getColumn('short_code')->setLength($size);
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20180913205455.php

@@ -66,4 +66,9 @@ final class Version20180913205455 extends AbstractMigration
     {
         // Nothing to rollback
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20180915110857.php

@@ -47,4 +47,9 @@ final class Version20180915110857 extends AbstractMigration
     {
         // Nothing to run
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20181020060559.php

@@ -65,4 +65,9 @@ final class Version20181020060559 extends AbstractMigration
     {
         // No down
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20181020065148.php

@@ -38,4 +38,9 @@ final class Version20181020065148 extends AbstractMigration
     {
         // No down
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20181110175521.php

@@ -34,4 +34,9 @@ final class Version20181110175521 extends AbstractMigration
     {
         return $schema->getTable('visits')->getColumn('user_agent');
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20190824075137.php

@@ -34,4 +34,9 @@ final class Version20190824075137 extends AbstractMigration
     {
         return $schema->getTable('visits')->getColumn('referer');
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20190930165521.php

@@ -52,4 +52,9 @@ final class Version20190930165521 extends AbstractMigration
         $schema->getTable('short_urls')->dropColumn('domain_id');
         $schema->dropTable('domains');
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20191001201532.php

@@ -46,4 +46,9 @@ final class Version20191001201532 extends AbstractMigration
         $shortUrls->dropIndex('unique_short_code_plus_domain');
         $shortUrls->addUniqueIndex(['short_code']);
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20191020074522.php

@@ -34,4 +34,9 @@ final class Version20191020074522 extends AbstractMigration
     {
         return $schema->getTable('short_urls')->getColumn('original_url');
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20200105165647.php

@@ -93,4 +93,9 @@ final class Version20200105165647 extends AbstractMigration
             $visitLocations->dropColumn($colName);
         }
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20200106215144.php

@@ -44,4 +44,9 @@ final class Version20200106215144 extends AbstractMigration
             ]);
         }
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20200110182849.php

@@ -50,4 +50,9 @@ final class Version20200110182849 extends AbstractMigration
     {
         // No need (and no way) to undo this migration
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20200323190014.php

@@ -42,4 +42,9 @@ final class Version20200323190014 extends AbstractMigration
 
         $visitLocations->dropColumn('is_empty');
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20200503170404.php

@@ -24,4 +24,9 @@ final class Version20200503170404 extends AbstractMigration
         $this->skipIf(! $visits->hasIndex(self::INDEX_NAME));
         $visits->dropIndex(self::INDEX_NAME);
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20201023090929.php

@@ -41,4 +41,9 @@ final class Version20201023090929 extends AbstractMigration
         $shortUrls->dropColumn('import_original_short_code');
         $shortUrls->dropIndex('unique_imports');
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20201102113208.php

@@ -83,4 +83,9 @@ final class Version20201102113208 extends AbstractMigration
         $shortUrls->removeForeignKey('FK_' . self::API_KEY_COLUMN);
         $shortUrls->dropColumn(self::API_KEY_COLUMN);
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20210102174433.php

@@ -49,4 +49,9 @@ final class Version20210102174433 extends AbstractMigration
         $schema->getTable(self::TABLE_NAME)->dropIndex('UQ_role_plus_api_key');
         $schema->dropTable(self::TABLE_NAME);
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20210118153932.php

@@ -23,4 +23,9 @@ final class Version20210118153932 extends AbstractMigration
     public function down(Schema $schema): void
     {
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20210202181026.php

@@ -33,4 +33,9 @@ final class Version20210202181026 extends AbstractMigration
         $shortUrls->dropColumn(self::TITLE);
         $shortUrls->dropColumn('title_was_auto_resolved');
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20210207100807.php

@@ -40,4 +40,9 @@ final class Version20210207100807 extends AbstractMigration
         $visits->dropColumn('visited_url');
         $visits->dropColumn('type');
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20210306165711.php

@@ -34,4 +34,9 @@ final class Version20210306165711 extends AbstractMigration
 
         $apiKeys->dropColumn(self::COLUMN);
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20210522051601.php

@@ -23,4 +23,9 @@ final class Version20210522051601 extends AbstractMigration
         $this->skipIf(! $shortUrls->hasColumn('crawlable'));
         $shortUrls->dropColumn('crawlable');
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20210522124633.php

@@ -25,4 +25,9 @@ final class Version20210522124633 extends AbstractMigration
         $this->skipIf(! $visits->hasColumn(self::POTENTIAL_BOT_COLUMN));
         $visits->dropColumn(self::POTENTIAL_BOT_COLUMN);
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20210720143824.php

@@ -38,4 +38,9 @@ final class Version20210720143824 extends AbstractMigration
         $domainsTable->dropColumn('regular_not_found_redirect');
         $domainsTable->dropColumn('invalid_short_url_redirect');
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations/Version20211002072605.php

@@ -23,4 +23,9 @@ final class Version20211002072605 extends AbstractMigration
         $this->skipIf(! $shortUrls->hasColumn('forward_query'));
         $shortUrls->dropColumn('forward_query');
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

data/migrations_template.txt

@@ -18,4 +18,9 @@ final class <className> extends AbstractMigration
     {
 <down>
     }
+
+    public function isTransactional(): bool
+    {
+        return $this->connection->getDatabasePlatform()->getName() !== 'mysql';
+    }
 }

module/CLI/src/Command/Db/CreateDatabaseCommand.php

@@ -67,7 +67,7 @@ class CreateDatabaseCommand extends AbstractDatabaseCommand
 
         // In order to create the new database, we have to use a connection where the dbname was not set.
         // Otherwise, it will fail to connect and will not be able to create the new database
-        $schemaManager = $this->noDbNameConn->getSchemaManager();
+        $schemaManager = $this->noDbNameConn->createSchemaManager();
         $databases = $schemaManager->listDatabases();
         $shlinkDatabase = $this->regularConn->getDatabase();
 
@@ -80,7 +80,7 @@ class CreateDatabaseCommand extends AbstractDatabaseCommand
     {
         // If at least one of the shlink tables exist, we will consider the database exists somehow.
         // Any inconsistency should be taken care by the migrations
-        $schemaManager = $this->regularConn->getSchemaManager();
+        $schemaManager = $this->regularConn->createSchemaManager();
         return ! empty($schemaManager->listTableNames());
     }
 }

module/CLI/test/Command/Db/CreateDatabaseCommandTest.php

@@ -46,10 +46,10 @@ class CreateDatabaseCommandTest extends TestCase
         $this->databasePlatform = $this->prophesize(AbstractPlatform::class);
 
         $this->regularConn = $this->prophesize(Connection::class);
-        $this->regularConn->getSchemaManager()->willReturn($this->schemaManager->reveal());
+        $this->regularConn->createSchemaManager()->willReturn($this->schemaManager->reveal());
         $this->regularConn->getDatabasePlatform()->willReturn($this->databasePlatform->reveal());
         $noDbNameConn = $this->prophesize(Connection::class);
-        $noDbNameConn->getSchemaManager()->willReturn($this->schemaManager->reveal());
+        $noDbNameConn->createSchemaManager()->willReturn($this->schemaManager->reveal());
 
         $command = new CreateDatabaseCommand(
             $locker->reveal(),

module/Rest/test/Action/HealthActionTest.php

@@ -6,6 +6,7 @@ namespace ShlinkioTest\Shlink\Rest\Action;
 
 use Doctrine\DBAL\Connection;
 use Doctrine\DBAL\Platforms\AbstractPlatform;
+use Doctrine\DBAL\Result;
 use Doctrine\ORM\EntityManagerInterface;
 use Exception;
 use Laminas\Diactoros\Response\JsonResponse;
@@ -27,8 +28,7 @@ class HealthActionTest extends TestCase
     public function setUp(): void
     {
         $this->conn = $this->prophesize(Connection::class);
-        $this->conn->executeQuery(Argument::cetera())->will(function (): void {
-        });
+        $this->conn->executeQuery(Argument::cetera())->willReturn($this->prophesize(Result::class)->reveal());
         $dbPlatform = $this->prophesize(AbstractPlatform::class);
         $dbPlatform->getDummySelectSQL()->willReturn('');
         $this->conn->getDatabasePlatform()->willReturn($dbPlatform->reveal());