OilfieldToolsNet/shlink
4
0
Fork 0
mirror of https://github.com/shlinkio/shlink.git synced 2025-02-25 18:45:27 -06:00
Code Issues Packages Projects Releases Wiki Activity

Use native query builders for all queries/sub-queries in tags list

Browse Source
This commit is contained in:
Alejandro Celaya
2023-02-24 18:01:32 +01:00
parent 658303d375
commit 652b0df054
1 changed files with 45 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
module/Core/src/Tag/Repository/TagRepository.php
View File

@@ -15,7 +15,6 @@ use Shlinkio\Shlink\Core\Tag\Model\TagsListFiltering;
use Shlinkio\Shlink\Core\Tag\Spec\CountTagsWithName; use Shlinkio\Shlink\Core\Tag\Spec\CountTagsWithName;
use Shlinkio\Shlink\Rest\ApiKey\Role; use Shlinkio\Shlink\Rest\ApiKey\Role;
use Shlinkio\Shlink\Rest\ApiKey\Spec\WithApiKeySpecsEnsuringJoin; use Shlinkio\Shlink\Rest\ApiKey\Spec\WithApiKeySpecsEnsuringJoin;
use Shlinkio\Shlink\Rest\ApiKey\Spec\WithInlinedApiKeySpecsEnsuringJoin;
use Shlinkio\Shlink\Rest\Entity\ApiKey; use Shlinkio\Shlink\Rest\Entity\ApiKey;
use function Functional\map; use function Functional\map;
@@ -46,93 +45,99 @@ class TagRepository extends EntitySpecificationRepository implements TagReposito
$orderDir = $filtering?->orderBy?->direction; $orderDir = $filtering?->orderBy?->direction;
$orderMainQuery = $orderField !== null && OrderableField::isAggregateField($orderField); $orderMainQuery = $orderField !== null && OrderableField::isAggregateField($orderField);
$subQb = $this->createQueryBuilder('t'); $conn = $this->getEntityManager()->getConnection();
$subQb->select('t.id', 't.name'); $tagsSubQb = $conn->createQueryBuilder();
$tagsSubQb->select('t.id', 't.name')->from('tags', 't');
if (! $orderMainQuery) { if (! $orderMainQuery) {
$subQb->orderBy('t.name', $orderDir ?? 'ASC') $tagsSubQb
->setMaxResults($filtering?->limit ?? PHP_INT_MAX) ->orderBy('t.name', $orderDir ?? 'ASC')
->setFirstResult($filtering?->offset ?? 0); ->setMaxResults($filtering?->limit ?? PHP_INT_MAX)
->setFirstResult($filtering?->offset ?? 0);
// TODO Check if applying limit/offset ot visits sub-queries is needed with large amounts of tags // TODO Check if applying limit/offset ot visits sub-queries is needed with large amounts of tags
} }
$conn = $this->getEntityManager()->getConnection(); $buildVisitsSubQb = static function (bool $excludeBots, string $aggregateAlias) use ($conn) {
$buildVisitsSubQuery = static function (bool $excludeBots, string $aggregateAlias) use ($conn) { $visitsSubQb = $conn->createQueryBuilder();
$visitsSubQuery = $conn->createQueryBuilder(); $commonJoinCondition = $visitsSubQb->expr()->eq('v.short_url_id', 's.id');
$commonJoinCondition = $visitsSubQuery->expr()->eq('v.short_url_id', 's.id');
$visitsJoin = ! $excludeBots $visitsJoin = ! $excludeBots
? $commonJoinCondition ? $commonJoinCondition
: $visitsSubQuery->expr()->and( : $visitsSubQb->expr()->and(
$commonJoinCondition, $commonJoinCondition,
$visitsSubQuery->expr()->eq('v.potential_bot', $conn->quote('0')), $visitsSubQb->expr()->eq('v.potential_bot', $conn->quote('0')),
); );
return $visitsSubQuery return $visitsSubQb
->select('st.tag_id AS tag_id', 'COUNT(DISTINCT v.id) AS ' . $aggregateAlias) ->select('st.tag_id AS tag_id', 'COUNT(DISTINCT v.id) AS ' . $aggregateAlias)
->from('visits', 'v') ->from('visits', 'v')
->join('v', 'short_urls', 's', $visitsJoin) // @phpstan-ignore-line ->join('v', 'short_urls', 's', $visitsJoin) // @phpstan-ignore-line
->join('s', 'short_urls_in_tags', 'st', $visitsSubQuery->expr()->eq('st.short_url_id', 's.id')) ->join('s', 'short_urls_in_tags', 'st', $visitsSubQb->expr()->eq('st.short_url_id', 's.id'))
->groupBy('st.tag_id'); ->groupBy('st.tag_id');
}; };
$allVisitsSubQuery = $buildVisitsSubQuery(false, 'visits'); $allVisitsSubQb = $buildVisitsSubQb(false, 'visits');
$nonBotVisitsSubQuery = $buildVisitsSubQuery(true, 'non_bot_visits'); $nonBotVisitsSubQb = $buildVisitsSubQb(true, 'non_bot_visits');
$searchTerm = $filtering?->searchTerm; $searchTerm = $filtering?->searchTerm;
if ($searchTerm !== null) { if ($searchTerm !== null) {
$subQb->andWhere($subQb->expr()->like('t.name', $conn->quote('%' . $searchTerm . '%'))); $tagsSubQb->andWhere($tagsSubQb->expr()->like('t.name', $conn->quote('%' . $searchTerm . '%')));
// TODO Check if applying this to all sub-queries makes it faster or slower // TODO Check if applying this to all sub-queries makes it faster or slower
} }
$apiKey = $filtering?->apiKey; $apiKey = $filtering?->apiKey;
$applyApiKeyToNativeQuery = static fn (?ApiKey $apiKey, NativeQueryBuilder $nativeQueryBuilder) => $applyApiKeyToNativeQb = static fn (?ApiKey $apiKey, NativeQueryBuilder $qb) =>
$apiKey?->mapRoles(static fn (Role $role, array $meta) => match ($role) { $apiKey?->mapRoles(static fn (Role $role, array $meta) => match ($role) {
Role::DOMAIN_SPECIFIC => $nativeQueryBuilder->andWhere( Role::DOMAIN_SPECIFIC => $qb->andWhere(
$nativeQueryBuilder->expr()->eq('s.domain_id', $conn->quote(Role::domainIdFromMeta($meta))), $qb->expr()->eq('s.domain_id', $conn->quote(Role::domainIdFromMeta($meta))),
), ),
Role::AUTHORED_SHORT_URLS => $nativeQueryBuilder->andWhere( Role::AUTHORED_SHORT_URLS => $qb->andWhere(
$nativeQueryBuilder->expr()->eq('s.author_api_key_id', $conn->quote($apiKey->getId())), $qb->expr()->eq('s.author_api_key_id', $conn->quote($apiKey->getId())),
), ),
}); });
// Apply API key specification to all sub-queries // Apply API key specification to all sub-queries
$this->applySpecification($subQb, new WithInlinedApiKeySpecsEnsuringJoin($apiKey), 't'); if ($apiKey && ! $apiKey->isAdmin()) {
$applyApiKeyToNativeQuery($apiKey, $allVisitsSubQuery); $tagsSubQb
$applyApiKeyToNativeQuery($apiKey, $nonBotVisitsSubQuery); ->join('t', 'short_urls_in_tags', 'st', $tagsSubQb->expr()->eq('st.tag_id', 't.id'))
->join('st', 'short_urls', 's', $tagsSubQb->expr()->eq('st.short_url_id', 's.id'));
}
$applyApiKeyToNativeQb($apiKey, $tagsSubQb);
$applyApiKeyToNativeQb($apiKey, $allVisitsSubQb);
$applyApiKeyToNativeQb($apiKey, $nonBotVisitsSubQb);
// A native query builder needs to be used here, because DQL and ORM query builders do not support // A native query builder needs to be used here, because DQL and ORM query builders do not support
// sub-queries at "from" and "join" level. // sub-queries at "from" and "join" level.
// If no sub-query is used, the whole list is loaded even with pagination, making it very inefficient. // If no sub-query is used, the whole list is loaded even with pagination, making it very inefficient.
$nativeQb = $conn->createQueryBuilder(); $mainQb = $conn->createQueryBuilder();
$nativeQb $mainQb
->select( ->select(
't.id_0 AS id', 't.id AS id',
't.name_1 AS name', 't.name AS name',
'COALESCE(v.visits, 0) AS visits', // COALESCE required for postgres to properly order 'COALESCE(v.visits, 0) AS visits', // COALESCE required for postgres to properly order
'COALESCE(v2.non_bot_visits, 0) AS non_bot_visits', // COALESCE required for postgres to properly order 'COALESCE(v2.non_bot_visits, 0) AS non_bot_visits', // COALESCE required for postgres to properly order
'COUNT(DISTINCT s.id) AS short_urls_count', 'COUNT(DISTINCT s.id) AS short_urls_count',
) )
->from('(' . $subQb->getQuery()->getSQL() . ')', 't') // @phpstan-ignore-line ->from('(' . $tagsSubQb->getSQL() . ')', 't')
->leftJoin('t', 'short_urls_in_tags', 'st', $nativeQb->expr()->eq('t.id_0', 'st.tag_id')) ->leftJoin('t', 'short_urls_in_tags', 'st', $mainQb->expr()->eq('t.id', 'st.tag_id'))
->leftJoin('st', 'short_urls', 's', $nativeQb->expr()->eq('s.id', 'st.short_url_id')) ->leftJoin('st', 'short_urls', 's', $mainQb->expr()->eq('s.id', 'st.short_url_id'))
->leftJoin('t', '(' . $allVisitsSubQuery->getSQL() . ')', 'v', $nativeQb->expr()->eq('t.id_0', 'v.tag_id')) ->leftJoin('t', '(' . $allVisitsSubQb->getSQL() . ')', 'v', $mainQb->expr()->eq('t.id', 'v.tag_id'))
->leftJoin('t', '(' . $nonBotVisitsSubQuery->getSQL() . ')', 'v2', $nativeQb->expr()->eq( ->leftJoin('t', '(' . $nonBotVisitsSubQb->getSQL() . ')', 'v2', $mainQb->expr()->eq(
't.id_0', 't.id',
'v2.tag_id', 'v2.tag_id',
)) ))
->groupBy('t.id_0', 't.name_1', 'v.visits', 'v2.non_bot_visits'); ->groupBy('t.id', 't.name', 'v.visits', 'v2.non_bot_visits');
// Apply API key role conditions to the native query too, as they will affect the amounts on the aggregates // Apply API key role conditions to the native query too, as they will affect the amounts on the aggregates
$applyApiKeyToNativeQuery($apiKey, $nativeQb); $applyApiKeyToNativeQb($apiKey, $mainQb);
if ($orderMainQuery) { if ($orderMainQuery) {
$nativeQb $mainQb
->orderBy(OrderableField::toSnakeCaseValidField($orderField), $orderDir ?? 'ASC') ->orderBy(OrderableField::toSnakeCaseValidField($orderField), $orderDir ?? 'ASC')
->setMaxResults($filtering?->limit ?? PHP_INT_MAX) ->setMaxResults($filtering?->limit ?? PHP_INT_MAX)
->setFirstResult($filtering?->offset ?? 0); ->setFirstResult($filtering?->offset ?? 0);
} }
// Add ordering by tag name, as a fallback in case of same amount, or as default ordering // Add ordering by tag name, as a fallback in case of same amount, or as default ordering
$nativeQb->addOrderBy('t.name_1', $orderMainQuery || $orderDir === null ? 'ASC' : $orderDir); $mainQb->addOrderBy('t.name', $orderMainQuery || $orderDir === null ? 'ASC' : $orderDir);
$rsm = new ResultSetMappingBuilder($this->getEntityManager()); $rsm = new ResultSetMappingBuilder($this->getEntityManager());
$rsm->addScalarResult('name', 'tag'); $rsm->addScalarResult('name', 'tag');
@@ -141,7 +146,7 @@ class TagRepository extends EntitySpecificationRepository implements TagReposito
$rsm->addScalarResult('short_urls_count', 'shortUrlsCount'); $rsm->addScalarResult('short_urls_count', 'shortUrlsCount');
return map( return map(
$this->getEntityManager()->createNativeQuery($nativeQb->getSQL(), $rsm)->getResult(), $this->getEntityManager()->createNativeQuery($mainQb->getSQL(), $rsm)->getResult(),
TagInfo::fromRawData(...), TagInfo::fromRawData(...),
); );
} }