diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 23b76317..188a7f02 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        php-version: ['7.4']
+        php-version: ['8.0']
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -30,7 +30,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        php-version: ['7.4']
+        php-version: ['8.0']
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -242,7 +242,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        php-version: ['7.4']
+        php-version: ['8.0']
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5bcfe116..c5d6f749 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,6 +24,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this
     * `disable_referrer_tracking`: If true, the referrer will not be tracked.
     * `disable_ua_tracking`: If true, the user agent will not be tracked.
 
+* [#955](https://github.com/shlinkio/shlink/issues/955) Added new option to set short URLs as crawlable, making them be listed in the robots.txt as Allowed.
+
 ### Changed
 * [#1036](https://github.com/shlinkio/shlink/issues/1036) Updated to `happyr/doctrine-specification` 2.0.
 * [#1039](https://github.com/shlinkio/shlink/issues/1039) Updated to `endroid/qr-code` 4.0.
diff --git a/data/migrations/Version20210522051601.php b/data/migrations/Version20210522051601.php
new file mode 100644
index 00000000..9e2bd19e
--- /dev/null
+++ b/data/migrations/Version20210522051601.php
@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+namespace ShlinkMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\DBAL\Types\Types;
+use Doctrine\Migrations\AbstractMigration;
+
+final class Version20210522051601 extends AbstractMigration
+{
+    public function up(Schema $schema): void
+    {
+        $shortUrls = $schema->getTable('short_urls');
+        $this->skipIf($shortUrls->hasColumn('crawlable'));
+        $shortUrls->addColumn('crawlable', Types::BOOLEAN, ['default' => false]);
+    }
+
+    public function down(Schema $schema): void
+    {
+        $shortUrls = $schema->getTable('short_urls');
+        $this->skipIf(! $shortUrls->hasColumn('crawlable'));
+        $shortUrls->dropColumn('crawlable');
+    }
+}
diff --git a/docs/async-api/async-api.json b/docs/async-api/async-api.json
index df9bc6d6..3360d897 100644
--- a/docs/async-api/async-api.json
+++ b/docs/async-api/async-api.json
@@ -116,6 +116,15 @@
                     "domain": {
                         "type": "string",
                         "description": "The domain in which the short URL was created. Null if it belongs to default domain."
+                    },
+                    "title": {
+                        "type": "string",
+                        "nullable": true,
+                        "description": "A descriptive title of the short URL."
+                    },
+                    "crawlable": {
+                        "type": "boolean",
+                        "description": "Tells if this URL will be included as 'Allow' in Shlink's robots.txt."
                     }
                 },
                 "example": {
@@ -133,7 +142,9 @@
                         "validUntil": null,
                         "maxVisits": 100
                     },
-                    "domain": "example.com"
+                    "domain": "example.com",
+                    "title": "The title",
+                    "crawlable": false
                 }
             },
             "ShortUrlMeta": {
diff --git a/docs/swagger/definitions/ShortUrl.json b/docs/swagger/definitions/ShortUrl.json
index 3e4c6ead..b2ffd3f6 100644
--- a/docs/swagger/definitions/ShortUrl.json
+++ b/docs/swagger/definitions/ShortUrl.json
@@ -41,6 +41,10 @@
             "type": "string",
             "nullable": true,
             "description": "A descriptive title of the short URL."
+        },
+        "crawlable": {
+            "type": "boolean",
+            "description": "Tells if this URL will be included as 'Allow' in Shlink's robots.txt."
         }
     }
 }
diff --git a/docs/swagger/paths/v1_short-urls.json b/docs/swagger/paths/v1_short-urls.json
index b034dcf3..8cf22045 100644
--- a/docs/swagger/paths/v1_short-urls.json
+++ b/docs/swagger/paths/v1_short-urls.json
@@ -140,7 +140,8 @@
                                         "maxVisits": 100
                                     },
                                     "domain": null,
-                                    "title": "Welcome to Steam"
+                                    "title": "Welcome to Steam",
+                                    "crawlable": false
                                 },
                                 {
                                     "shortCode": "12Kb3",
@@ -157,7 +158,8 @@
                                         "maxVisits": null
                                     },
                                     "domain":  null,
-                                    "title": null
+                                    "title": null,
+                                    "crawlable": false
                                 },
                                 {
                                     "shortCode": "123bA",
@@ -172,7 +174,8 @@
                                         "maxVisits": null
                                     },
                                     "domain":  "example.com",
-                                    "title": null
+                                    "title": null,
+                                    "crawlable": false
                                 }
                             ],
                             "pagination": {
@@ -273,6 +276,10 @@
                             "title": {
                                 "type": "string",
                                 "description": "A descriptive title of the short URL."
+                            },
+                            "crawlable": {
+                                "type": "boolean",
+                                "description": "Tells if this URL will be included as 'Allow' in Shlink's robots.txt."
                             }
                         }
                     }
@@ -305,7 +312,9 @@
                             "validUntil": null,
                             "maxVisits": 500
                         },
-                        "domain": null
+                        "domain": null,
+                        "title": null,
+                        "crawlable": false
                     }
                 }
             },
diff --git a/docs/swagger/paths/v1_short-urls_shorten.json b/docs/swagger/paths/v1_short-urls_shorten.json
index b6184d8d..90c3eda5 100644
--- a/docs/swagger/paths/v1_short-urls_shorten.json
+++ b/docs/swagger/paths/v1_short-urls_shorten.json
@@ -74,7 +74,8 @@
                             "maxVisits": 100
                         },
                         "domain": null,
-                        "title": null
+                        "title": null,
+                        "crawlable": false
                     },
                     "text/plain": "https://doma.in/abc123"
                 }
diff --git a/docs/swagger/paths/v1_short-urls_{shortCode}.json b/docs/swagger/paths/v1_short-urls_{shortCode}.json
index 2281d9b8..8691c0b5 100644
--- a/docs/swagger/paths/v1_short-urls_{shortCode}.json
+++ b/docs/swagger/paths/v1_short-urls_{shortCode}.json
@@ -54,7 +54,8 @@
                             "maxVisits": 100
                         },
                         "domain": null,
-                        "title": null
+                        "title": null,
+                        "crawlable": false
                     }
                 }
             },
@@ -147,6 +148,10 @@
                                 "type": "string",
                                 "description": "A descriptive title of the short URL.",
                                 "nullable": true
+                            },
+                            "crawlable": {
+                                "type": "boolean",
+                                "description": "Tells if this URL will be included as 'Allow' in Shlink's robots.txt."
                             }
                         }
                     }
@@ -184,7 +189,8 @@
                             "maxVisits": 100
                         },
                         "domain": null,
-                        "title": "Shlink - The URL shortener"
+                        "title": "Shlink - The URL shortener",
+                        "crawlable": false
                     }
                 }
             },
diff --git a/module/Core/config/dependencies.config.php b/module/Core/config/dependencies.config.php
index b84c74a4..7dfd5df2 100644
--- a/module/Core/config/dependencies.config.php
+++ b/module/Core/config/dependencies.config.php
@@ -48,6 +48,7 @@ return [
             Action\RedirectAction::class => ConfigAbstractFactory::class,
             Action\PixelAction::class => ConfigAbstractFactory::class,
             Action\QrCodeAction::class => ConfigAbstractFactory::class,
+            Action\RobotsAction::class => ConfigAbstractFactory::class,
 
             ShortUrl\Resolver\PersistenceShortUrlRelationResolver::class => ConfigAbstractFactory::class,
             ShortUrl\Helper\ShortUrlStringifier::class => ConfigAbstractFactory::class,
@@ -57,6 +58,8 @@ return [
             Mercure\MercureUpdatesGenerator::class => ConfigAbstractFactory::class,
 
             Importer\ImportedLinksProcessor::class => ConfigAbstractFactory::class,
+
+            Crawling\CrawlingHelper::class => ConfigAbstractFactory::class,
         ],
 
         'aliases' => [
@@ -129,6 +132,7 @@ return [
             ShortUrl\Helper\ShortUrlStringifier::class,
             'Logger_Shlink',
         ],
+        Action\RobotsAction::class => [Crawling\CrawlingHelper::class],
 
         ShortUrl\Resolver\PersistenceShortUrlRelationResolver::class => ['em'],
         ShortUrl\Helper\ShortUrlStringifier::class => ['config.url_shortener.domain', 'config.router.base_path'],
@@ -146,6 +150,8 @@ return [
             Service\ShortUrl\ShortCodeHelper::class,
             Util\DoctrineBatchHelper::class,
         ],
+
+        Crawling\CrawlingHelper::class => ['em'],
     ],
 
 ];
diff --git a/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.ShortUrl.php b/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.ShortUrl.php
index 751e513c..a9269d36 100644
--- a/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.ShortUrl.php
+++ b/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.ShortUrl.php
@@ -95,4 +95,9 @@ return static function (ClassMetadata $metadata, array $emConfig): void {
             ->columnName('title_was_auto_resolved')
             ->option('default', false)
             ->build();
+
+    $builder->createField('crawlable', Types::BOOLEAN)
+            ->columnName('crawlable')
+            ->option('default', false)
+            ->build();
 };
diff --git a/module/Core/config/routes.config.php b/module/Core/config/routes.config.php
index a95e8e96..c3f4b66a 100644
--- a/module/Core/config/routes.config.php
+++ b/module/Core/config/routes.config.php
@@ -9,6 +9,14 @@ use Shlinkio\Shlink\Core\Action;
 return [
 
     'routes' => [
+        [
+            'name' => Action\RobotsAction::class,
+            'path' => '/robots.txt',
+            'middleware' => [
+                Action\RobotsAction::class,
+            ],
+            'allowed_methods' => [RequestMethod::METHOD_GET],
+        ],
         [
             'name' => Action\RedirectAction::class,
             'path' => '/{shortCode}',
diff --git a/module/Core/src/Action/RobotsAction.php b/module/Core/src/Action/RobotsAction.php
new file mode 100644
index 00000000..31539b92
--- /dev/null
+++ b/module/Core/src/Action/RobotsAction.php
@@ -0,0 +1,49 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\Action;
+
+use Fig\Http\Message\StatusCodeInterface;
+use GuzzleHttp\Psr7\Response;
+use Psr\Http\Message\ResponseInterface;
+use Psr\Http\Message\ServerRequestInterface;
+use Psr\Http\Server\RequestHandlerInterface;
+use Shlinkio\Shlink\Core\Crawling\CrawlingHelperInterface;
+
+use function sprintf;
+
+use const PHP_EOL;
+
+class RobotsAction implements RequestHandlerInterface, StatusCodeInterface
+{
+    private CrawlingHelperInterface $crawlingHelper;
+
+    public function __construct(CrawlingHelperInterface $crawlingHelper)
+    {
+        $this->crawlingHelper = $crawlingHelper;
+    }
+
+    public function handle(ServerRequestInterface $request): ResponseInterface
+    {
+        return new Response(self::STATUS_OK, ['Content-type' => 'text/plain'], $this->buildRobots());
+    }
+
+    private function buildRobots(): iterable
+    {
+        yield <<<ROBOTS
+        # For more information about the robots.txt standard, see:
+        # https://www.robotstxt.org/orig.html
+
+        User-agent: *
+
+        ROBOTS;
+
+        $shortCodes = $this->crawlingHelper->listCrawlableShortCodes();
+        foreach ($shortCodes as $shortCode) {
+            yield sprintf('Allow: /%s%s', $shortCode, PHP_EOL);
+        }
+
+        yield 'Disallow: /';
+    }
+}
diff --git a/module/Core/src/Crawling/CrawlingHelper.php b/module/Core/src/Crawling/CrawlingHelper.php
new file mode 100644
index 00000000..5f688645
--- /dev/null
+++ b/module/Core/src/Crawling/CrawlingHelper.php
@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\Crawling;
+
+use Doctrine\ORM\EntityManagerInterface;
+use Shlinkio\Shlink\Core\Entity\ShortUrl;
+use Shlinkio\Shlink\Core\Repository\ShortUrlRepositoryInterface;
+
+class CrawlingHelper implements CrawlingHelperInterface
+{
+    private EntityManagerInterface $em;
+
+    public function __construct(EntityManagerInterface $em)
+    {
+        $this->em = $em;
+    }
+
+    public function listCrawlableShortCodes(): iterable
+    {
+        /** @var ShortUrlRepositoryInterface $repo */
+        $repo = $this->em->getRepository(ShortUrl::class);
+        yield from $repo->findCrawlableShortCodes();
+    }
+}
diff --git a/module/Core/src/Crawling/CrawlingHelperInterface.php b/module/Core/src/Crawling/CrawlingHelperInterface.php
new file mode 100644
index 00000000..635a4fc9
--- /dev/null
+++ b/module/Core/src/Crawling/CrawlingHelperInterface.php
@@ -0,0 +1,13 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\Crawling;
+
+interface CrawlingHelperInterface
+{
+    /**
+     * @return string[]|iterable
+     */
+    public function listCrawlableShortCodes(): iterable;
+}
diff --git a/module/Core/src/Entity/ShortUrl.php b/module/Core/src/Entity/ShortUrl.php
index 3fe2932b..6f502da3 100644
--- a/module/Core/src/Entity/ShortUrl.php
+++ b/module/Core/src/Entity/ShortUrl.php
@@ -42,6 +42,7 @@ class ShortUrl extends AbstractEntity
     private ?ApiKey $authorApiKey = null;
     private ?string $title = null;
     private bool $titleWasAutoResolved = false;
+    private bool $crawlable = false;
 
     private function __construct()
     {
@@ -78,6 +79,7 @@ class ShortUrl extends AbstractEntity
         $instance->authorApiKey = $meta->getApiKey();
         $instance->title = $meta->getTitle();
         $instance->titleWasAutoResolved = $meta->titleWasAutoResolved();
+        $instance->crawlable = $meta->isCrawlable();
 
         return $instance;
     }
@@ -200,6 +202,11 @@ class ShortUrl extends AbstractEntity
         return $this->title;
     }
 
+    public function crawlable(): bool
+    {
+        return $this->crawlable;
+    }
+
     public function update(
         ShortUrlEdit $shortUrlEdit,
         ?ShortUrlRelationResolverInterface $relationResolver = null
@@ -220,6 +227,9 @@ class ShortUrl extends AbstractEntity
             $relationResolver = $relationResolver ?? new SimpleShortUrlRelationResolver();
             $this->tags = $relationResolver->resolveTags($shortUrlEdit->tags());
         }
+        if ($shortUrlEdit->crawlableWasProvided()) {
+            $this->crawlable = $shortUrlEdit->crawlable();
+        }
         if (
             $this->title === null
             || $shortUrlEdit->titleWasProvided()
diff --git a/module/Core/src/Model/ShortUrlEdit.php b/module/Core/src/Model/ShortUrlEdit.php
index 3327aad4..32c1ca1e 100644
--- a/module/Core/src/Model/ShortUrlEdit.php
+++ b/module/Core/src/Model/ShortUrlEdit.php
@@ -30,6 +30,8 @@ final class ShortUrlEdit implements TitleResolutionModelInterface
     private ?string $title = null;
     private bool $titleWasAutoResolved = false;
     private ?bool $validateUrl = null;
+    private bool $crawlablePropWasProvided = false;
+    private bool $crawlable = false;
 
     private function __construct()
     {
@@ -61,6 +63,7 @@ final class ShortUrlEdit implements TitleResolutionModelInterface
         $this->maxVisitsPropWasProvided = array_key_exists(ShortUrlInputFilter::MAX_VISITS, $data);
         $this->tagsPropWasProvided = array_key_exists(ShortUrlInputFilter::TAGS, $data);
         $this->titlePropWasProvided = array_key_exists(ShortUrlInputFilter::TITLE, $data);
+        $this->crawlablePropWasProvided = array_key_exists(ShortUrlInputFilter::CRAWLABLE, $data);
 
         $this->longUrl = $inputFilter->getValue(ShortUrlInputFilter::LONG_URL);
         $this->validSince = parseDateField($inputFilter->getValue(ShortUrlInputFilter::VALID_SINCE));
@@ -69,6 +72,7 @@ final class ShortUrlEdit implements TitleResolutionModelInterface
         $this->validateUrl = getOptionalBoolFromInputFilter($inputFilter, ShortUrlInputFilter::VALIDATE_URL);
         $this->tags = $inputFilter->getValue(ShortUrlInputFilter::TAGS);
         $this->title = $inputFilter->getValue(ShortUrlInputFilter::TITLE);
+        $this->crawlable = $inputFilter->getValue(ShortUrlInputFilter::CRAWLABLE);
     }
 
     public function longUrl(): ?string
@@ -162,4 +166,14 @@ final class ShortUrlEdit implements TitleResolutionModelInterface
     {
         return $this->validateUrl;
     }
+
+    public function crawlable(): bool
+    {
+        return $this->crawlable;
+    }
+
+    public function crawlableWasProvided(): bool
+    {
+        return $this->crawlablePropWasProvided;
+    }
 }
diff --git a/module/Core/src/Model/ShortUrlMeta.php b/module/Core/src/Model/ShortUrlMeta.php
index df25735c..06e0eee7 100644
--- a/module/Core/src/Model/ShortUrlMeta.php
+++ b/module/Core/src/Model/ShortUrlMeta.php
@@ -31,6 +31,7 @@ final class ShortUrlMeta implements TitleResolutionModelInterface
     private array $tags = [];
     private ?string $title = null;
     private bool $titleWasAutoResolved = false;
+    private bool $crawlable = false;
 
     private function __construct()
     {
@@ -80,6 +81,7 @@ final class ShortUrlMeta implements TitleResolutionModelInterface
         $this->apiKey = $inputFilter->getValue(ShortUrlInputFilter::API_KEY);
         $this->tags = $inputFilter->getValue(ShortUrlInputFilter::TAGS);
         $this->title = $inputFilter->getValue(ShortUrlInputFilter::TITLE);
+        $this->crawlable = $inputFilter->getValue(ShortUrlInputFilter::CRAWLABLE);
     }
 
     public function getLongUrl(): string
@@ -188,4 +190,9 @@ final class ShortUrlMeta implements TitleResolutionModelInterface
 
         return $copy;
     }
+
+    public function isCrawlable(): bool
+    {
+        return $this->crawlable;
+    }
 }
diff --git a/module/Core/src/Repository/ShortUrlRepository.php b/module/Core/src/Repository/ShortUrlRepository.php
index fe3b170c..eacf293b 100644
--- a/module/Core/src/Repository/ShortUrlRepository.php
+++ b/module/Core/src/Repository/ShortUrlRepository.php
@@ -288,4 +288,28 @@ class ShortUrlRepository extends EntitySpecificationRepository implements ShortU
             $qb->andWhere($qb->expr()->isNull('s.domain'));
         }
     }
+
+    public function findCrawlableShortCodes(): iterable
+    {
+        $blockSize = 1000;
+        $qb = $this->getEntityManager()->createQueryBuilder();
+        $qb->select('DISTINCT s.shortCode')
+           ->from(ShortUrl::class, 's')
+           ->where($qb->expr()->eq('s.crawlable', ':crawlable'))
+           ->setParameter('crawlable', true)
+           ->setMaxResults($blockSize);
+
+        $page = 0;
+        do {
+            $qbClone = (clone $qb)->setFirstResult($blockSize * $page);
+            $iterator = $qbClone->getQuery()->toIterable();
+            $resultsFound = false;
+            $page++;
+
+            foreach ($iterator as ['shortCode' => $shortCode]) {
+                $resultsFound = true;
+                yield $shortCode;
+            }
+        } while ($resultsFound);
+    }
 }
diff --git a/module/Core/src/Repository/ShortUrlRepositoryInterface.php b/module/Core/src/Repository/ShortUrlRepositoryInterface.php
index 29485eeb..5d8fa924 100644
--- a/module/Core/src/Repository/ShortUrlRepositoryInterface.php
+++ b/module/Core/src/Repository/ShortUrlRepositoryInterface.php
@@ -41,4 +41,6 @@ interface ShortUrlRepositoryInterface extends ObjectRepository, EntitySpecificat
     public function findOneMatching(ShortUrlMeta $meta): ?ShortUrl;
 
     public function findOneByImportedUrl(ImportedShlinkUrl $url): ?ShortUrl;
+
+    public function findCrawlableShortCodes(): iterable;
 }
diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php
index d9c18977..35d6a535 100644
--- a/module/Core/src/Repository/VisitRepository.php
+++ b/module/Core/src/Repository/VisitRepository.php
@@ -66,11 +66,11 @@ class VisitRepository extends EntitySpecificationRepository implements VisitRepo
 
         do {
             $qb = (clone $originalQueryBuilder)->andWhere($qb->expr()->gt('v.id', $lastId));
-            $iterator = $qb->getQuery()->iterate();
+            $iterator = $qb->getQuery()->toIterable();
             $resultsFound = false;
 
             /** @var Visit $visit */
-            foreach ($iterator as $key => [$visit]) {
+            foreach ($iterator as $key => $visit) {
                 $resultsFound = true;
                 yield $key => $visit;
             }
diff --git a/module/Core/src/ShortUrl/Transformer/ShortUrlDataTransformer.php b/module/Core/src/ShortUrl/Transformer/ShortUrlDataTransformer.php
index 49918867..52b98c36 100644
--- a/module/Core/src/ShortUrl/Transformer/ShortUrlDataTransformer.php
+++ b/module/Core/src/ShortUrl/Transformer/ShortUrlDataTransformer.php
@@ -35,6 +35,7 @@ class ShortUrlDataTransformer implements DataTransformerInterface
             'meta' => $this->buildMeta($shortUrl),
             'domain' => $shortUrl->getDomain(),
             'title' => $shortUrl->title(),
+            'crawlable' => $shortUrl->crawlable(),
         ];
     }
 
diff --git a/module/Core/src/Validation/ShortUrlInputFilter.php b/module/Core/src/Validation/ShortUrlInputFilter.php
index b5d4fa07..c7cdaa43 100644
--- a/module/Core/src/Validation/ShortUrlInputFilter.php
+++ b/module/Core/src/Validation/ShortUrlInputFilter.php
@@ -32,6 +32,7 @@ class ShortUrlInputFilter extends InputFilter
     public const API_KEY = 'apiKey';
     public const TAGS = 'tags';
     public const TITLE = 'title';
+    public const CRAWLABLE = 'crawlable';
 
     private function __construct(array $data, bool $requireLongUrl)
     {
@@ -105,5 +106,7 @@ class ShortUrlInputFilter extends InputFilter
         $this->add($this->createTagsInput(self::TAGS, false));
 
         $this->add($this->createInput(self::TITLE, false));
+
+        $this->add($this->createBooleanInput(self::CRAWLABLE, false));
     }
 }
diff --git a/module/Core/test-db/Repository/ShortUrlRepositoryTest.php b/module/Core/test-db/Repository/ShortUrlRepositoryTest.php
index cf082d85..bd5b22d4 100644
--- a/module/Core/test-db/Repository/ShortUrlRepositoryTest.php
+++ b/module/Core/test-db/Repository/ShortUrlRepositoryTest.php
@@ -436,4 +436,37 @@ class ShortUrlRepositoryTest extends DatabaseTestCase
         self::assertNull($this->repo->findOneByImportedUrl($buildImported('my-cool-slug', 'doma.in')));
         self::assertNull($this->repo->findOneByImportedUrl($buildImported('another-slug')));
     }
+
+    /** @test */
+    public function findCrawlableShortCodesReturnsExpectedResult(): void
+    {
+        $createShortUrl = fn (bool $crawlable) => ShortUrl::fromMeta(
+            ShortUrlMeta::fromRawData(['crawlable' => $crawlable, 'longUrl' => 'foo.com']),
+        );
+
+        $shortUrl1 = $createShortUrl(true);
+        $this->getEntityManager()->persist($shortUrl1);
+        $shortUrl2 = $createShortUrl(false);
+        $this->getEntityManager()->persist($shortUrl2);
+        $shortUrl3 = $createShortUrl(true);
+        $this->getEntityManager()->persist($shortUrl3);
+        $shortUrl4 = $createShortUrl(true);
+        $this->getEntityManager()->persist($shortUrl4);
+        $shortUrl5 = $createShortUrl(false);
+        $this->getEntityManager()->persist($shortUrl5);
+        $this->getEntityManager()->flush();
+
+        $iterable = $this->repo->findCrawlableShortCodes();
+        $results = [];
+        foreach ($iterable as $shortCode) {
+            $results[] = $shortCode;
+        }
+
+        self::assertCount(3, $results);
+        self::assertContains($shortUrl1->getShortCode(), $results);
+        self::assertContains($shortUrl3->getShortCode(), $results);
+        self::assertContains($shortUrl4->getShortCode(), $results);
+        self::assertNotContains($shortUrl2->getShortCode(), $results);
+        self::assertNotContains($shortUrl5->getShortCode(), $results);
+    }
 }
diff --git a/module/Core/test/Action/RobotsActionTest.php b/module/Core/test/Action/RobotsActionTest.php
new file mode 100644
index 00000000..ad8a02d1
--- /dev/null
+++ b/module/Core/test/Action/RobotsActionTest.php
@@ -0,0 +1,75 @@
+<?php
+
+declare(strict_types=1);
+
+namespace ShlinkioTest\Shlink\Core\Action;
+
+use Laminas\Diactoros\ServerRequestFactory;
+use PHPUnit\Framework\TestCase;
+use Prophecy\PhpUnit\ProphecyTrait;
+use Prophecy\Prophecy\ObjectProphecy;
+use Shlinkio\Shlink\Core\Action\RobotsAction;
+use Shlinkio\Shlink\Core\Crawling\CrawlingHelperInterface;
+
+class RobotsActionTest extends TestCase
+{
+    use ProphecyTrait;
+
+    private RobotsAction $action;
+    private ObjectProphecy $helper;
+
+    protected function setUp(): void
+    {
+        $this->helper = $this->prophesize(CrawlingHelperInterface::class);
+        $this->action = new RobotsAction($this->helper->reveal());
+    }
+
+    /**
+     * @test
+     * @dataProvider provideShortCodes
+     */
+    public function buildsRobotsLinesFromCrawlableShortCodes(array $shortCodes, string $expected): void
+    {
+        $getShortCodes = $this->helper->listCrawlableShortCodes()->willReturn($shortCodes);
+
+        $response = $this->action->handle(ServerRequestFactory::fromGlobals());
+
+        self::assertEquals(200, $response->getStatusCode());
+        self::assertEquals($expected, $response->getBody()->__toString());
+        self::assertEquals('text/plain', $response->getHeaderLine('Content-Type'));
+        $getShortCodes->shouldHaveBeenCalledOnce();
+    }
+
+    public function provideShortCodes(): iterable
+    {
+        yield 'three short codes' => [['foo', 'bar', 'baz'], <<<ROBOTS
+        # For more information about the robots.txt standard, see:
+        # https://www.robotstxt.org/orig.html
+
+        User-agent: *
+        Allow: /foo
+        Allow: /bar
+        Allow: /baz
+        Disallow: /
+        ROBOTS];
+        yield 'five short codes' => [['foo', 'bar', 'some', 'thing', 'baz'], <<<ROBOTS
+        # For more information about the robots.txt standard, see:
+        # https://www.robotstxt.org/orig.html
+
+        User-agent: *
+        Allow: /foo
+        Allow: /bar
+        Allow: /some
+        Allow: /thing
+        Allow: /baz
+        Disallow: /
+        ROBOTS];
+        yield 'no short codes' => [[], <<<ROBOTS
+        # For more information about the robots.txt standard, see:
+        # https://www.robotstxt.org/orig.html
+
+        User-agent: *
+        Disallow: /
+        ROBOTS];
+    }
+}
diff --git a/module/Core/test/Crawling/CrawlingHelperTest.php b/module/Core/test/Crawling/CrawlingHelperTest.php
new file mode 100644
index 00000000..2c65ebac
--- /dev/null
+++ b/module/Core/test/Crawling/CrawlingHelperTest.php
@@ -0,0 +1,43 @@
+<?php
+
+declare(strict_types=1);
+
+namespace ShlinkioTest\Shlink\Core\Crawling;
+
+use Doctrine\ORM\EntityManagerInterface;
+use PHPUnit\Framework\TestCase;
+use Prophecy\PhpUnit\ProphecyTrait;
+use Prophecy\Prophecy\ObjectProphecy;
+use Shlinkio\Shlink\Core\Crawling\CrawlingHelper;
+use Shlinkio\Shlink\Core\Entity\ShortUrl;
+use Shlinkio\Shlink\Core\Repository\ShortUrlRepositoryInterface;
+
+class CrawlingHelperTest extends TestCase
+{
+    use ProphecyTrait;
+
+    private CrawlingHelper $helper;
+    private ObjectProphecy $em;
+
+    protected function setUp(): void
+    {
+        $this->em = $this->prophesize(EntityManagerInterface::class);
+        $this->helper = new CrawlingHelper($this->em->reveal());
+    }
+
+    /** @test */
+    public function listCrawlableShortCodesDelegatesIntoRepository(): void
+    {
+        $repo = $this->prophesize(ShortUrlRepositoryInterface::class);
+        $findCrawlableShortCodes = $repo->findCrawlableShortCodes()->willReturn([]);
+        $getRepo = $this->em->getRepository(ShortUrl::class)->willReturn($repo->reveal());
+
+        $result = $this->helper->listCrawlableShortCodes();
+        foreach ($result as $shortCode) {
+            // Result is a generator and therefore, it needs to be iterated
+        }
+
+        $findCrawlableShortCodes->shouldHaveBeenCalledOnce();
+        $getRepo->shouldHaveBeenCalledOnce();
+    }
+}
diff --git a/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php b/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php
index b4361ca5..1d460623 100644
--- a/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php
+++ b/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php
@@ -59,6 +59,7 @@ class MercureUpdatesGeneratorTest extends TestCase
                 ],
                 'domain' => null,
                 'title' => $title,
+                'crawlable' => false,
             ],
             'visit' => [
                 'referer' => '',
diff --git a/module/Rest/test-api/Action/ListShortUrlsTest.php b/module/Rest/test-api/Action/ListShortUrlsTest.php
index f81524ae..95d77dc6 100644
--- a/module/Rest/test-api/Action/ListShortUrlsTest.php
+++ b/module/Rest/test-api/Action/ListShortUrlsTest.php
@@ -26,6 +26,7 @@ class ListShortUrlsTest extends ApiTestCase
         ],
         'domain' => null,
         'title' => 'My cool title',
+        'crawlable' => true,
     ];
     private const SHORT_URL_DOCS = [
         'shortCode' => 'ghi789',
@@ -41,6 +42,7 @@ class ListShortUrlsTest extends ApiTestCase
         ],
         'domain' => null,
         'title' => null,
+        'crawlable' => false,
     ];
     private const SHORT_URL_CUSTOM_SLUG_AND_DOMAIN = [
         'shortCode' => 'custom-with-domain',
@@ -56,6 +58,7 @@ class ListShortUrlsTest extends ApiTestCase
         ],
         'domain' => 'some-domain.com',
         'title' => null,
+        'crawlable' => false,
     ];
     private const SHORT_URL_META = [
         'shortCode' => 'def456',
@@ -73,6 +76,7 @@ class ListShortUrlsTest extends ApiTestCase
         ],
         'domain' => null,
         'title' => null,
+        'crawlable' => false,
     ];
     private const SHORT_URL_CUSTOM_SLUG = [
         'shortCode' => 'custom',
@@ -88,6 +92,7 @@ class ListShortUrlsTest extends ApiTestCase
         ],
         'domain' => null,
         'title' => null,
+        'crawlable' => false,
     ];
     private const SHORT_URL_CUSTOM_DOMAIN = [
         'shortCode' => 'ghi789',
@@ -105,6 +110,7 @@ class ListShortUrlsTest extends ApiTestCase
         ],
         'domain' => 'example.com',
         'title' => null,
+        'crawlable' => false,
     ];
 
     /**
diff --git a/module/Rest/test-api/Fixtures/ShortUrlsFixture.php b/module/Rest/test-api/Fixtures/ShortUrlsFixture.php
index bfc65aa0..ccc83525 100644
--- a/module/Rest/test-api/Fixtures/ShortUrlsFixture.php
+++ b/module/Rest/test-api/Fixtures/ShortUrlsFixture.php
@@ -35,6 +35,7 @@ class ShortUrlsFixture extends AbstractFixture implements DependentFixtureInterf
                 'longUrl' => 'https://shlink.io',
                 'tags' => ['foo'],
                 'title' => 'My cool title',
+                'crawlable' => true,
             ]), $relationResolver),
             '2018-05-01',
         );
diff --git a/phpstan.neon b/phpstan.neon
index fbddc81c..80f1b083 100644
--- a/phpstan.neon
+++ b/phpstan.neon
@@ -3,4 +3,3 @@ parameters:
 	checkGenericClassInNonGenericObjectType: false
 	ignoreErrors:
 		- '#If condition is always false#'
-		- '#setOrderBy\(\) expects array\<int, string\>, array\<string, string\> given#'
diff --git a/public/robots.txt b/public/robots.txt
deleted file mode 100644
index dfbc662a..00000000
--- a/public/robots.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-# For more information about the robots.txt standard, see:
-# http://www.robotstxt.org/orig.html
-
-User-agent: *
-Disallow: /