'text/plain'], $this->buildRobots()); } private function buildRobots(): iterable { yield <<allowAllShortUrls) { // Disallow rest URLs, but allow all short codes yield 'Disallow: /rest/'; return; } $shortCodes = $this->crawlingHelper->listCrawlableShortCodes(); foreach ($shortCodes as $shortCode) { yield sprintf('Allow: /%s%s', $shortCode, PHP_EOL); } yield 'Disallow: /'; } }