'text/plain'], $this->buildRobots()); } private function buildRobots(): iterable { yield <<robotsOptions->hasUserAgents() ? $this->robotsOptions->userAgents : ['*']; foreach ($userAgents as $userAgent) { yield sprintf('User-agent: %s%s', $userAgent, PHP_EOL); } if ($this->robotsOptions->allowAllShortUrls) { // Disallow rest URLs, but allow all short codes yield 'Disallow: /rest/'; return; } $shortCodes = $this->crawlingHelper->listCrawlableShortCodes(); foreach ($shortCodes as $shortCode) { yield sprintf('Allow: /%s%s', $shortCode, PHP_EOL); } yield 'Disallow: /'; } }