{"category":"generate","endpoint":"https://setget.net/v1/tools/robots-txt-generate","examples":[{"input":{"block_ai":true,"mode":"allow_all","sitemaps":["https://example.com/sitemap.xml"]},"output":{"groups":2,"text":"User-agent: *\nDisallow:\n\nUser-agent: GPTBot\nUser-agent: ClaudeBot\nUser-agent: CCBot\nUser-agent: Google-Extended\nUser-agent: PerplexityBot\nDisallow: /\n\nSitemap: https://example.com/sitemap.xml\n"}},{"input":{"mode":"custom","rules":[{"allow":["/admin/help"],"disallow":["/admin/","/cart"],"user_agent":"*"},{"crawl_delay":10,"disallow":["/"],"user_agent":"Bingbot"}]},"output":{"groups":2,"text":"User-agent: *\nAllow: /admin/help\nDisallow: /admin/\nDisallow: /cart\n\nUser-agent: Bingbot\nDisallow: /\nCrawl-delay: 10\n"},"title":"Custom rules for two crawlers"}],"input_schema":{"additionalProperties":false,"properties":{"block_ai":{"default":false,"description":"Add a rule that blocks common AI crawlers: GPTBot, ClaudeBot, CCBot, Google-Extended, PerplexityBot.","type":"boolean"},"crawl_delay":{"description":"Seconds crawlers should wait between requests, for the general rule. Google ignores this.","maximum":3600,"minimum":0,"type":"number"},"mode":{"default":"allow_all","description":"allow_all lets crawlers visit everything, block_all shuts them out, custom uses your rules.","enum":["allow_all","block_all","custom"],"examples":["allow_all"],"type":"string"},"rules":{"description":"For custom mode: a list of {\"user_agent\": \"*\", \"disallow\": [\"/admin/\"], \"allow\": [\"/admin/help\"], \"crawl_delay\": 5}.","examples":[[{"disallow":["/admin/","/cart"],"user_agent":"*"}]]},"sitemaps":{"description":"Full addresses of your sitemap files.","examples":[["https://example.com/sitemap.xml"]],"maxItems":50,"type":"array"}},"required":[],"type":"object"},"keywords":["robots.txt generator","robots txt","block ai crawlers","block gptbot","disallow crawler","robots file","seo robots"],"related":["sitemap-generate","htaccess-redirects"],"slug":"robots-txt-generate","summary":"Build a robots.txt: allow or block all crawlers, add your own rules, block AI crawlers and list sitemaps.","tier":"inline","title":"robots.txt generator","url":"https://setget.net/tools/robots-txt-generate"}
