{"category":"web","endpoint":"https://setget.net/v1/tools/robots-test","examples":[{"input":{"paths":["/example/page.html","/example/allowed.gif","/","/publications/"],"robots":"User-agent: *\nDisallow: *.gif$\nDisallow: /example/\nAllow: /publications/\n\nUser-agent: foobot\nDisallow: /\nAllow: /example/page.html\nAllow: /example/allowed.gif\n\nSitemap: https://example.com/sitemap.xml","user_agent":"foobot"},"output":{"crawl_delay":null,"matched_group":"foobot","results":[{"allowed":true,"line":8,"path":"/example/page.html","rule":"Allow: /example/page.html"},{"allowed":true,"line":9,"path":"/example/allowed.gif","rule":"Allow: /example/allowed.gif"},{"allowed":false,"line":7,"path":"/","rule":"Disallow: /"},{"allowed":false,"line":7,"path":"/publications/","rule":"Disallow: /"}],"sitemaps":["https://example.com/sitemap.xml"],"user_agent":"foobot"},"title":"The example from RFC 9309, as foobot"},{"input":{"paths":["/example/page.html","/images/logo.gif","/images/logo.gif?v=2","/about"],"robots":"User-agent: *\nDisallow: *.gif$\nDisallow: /example/\nAllow: /publications/","user_agent":"Mozilla/5.0 (compatible; OtherBot/1.0)"},"output":{"crawl_delay":null,"matched_group":"*","results":[{"allowed":false,"line":3,"path":"/example/page.html","rule":"Disallow: /example/"},{"allowed":false,"line":2,"path":"/images/logo.gif","rule":"Disallow: *.gif$"},{"allowed":true,"line":null,"path":"/images/logo.gif?v=2","rule":null},{"allowed":true,"line":null,"path":"/about","rule":null}],"sitemaps":[],"user_agent":"Mozilla/5.0 (compatible; OtherBot/1.0)"},"title":"Wildcards for any other crawler"}],"input_schema":{"additionalProperties":false,"properties":{"paths":{"description":"URL paths (or full URLs) to test.","examples":[["/private/a","/private/press/x"]],"maxItems":500,"type":"array"},"robots":{"description":"The contents of the robots.txt file.","examples":["User-agent: *\nDisallow: /private/\nAllow: /private/press/"],"type":"string"},"user_agent":{"description":"The crawler to test as: a product name such as Googlebot, or a full User-Agent string.","examples":["Googlebot"],"maxLength":1000,"type":"string"}},"required":["robots","user_agent","paths"],"type":"object"},"keywords":["robots.txt","robots tester","disallow","crawler","googlebot","gptbot","block ai crawlers","seo","crawl-delay","sitemap"],"related":["sitemap-validate","user-agent-parse","glob-test","gitignore-test"],"slug":"robots-test","summary":"Check which URL paths a robots.txt file allows for a crawler, and which rule decided, following RFC 9309.","tier":"inline","title":"robots.txt tester","url":"https://setget.net/tools/robots-test"}
