diff --git a/robots.txt b/robots.txt index 0f853392..fc5c7a75 100644 --- a/robots.txt +++ b/robots.txt @@ -1,4 +1,16 @@ User-agent: * +Crawl-delay: 120 Disallow: /user/ Disallow: /mailreporter/ -Crawl-delay: 120 +User-agent: Amazonbot +Disallow: / +User-agent: Bytespider +Disallow: / +User-Agent: PerplexityBot +Disallow: / +User-agent: ClaudeBot +Disallow: / +User-agent: OAI-SearchBot +Disallow: / +User-agent: GPTBot +Disallow: / \ No newline at end of file