-
Notifications
You must be signed in to change notification settings - Fork 0
/
robots.txt
38 lines (28 loc) · 866 Bytes
/
robots.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
# source: https://neil-clarke.com/block-the-bots-that-feed-ai-models-by-scraping-your-website/
# Data from Common Crawl is used to train ChatGPT, Bard, etc.
User-agent: CCBot
Disallow: /
# Stops ChatGPT users from instructing ChatGPT to access our site
User-agent: ChatGPT-User
Disallow: /
# Don't add any content to the GPT model
User-agent: GPTBot
Disallow: /
# Blocks Bard and VertexAI. Does not impact search indexing.
User-agent: Google-Extended
Disallow: /
# webz.io. They sell data for training LLMs
User-agent: Omgilibot
Disallow: /
User-agent: Omgili
Disallow: /
# Specific to AI. Won't prevent previews from showing up correctly on Facebook posts
User-agent: FacebookBot
Disallow: /
User-agent: anthropic-ai
Disallow: /
# ByteDance's bot for gathering LLM training data
User-agent: Bytespider
Disallow: /
User-agent: ImagesiftBot
Disallow: /