From 70ea3e8cc00fc59b5dfacf0908323123ee426806 Mon Sep 17 00:00:00 2001
From: tukwan <7630720+tukwan@users.noreply.github.com>
Date: Tue, 26 Nov 2024 11:12:43 +0400
Subject: [PATCH] chore: add robots.txt

---
 static/robots.txt | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 static/robots.txt

diff --git a/static/robots.txt b/static/robots.txt
new file mode 100644
index 00000000..eec09d82
--- /dev/null
+++ b/static/robots.txt
@@ -0,0 +1,3 @@
+# Allow everything to be crawled except Cloudflare's /cdn-cgi/
+User-agent: *
+Disallow: /cdn-cgi/