Initial commit of akmon project

doc_seo/templates/ai.txt

@@ -0,0 +1,18 @@
+# ai.txt (informal, emerging convention)
+# Place at https://example.com/ai.txt
+# Declare your AI crawling/usage policies.
+
+Contact: ai-policy@example.com
+
+# Allowed use cases
+Allow: index, summarize, link-preview
+
+# Disallowed use cases
+Disallow: model-training, derivative-commercialization
+
+# Preferred attribution
+Attribution: link-required
+
+# Rate and caching guidance (non-binding)
+Rate-Limit: 1 rps
+Cache-TTL: 6h

doc_seo/templates/robots.txt

@@ -0,0 +1,39 @@
+# robots.txt (template)
+# Adjust Allow/Disallow and Crawl-delay per your policy.
+
+User-agent: *
+Allow: /
+# Disallow sensitive or duplicate areas
+Disallow: /admin/
+Disallow: /api/
+Disallow: /cart/
+
+# Sitemaps
+Sitemap: https://example.com/sitemap_index.xml
+
+# Optional crawl pacing (not all bots honor this)
+# Crawl-delay: 2
+
+# AI/LLM bots (adjust per policy)
+User-agent: GPTBot
+Allow: /
+
+User-agent: CCBot
+Allow: /
+
+User-agent: ClaudeBot
+Allow: /
+
+User-agent: PerplexityBot
+Allow: /
+
+User-agent: Google-Extended
+Allow: /
+
+User-agent: Applebot
+Allow: /
+
+User-agent: Bytespider
+Allow: /
+
+# If you want to restrict some bots, switch Allow to Disallow or remove their block.