## robots.txt


## block search engine bots and crawlers
#User-agent: Amazonbot                   # Amazon's user agent
#User-agent: Applebot                    # Apple's user agent
#User-agent: Baiduspider
#User-agent: Bingbot
#User-agent: DuckDuckBot
#User-agent: Googlebot
#User-agent: Slurp
#User-agent: Sogou web spider
#User-agent: YandexBot
##
## block seo and marketing bots
User-agent: AhrefsBot                   # ahrefs db & yep search engine
User-agent: AhrefsSiteAudit             # ahrefs' site audit tool
User-agent: barkrowler                  # Babbar's bot
User-agent: BLEXBot                     # seranking
#User-agent: Brightbot 1.0
User-agent: DataForSeoBot
User-agent: DotBot                      # Moz Link Index 
#User-agent: DataForSeoBot
#User-agent: domainsproject.org
#User-agent: keys-so-bot
User-agent: MJ12bot                     # Majestic map of the Internet
User-agent: rogerbot                    # Moz Pro Site Crawl
User-agent: SemrushBot                  # Semrush
User-agent: SEOkicks                    # SEOkicks
#User-agent: SerpstatBot
##
## block ai llm bots and crawlers
User-agent: Applebot-Extended           # Apple LLM Crawler
User-agent:  Bytespider                 # ByteDance LLM crawler
#User-agent: CCBot                       # Common Crawl LLM Crawler
User-agent: ChatGLM-Spider
#User-agent: ChatGPT-User                # ChatGPT user actions
User-agent: ClaudeBot                   # Claude LLM Crawler
#User-agent: Claude-User                 # Claude AI user actions
User-agent: cohere-ai
User-agent: Google-Extended             # Google LLM Crawler
User-agent: GPTBot                      # GPT LLM Crawler
User-agent: meta-externalagent
User-agent: PerplexityBot
##
## block ai search engine bots and crawlers
#User-agent: Claude-SearchBot
#User-agent: OAI-SearchBot
##
# block research and archival bots
#User-agent: ia_archiver
#User-agent: archive.org_bot
##
# block monitoring and uptime bots
#User-agent: UptimeRobot
#User-agent: PingdomBot
#User-agent: StatusCake
#User-agent: NewRelic
##
## block other unwanted crawlers
#User-agent: daumoa
#User-agent: EzoicBot
#User-agent: Mail.RU
##
Disallow: /



## all (other) robots
User-agent: *
##
# disallow all bots on entire server
#Disallow: /
# disallow all bots on the /secrets directory
#Disallow: /secrets/
# allow all bots on entire server
Disallow:
##
# delay in seconds - nonstandard
#Crawl-delay: 5
Crawl-delay: 30



## where to find the sitemap(s) - nonstandard
Sitemap: https://www.cislik.de/sitemap-index.xml
Sitemap: https://www.cislik.de/sitemap.xml