User-agent: *
Allow: /wp-admin/admin-ajax.php
Allow: /*/*.css
Allow: /*/*.js
Allow: /ads.txt
Allow: /app-ads.txt
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /readme.html
Disallow: /license.txt
Disallow: /xmlrpc.php
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /wp-signup.php
Disallow: */disclaimer/*
Disallow: *?attachment_id=
Disallow: /privacy-policy

# Khusus Googlebot
User-agent: Googlebot
Allow: /

# Googlebot-Image (untuk mengindeks gambar)
User-agent: Googlebot-Image
Allow: /wp-content/uploads/

# Mediapartners-Google (untuk AdSense)
User-agent: Mediapartners-Google
Allow: /

# Google Ads bot
User-agent: AdsBot-Google
Allow: /
User-agent: AdsBot-Google-Mobile
Allow: /

# Bingbot & MSN
User-agent: Bingbot
Allow: /
User-agent: Msnbot
Allow: /
User-agent: msnbot-media
Allow: /wp-content/uploads/

# Applebot (bot pencarian Apple)
User-agent: Applebot
Allow: /

# Yandex bot
User-agent: Yandex
Allow: /
User-agent: YandexImages
Allow: /wp-content/uploads/

# Bot Yahoo (Slurp)
User-agent: Slurp
Allow: /

# Bot DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Bot Qwant
User-agent: Qwantify
Allow: /

# Blokir Bot Buruk & Web Scrapers
User-agent: ia_archiver
Disallow: /
User-agent: archive.org_bot
Disallow: /
User-agent: SiteExplorer
Disallow: /
User-agent: spbot
Disallow: /
User-agent: WBSearchBot
Disallow: /
User-agent: linkdexbot
Disallow: /
User-agent: Screaming Frog SEO Spider
Disallow: /
User-agent: netEstate NE Crawler
Disallow: /
User-agent: Moreover
Disallow: /
User-agent: sentibot
Disallow: /
User-agent: Aboundexbot
Disallow: /
User-agent: proximic
Disallow: /
User-agent: oBot
Disallow: /
User-agent: meanpathbot
Disallow: /
User-agent: Nutch
Disallow: /
User-agent: TurnitinBot
Disallow: /
User-agent: ZoominfoBot
Disallow: /
User-agent: ZmEu
Disallow: /
User-agent: grapeshot
Disallow: /
User-agent: python-requests
Disallow: /
User-agent: Go-http-client
Disallow: /
User-agent: Apache-HttpClient
Disallow: /
User-agent: libwww-perl
Disallow: /
User-agent: curl
Disallow: /
User-agent: wget
Disallow: /
User-agent: MJ12bot
Disallow: /
User-agent: SemrushBot
Disallow: /
User-agent: BLEXBot
Disallow: /
User-agent: AhrefsBot
Disallow: /

# Blokir ChatGPT dan AI Scraper Bots
User-agent: GPTBot
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /

# Sitemap untuk membantu indexing
Sitemap: https://diskursusnetwork.com/sitemap.xml