# robots.txt — AutoYard / autoyard.lt
# Automated bulk collection of private HTML is prohibited without written permission.
# Sanctioned AI/API agents: use the structured surface at https://staging.autoyard.lt/api/v1/ and https://staging.autoyard.lt/llms.txt

# --- Search engines: public HTML crawl, sensitive paths blocked ---
User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /accounts/
Disallow: /payments/
Disallow: /subscriptions/
Disallow: /messaging/
Disallow: /chat/
Disallow: /dashboard/
Disallow: /security/
Disallow: /monitoring/
Disallow: /deployments/
Crawl-delay: 5

User-agent: Bingbot
Allow: /
Disallow: /admin/
Disallow: /accounts/
Disallow: /payments/
Disallow: /subscriptions/
Disallow: /messaging/
Disallow: /chat/
Disallow: /dashboard/
Disallow: /security/
Disallow: /monitoring/
Disallow: /deployments/
Crawl-delay: 10

User-agent: DuckDuckBot
Allow: /
Disallow: /admin/
Disallow: /accounts/
Disallow: /payments/
Disallow: /subscriptions/
Disallow: /messaging/
Disallow: /chat/
Disallow: /dashboard/
Disallow: /security/
Disallow: /monitoring/
Disallow: /deployments/
Crawl-delay: 10

User-agent: Yandex
Allow: /
Disallow: /admin/
Disallow: /accounts/
Disallow: /payments/
Disallow: /subscriptions/
Disallow: /messaging/
Disallow: /chat/
Disallow: /dashboard/
Disallow: /security/
Disallow: /monitoring/
Disallow: /deployments/
Crawl-delay: 15

# --- Sanctioned AI / agent crawlers: structured surface only ---
User-agent: GPTBot
Allow: /api/v1/
Allow: /api/v1/docs/
Allow: /api/v1/schema/
Allow: /api/v1/agent-docs/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/security.txt
Disallow: /admin/
Disallow: /accounts/
Disallow: /payments/
Disallow: /subscriptions/
Disallow: /messaging/
Disallow: /chat/
Disallow: /dashboard/
Disallow: /security/
Disallow: /monitoring/
Disallow: /deployments/
Disallow: /

User-agent: OAI-SearchBot
Allow: /api/v1/
Allow: /api/v1/docs/
Allow: /api/v1/schema/
Allow: /api/v1/agent-docs/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/security.txt
Disallow: /admin/
Disallow: /accounts/
Disallow: /payments/
Disallow: /subscriptions/
Disallow: /messaging/
Disallow: /chat/
Disallow: /dashboard/
Disallow: /security/
Disallow: /monitoring/
Disallow: /deployments/
Disallow: /

User-agent: ChatGPT-User
Allow: /api/v1/
Allow: /api/v1/docs/
Allow: /api/v1/schema/
Allow: /api/v1/agent-docs/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/security.txt
Disallow: /admin/
Disallow: /accounts/
Disallow: /payments/
Disallow: /subscriptions/
Disallow: /messaging/
Disallow: /chat/
Disallow: /dashboard/
Disallow: /security/
Disallow: /monitoring/
Disallow: /deployments/
Disallow: /

User-agent: ClaudeBot
Allow: /api/v1/
Allow: /api/v1/docs/
Allow: /api/v1/schema/
Allow: /api/v1/agent-docs/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/security.txt
Disallow: /admin/
Disallow: /accounts/
Disallow: /payments/
Disallow: /subscriptions/
Disallow: /messaging/
Disallow: /chat/
Disallow: /dashboard/
Disallow: /security/
Disallow: /monitoring/
Disallow: /deployments/
Disallow: /

User-agent: anthropic-ai
Allow: /api/v1/
Allow: /api/v1/docs/
Allow: /api/v1/schema/
Allow: /api/v1/agent-docs/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/security.txt
Disallow: /admin/
Disallow: /accounts/
Disallow: /payments/
Disallow: /subscriptions/
Disallow: /messaging/
Disallow: /chat/
Disallow: /dashboard/
Disallow: /security/
Disallow: /monitoring/
Disallow: /deployments/
Disallow: /

User-agent: Claude-Web
Allow: /api/v1/
Allow: /api/v1/docs/
Allow: /api/v1/schema/
Allow: /api/v1/agent-docs/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/security.txt
Disallow: /admin/
Disallow: /accounts/
Disallow: /payments/
Disallow: /subscriptions/
Disallow: /messaging/
Disallow: /chat/
Disallow: /dashboard/
Disallow: /security/
Disallow: /monitoring/
Disallow: /deployments/
Disallow: /

User-agent: Google-Extended
Allow: /api/v1/
Allow: /api/v1/docs/
Allow: /api/v1/schema/
Allow: /api/v1/agent-docs/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/security.txt
Disallow: /admin/
Disallow: /accounts/
Disallow: /payments/
Disallow: /subscriptions/
Disallow: /messaging/
Disallow: /chat/
Disallow: /dashboard/
Disallow: /security/
Disallow: /monitoring/
Disallow: /deployments/
Disallow: /

User-agent: PerplexityBot
Allow: /api/v1/
Allow: /api/v1/docs/
Allow: /api/v1/schema/
Allow: /api/v1/agent-docs/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/security.txt
Disallow: /admin/
Disallow: /accounts/
Disallow: /payments/
Disallow: /subscriptions/
Disallow: /messaging/
Disallow: /chat/
Disallow: /dashboard/
Disallow: /security/
Disallow: /monitoring/
Disallow: /deployments/
Disallow: /

User-agent: Applebot-Extended
Allow: /api/v1/
Allow: /api/v1/docs/
Allow: /api/v1/schema/
Allow: /api/v1/agent-docs/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/security.txt
Disallow: /admin/
Disallow: /accounts/
Disallow: /payments/
Disallow: /subscriptions/
Disallow: /messaging/
Disallow: /chat/
Disallow: /dashboard/
Disallow: /security/
Disallow: /monitoring/
Disallow: /deployments/
Disallow: /

User-agent: CCBot
Allow: /api/v1/
Allow: /api/v1/docs/
Allow: /api/v1/schema/
Allow: /api/v1/agent-docs/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /.well-known/security.txt
Disallow: /admin/
Disallow: /accounts/
Disallow: /payments/
Disallow: /subscriptions/
Disallow: /messaging/
Disallow: /chat/
Disallow: /dashboard/
Disallow: /security/
Disallow: /monitoring/
Disallow: /deployments/
Disallow: /

# --- Known hostile scrapers: fully disallowed ---
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

# --- Default ---
User-agent: *
Disallow: /api/
Disallow: /admin/
Disallow: /accounts/
Disallow: /payments/
Disallow: /subscriptions/
Disallow: /messaging/
Disallow: /chat/
Disallow: /dashboard/
Disallow: /security/
Disallow: /monitoring/
Disallow: /deployments/
Allow: /llms.txt
Allow: /.well-known/security.txt
Allow: /
Crawl-delay: 30

# Honeypot paths — any request here is guaranteed to be a scraper
Disallow: /admin-portal/
Disallow: /internal-api/
Disallow: /export/listings/
Disallow: /bulk-data/

Sitemap: https://staging.autoyard.lt/lt/about/sitemap.xml