# 7thavenuerealty.com — robots policy
# Automated scraping, harvesting, or bulk downloading of site content —
# including listing data and images — is prohibited. See /terms.html.
# Search engines and AI search / answer services are welcome to read and cite
# our public pages (see /terms.html).

User-agent: *
Allow: /
Disallow: /docs/
Disallow: /scripts/
# The pages' data files (/idx/data/, /new-builds/*.json, /rohit-homes/*.json, /communities/data/, /listing/pages.json) are
# NOT disallowed here (Oct 25, 2026): the MLS® search, New Builds, the calculators and the community guides draw their
# content from them, and a search engine's renderer may only fetch what robots.txt allows — blocked, Google's copy of
# those pages showed no homes. They stay out of the index by an X-Robots-Tag: noindex header (netlify.toml) instead.

# AI model-training crawlers: welcome to learn from our guides, articles and tools —
# but not from MLS® listing pages or data (licensed for consumers buying or selling only).
# (AI search and answer agents — OAI-SearchBot, ChatGPT-User, Claude-SearchBot, Claude-User,
# PerplexityBot, Perplexity-User — follow the * rules above, like Google and Bing.)
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: meta-externalagent
Allow: /
Disallow: /listing/
Disallow: /idx/
# the MLS® listing photos (the same-domain photo proxy), like the listing pages, are not for model training
Disallow: /idx-img/
Disallow: /docs/
Disallow: /scripts/
Disallow: /idx/data/
Disallow: /rohit-homes/*.json
Disallow: /listing/pages.json
Disallow: /new-builds/*.json
Disallow: /communities/data/

# Aggressive scrapers that ignore crawl manners
User-agent: Bytespider
Disallow: /

# Bulk-copy / site-mirroring tools are not permitted
User-agent: HTTrack
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: Offline Explorer
Disallow: /

Sitemap: https://www.7thavenuerealty.com/sitemap.xml
Sitemap: https://www.7thavenuerealty.com/rohit-homes/sitemap.xml
Sitemap: https://www.7thavenuerealty.com/listing/sitemap.xml