# Global allow for Mediapartners - this is used by Google to place ads in content, # not indexing purposes. User-agent: Mediapartners-Google Disallow: # Throttles to decrease load User-agent: Yandex Crawl-delay: 3 User-agent: Baiduspider Crawl-delay: 3 User-agent: bingbot Crawl-delay: 2 User-agent: Slurp Crawl-delay: 2 User-agent: FacebookBot Crawl-delay: 3 # Temporary blocks of uninteresting bots User-Agent: Genieo Disallow: / User-Agent: ecoResearch Disallow: / User-Agent: Spinn3r Disallow: / User-Agent: BLP_bbot Disallow: / User-Agent: R6_CommentReader Disallow: / User-Agent: omgilibot Disallow: / User-Agent: EasouSpider Disallow: / User-Agent: A6-Indexer Disallow: / User-Agent: magpie-crawler Disallow: / User-agent: GPTBot Disallow: / User-agent: anthropic-ai Disallow: / User-agent: ClaudeBot Disallow: / User-agent: Claude-Web Disallow: / User-agent: PerplexityBot Disallow: / User-agent: CCBot Disallow: / User-agent: ChatGPT-User Disallow: / User-agent: omgili Disallow: / # Prevent bots from wasting crawl budget on template/non-content pages User-Agent: * Disallow: /search Disallow: /web Disallow: /mrcontent Disallow: /webapi-public Disallow: /adperfect Disallow: /*amp.html?* Disallow: *.ece* Allow: *.ece*.pdf # Block indexing of specific sections/content at the market's request User-Agent: * Disallow: /advertise/advertorial Disallow: /advertise/advertorial/bridal Disallow: /advertise/circulars Disallow: /advertise/classifieds Disallow: /advertise/sponsored-content Disallow: /advertise/sponsored-content/sponsored-posts Disallow: /advertise/welcome Disallow: /advertise/welcome/welcome-page-lg Disallow: /advertise/welcome/welcome-page-md Disallow: /advertise/welcome/welcome-page-sm Disallow: /advertise/welcome/welcome-page-xs Disallow: /customer-service/24hoursale Disallow: /customer-service/404 Disallow: /customer-service/410 Disallow: /customer-service/contact Disallow: /customer-service/contests-promotions Disallow: /customer-service/digitalsavings Disallow: /customer-service/newsletter-signup Disallow: /customer-service/plus Disallow: /customer-service/plus/trial-offer Disallow: /customer-service/rss Disallow: /customer-service/social-media Disallow: /customer-service/subscription/special-offers Disallow: /customer-service/subscription/special-offers/cyber Disallow: /customer-service/subscription/special-offers/dbm Disallow: /customer-service/subscription/special-offers/holiday Disallow: /customer-service/subscription/special-offers/offer-a Disallow: /customer-service/subscription/special-offers/offer-b Disallow: /customer-service/subscription/special-offers/offer-c Disallow: /customer-service/subscription/special-offers/offer-j Disallow: /customer-service/subscription/special-offers/offer-k Disallow: /customer-service/subscription/special-offers/others Disallow: /customer-service/social-media Disallow: /entertainment/best-of-dfw Disallow: /entertainment/events Disallow: /entertainment/events/things-to-do Disallow: /entertainment/movies Disallow: /entertainment/music Disallow: /entertainment/nightlife Disallow: /entertainment/nightlife/sports-scene Disallow: /entertainment/performing-arts Disallow: /entertainment/restaurants/bbq Disallow: /entertainment/restaurants/brew Disallow: /entertainment/restaurants/burgers Disallow: /entertainment/tv Disallow: /fort-worth-dallas-weddings/contact-us Disallow: /living/books Disallow: /living/family Disallow: /living/family/moms Disallow: /living/family/moms/maris-moments Disallow: /living/food-drink/weekend-chef Disallow: /living/health-fitness Disallow: /living/home-garden/neil-sperry Disallow: /living/indulge Disallow: /living/indulge/indulge-food-drink Disallow: /living/indulge/indulge-getting-around Disallow: /living/indulge/indulge-homes Disallow: /living/indulge/indulge-in-focus Disallow: /living/indulge/indulge-scene Disallow: /living/indulge/indulge-style Disallow: /living/indulge/indulge-the-list Disallow: /living/keller-magazine Disallow: /living/liv-columns-blogs Disallow: /living/liv-columns-blogs/social-eyes Disallow: /living/liv-columns-blogs/social-eyes/social-eyes-photos Disallow: /living/liv-columns-blogs/social-eyes/social-eyes-submissions Disallow: /living/religion Disallow: /news/business/aviation/american-airlines Disallow: /news/business/aviation/sky-talk-blog Disallow: /news/business/barnett-shale Disallow: /news/business/biz-columns-blogs Disallow: /news/business/biz-columns-blogs/baker-ahles-kaskovich Disallow: /news/business/biz-columns-blogs/teresa-mcusic Disallow: /news/business/technology Disallow: /news/local/community Disallow: /news/local/community/arlington-citizen-journal Disallow: /news/local/community/fort-worth/fw-stock-show Disallow: /news/local/community/fort-worth/hog-blog Disallow: /news/local/community/keller-citizen Disallow: /news/local/community/mansfield-news-mirror Disallow: /news/local/community/mansfield-news-mirror/mnm-faith Disallow: /news/local/community/mansfield-news-mirror/mnm-living Disallow: /news/local/community/mansfield-news-mirror/mnm-news Disallow: /news/local/community/mansfield-news-mirror/mnm-opinion Disallow: /news/local/community/mansfield-news-mirror/mnm-sports Disallow: /news/local/community/weatherford-star-telegram Disallow: /news/local/community/weatherford-star-telegram/wt-living Disallow: /news/local/community/weatherford-star-telegram/wt-news Disallow: /news/local/community/weatherford-star-telegram/wt-opinion Disallow: /news/local/community/weatherford-star-telegram/wt-sports Disallow: /news/local/obituaries/current-obituaries Disallow: /news/nation-world/ebola Disallow: /news/politics-government/national-politics Disallow: /news/politics-government/state-politics/politex-blog Disallow: /news/politics-government/texas-senate-race Disallow: /news/special-reports Disallow: /news/special-reports/jfk Disallow: /news/special-reports/van-cliburn Disallow: /news/special-reports/van-cliburn/van-cliburn-blog Disallow: /news/traffic/gas-prices Disallow: /news/traffic/honkin-mad-blog Disallow: /news/traffic/your-commute Disallow: /opinion/letters-to-the-editor/cheers-jeers Disallow: /opinion/opn-columns-blogs/michael-ryan Disallow: /sports/boxing Disallow: /sports/college/baseball Disallow: /sports/college/big-12 Disallow: /sports/college/big-12/baylor-bears Disallow: /sports/college/big-12/oklahoma-sooners Disallow: /sports/college/big-12/texas-tech Disallow: /sports/college/big-12/university-of-texas Disallow: /sports/college/college-confidential-blog Disallow: /sports/college/football Disallow: /sports/college/sec Disallow: /sports/college/sec/texas-a-m Disallow: /sports/college/unt Disallow: /sports/college/uta Disallow: /sports/dfwvarsity/prep-baseball Disallow: /sports/dfwvarsity/prep-softball Disallow: /sports/dfwvarsity/prep-volleyball Disallow: /sports/nfl/houston-texans Disallow: /sports/other-sports Disallow: /sports/other-sports/fort-worth-cats Disallow: /sports/outdoors Disallow: /sports/spt-columns-blogs/clarence-hill Disallow: /sports/spt-columns-blogs/gil-lebreton Disallow: /sports/spt-columns-blogs/jimmy-burch Disallow: /sports/spt-columns-blogs/the-big-mac-blog Disallow: /news/state Disallow: /sports/spt-columns-blogs Disallow: /opinion/opn-columns-blogs Disallow: /homes/foreclosures Disallow: /news/corrections Disallow: /advertise/sponsored Disallow: /site-services/newsletters Disallow: /site-services/testing Disallow: /mobile/m-best-week Disallow: /mobile/m-business Disallow: /mobile/m-life-arts Disallow: /mobile/m-local Disallow: /mobile/m-opinion Disallow: /mobile/m-sports Disallow: /mobile/m-top-stories Disallow: /site-services/newsletters/business-insider Disallow: /site-services/newsletters/mom2mom-dfw-newsletter Disallow: /site-services/newsletters/sports-tip-sheet Disallow: /site-services/promos Disallow: /site-services/promos/ez-pay-contest Disallow: /welcome-page Allow: /entertainment/nightlife/party-pics Allow: /entertainment/tv/listings Allow: /news/local/community/northeast-tarrant Allow: /news/special-reports/databases Allow: /news/state/texas Allow: /sports/college/big-12/texas-christian-university Allow: /sports/nfl/dallas-cowboys Allow: /sports/nhl/dallas-stars Allow: /sports/spt-columns-blogs/mac-engel Allow: /sports/mlb/texas-rangers Allow: /sports/nba/dallas-mavericks Allow: /opinion/opn-columns-blogs/other-voices Allow: /customer-service/subscription/special-offers/digital # Sitemap files Sitemap: https://www.star-telegram.com/sitemap/story/update.xml Sitemap: https://www.star-telegram.com/sitemap/story/archive.xml Sitemap: https://www.star-telegram.com/sitemap/story/googlenews.xml Sitemap: https://www.star-telegram.com/sitemap/blogpost/update.xml Sitemap: https://www.star-telegram.com/sitemap/blogpost/archive.xml Sitemap: https://www.star-telegram.com/sitemap/blogpost/googlenews.xml Sitemap: https://www.star-telegram.com/sitemap/section/update.xml Sitemap: https://www.star-telegram.com/sitemap/section/archive.xml Sitemap: https://www.star-telegram.com/sitemap/topic/archive.xml Sitemap: https://www.star-telegram.com/sitemap/topic/update.xml