User-agent: *
Disallow: /webapp/
Disallow: /search_result.asp*
Disallow: /cdn-cgi/

User-agent: YandexBot
Crawl-delay: 4
User-agent: Baiduspider
Crawl-delay: 4

#block user agents - killing site performance#
User-agent: NerdyBot
User-agent: 008
User-agent: 80legs
User-agent: AddThis.com
User-agent: AdxPsfFetcher-Google
User-agent: AhrefsBot
User-agent: AhrefsBot/5.0
User-agent: aiHitBot
User-agent: BLEXBot
User-agent: cb
User-agent: CCBot
User-agent: check_http
User-agent: Cloudinary
User-agent: coccoc
User-agent: ConveraCrawler
User-agent: crawler@wishpond.com
User-agent: CRAZYWEBCRAWLER+0.9.8
User-agent: CrazyWebCrawler-Spider
User-agent: curl
User-agent: Domain+Re-Animator+Bot
User-agent: DomainSigmaCrawler
User-agent: Dotbot
User-agent: EtaoSpider
User-agent: Exabot
User-agent: Ezooms
User-agent: FatBot
User-agent: GigablastOpenSource
User-agent: GrapeshotCrawler
User-agent: GSiteCrawler
User-agent: GumGum Bot
User-agent: GWPImages
User-agent: Halebot
User-agent: ia_archiver
User-agent: ICC-Crawler
User-agent: InternetSeer.com
User-agent: Konqueror
User-agent: kw-lp-suggest
User-agent: kw-lp-suggest Count
User-agent: Linkdex
User-agent: Lipperhey Spider
User-agent: MaxPointCrawler
User-agent: MJ12bot
User-agent: NewRelicPinger
User-agent: Nutch
User-agent: OrangeBot
User-agent: Panopta
User-agent: Panopta+v1.1
User-agent: Riddler
User-agent: Rogerbot
User-agent: ruby
User-agent: ScoutJet
User-agent: scrapy
User-agent: scrapybot
User-agent: ShowyouBot
User-agent: SMTBot
User-agent: spbot
User-agent: SpyFu
User-agent: uber
User-agent: uber spider
User-agent: uptime files - website monitoring service
User-agent: voltron
User-agent: wishpond
User-agent: Xenu
User-agent: Xenu Link Sleuth
User-agent: nagios
Disallow: /



sitemap: https://www.bigparts.com/sitemap.xml