dokuwiki-plugin-botmon/data/known-bots.json

[	{"id": "bingbot",
	 "n": "Bing Bot",
	 "r": ["bingbot"],
	 "rx": ["\\sbingbot\\/(\\d+\\.\\d+);"],
	 "url": "http://www.bing.com/bingbot.htm"
	},
	{"id": "googlebot",
	 "n": "Google Bot",
	 "r": ["Googlebot"],
	 "rx": ["Googlebot\\/(\\d+\\.\\d+)", "Googlebot-Image\\/(\\d+\\.\\d+)"],
	 "url": "http://www.google.com/bot.html"
	},
	{"id": "googleads",
	 "n": "Google Ads Bot",
	 "r": ["AdsBot-Google", "AdsBot-Google-Mobile", "Mediapartners-Google"],
	 "rx": ["AdsBot-Google;","AdsBot-Google-Mobile;", "Mediapartners-Google\\/(\\d+\\.\\d+);"],
	 "url": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers"
	},
	{"id": "googleapi",
	 "n": "Google API Crawler",
	 "r": ["APIs-Google"],
	 "rx": ["APIs-Google"],
	 "url": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers"
	},
	{"id": "applebot",
	 "n": "Applebot",
	 "r": ["Applebot"],
	 "rx": ["Applebot\\/(\\d+\\.\\d+);"],
	 "url": "http://www.apple.com/go/applebot"
	},
	{"id": "slurp",
	 "n": "Slurp (Yahoo!)",
	 "r": ["Slurp"],
	 "rx": ["Slurp[\\s;\\)]"],
	 "url": "http://help.yahoo.com/help/us/ysearch/slurp"
	},
	{"id": "ddg",
	 "n": "DuckDuckGo Bots",
	 "r": ["DuckDuckBot","DuckAssistBot","DuckDuckGo-Favicons-Bot"],
	 "rx": ["DuckDuckBot\\/(\\d+\\.\\d+);", "DuckAssistBot\\/(\\d+\\.\\d+);", "DuckDuckGo-Favicons-Bot\\/(\\d+\\.\\d+);"],
	 "url": "https://duckduckgo.com/duckduckbot.html"
	},
	{"id": "openai",
	 "n": "OpenAI/ChatGPT Bots",
	 "r": ["OAI-SearchBot", "ChatGPT-User", "GPTBot"],
	 "rx": ["OAI-SearchBot\\/(\\d+\\.\\d+);", "ChatGPT-User\\/(\\d+\\.\\d+);", "GPTBot\\/(\\d+\\.\\d+);"],
	 "url": "https://platform.openai.com/docs/bots/"
	},
	{"id": "claude",
	 "n": "Anthropic Claude Bots",
	 "r": ["ClaudeBot", "Claude-User", "Claude-SearchBot"],
	 "rx": ["ClaudeBot\\/(\\d+\\.\\d+);"],
	 "url": "https://darkvisitors.com/agents/claudebot"
	},
	{"id": "perplexity",
	 "n": "Perplexity Crawlers",
	 "r": ["PerplexityBot", "Perplexity‑User"],
	 "rx": ["PerplexityBot\\/(\\d+\\.\\d+);", "Perplexity‑User\\/(\\d+\\.\\d+);"],
	 "url": "https://perplexity.ai/perplexitybot"
	},
	{"id": "metabots",
	 "n": "Meta/Facebook Bots",
	 "r": ["facebookexternalhit", "facebookcatalog","meta-webindexer","meta-externalads","meta-externalagent","meta-externalfetcher"],
	 "rx": ["facebook\\w+\\/(\\d+\\.\\d+)", "meta-\\w+\\/(\\d+\\.\\d+)"],
	 "url": "https://developers.facebook.com/docs/sharing/webmasters/crawler"
	},
	{"id": "qwant",
	 "n": "Qwant Web crawler",
	 "r": ["Qwantbot", "Qwantbot-news"],
	 "rx": ["Qwantbot\\/(\\d+\\.\\d+)[_;\\s]"],
	 "url": "https://help.qwant.com/bot/"
	},
	{"id": "yandex",
	 "n": "Yandex Bots",
	 "r": ["YandexBot", "YandexAdNet", "YandexBlogs", "YandexImages", "YandexImageResizer", "YandexMarket", "YandexMedia", "YandexOntoDB", "YandexSitelinks","YandexSpravBot", "YandexVertis", "YandexVerticals", "YandexVideo", "YandexWebmaster", "YandexComBot"],
	 "rx": ["Yandex\\w+\\/(\\d+\\.\\d+);"],
	 "url": "http://yandex.com/bots"
	},
	{"id": "seznambot",
	 "n": "SeznamBot (CZ)",
	 "r": ["SeznamBot"],
	 "rx": ["SeznamBot\\/(\\d+\\.\\d+);"],
	 "url": "https://o-seznam.cz/napoveda/vyhledavani/en/seznambot-crawler/"
	},
	{"id": "ahrefs",
	 "n": "Ahrefs Bots (SEO/marketing)",
	 "r": ["AhrefsBot", "AhrefsSiteAudit"],
	 "rx": ["AhrefsBot\\/(\\d+\\.\\d+);", "AhrefsSiteAudit\\/(\\d+\\.\\d+);"],
	 "url": "https://ahrefs.com/robot/"
	},
	{"id": "ccbot",
	 "n": "Common Crawl Bot (AI-Scraper)",
	 "r": ["CCBot"],
	 "rx": ["CCBot\\/(\\d+\\.\\d+)[\\s\\.;]*"],
	 "url": "https://commoncrawl.org/bot.html"
	},
	{"id": "mjbot",
	 "n": "Majestic Crawler (UK)",
	 "r": ["MJ12bot"],
	 "rx": ["MJ12bot\\/v?(\\d+\\.\\d+)[\\s\\.;]"],
	 "url": "http://www.majestic12.co.uk/bot.php"
	},
	{"id": "petal",
	 "n": "PetalSearch Bot (CN)",
	 "r": ["PetalBot", "AspiegelBot"],
	 "rx": ["[\\s;]PetalBot[\\s\\/;]", "AspiegelBot[\\)$]"],
	 "url": "https://webmaster.petalsearch.com/site/petalbot"
	},
	{"id": "barkrowler",
	 "n": "Barkrowler (Babbar Bot)",
	 "r": ["Barkrowler"],
	 "rx": ["[\\s;^]Barkrowler\\/(\\d+\\.\\d+)?"],
	 "url": "https://babbar.tech/crawler"
	},
	{"id": "semrush",
	 "n": "SemrushBot",
	 "r": ["SemrushBot"],
	 "rx": ["SemrushBot-?[A-Z]*\\/(\\d+\\.?\\d*)[;~\\s]"],
	 "url": "http://www.semrush.com/bot.html"
	},
	{"id": "bytespider",
	 "n": "Bytespider (ByteDance, TikTok)",
	 "r": ["Bytespider"],
	 "rx": ["Bytespider[;$]"],
	 "url": "https://darkvisitors.com/agents/bytespider"
	},
	{"id": "dfseo",
	 "n": "DataForSEO Bot",
	 "r": ["DataForSeoBot"],
	 "rx": ["DataForSeoBot[;\\/](\\d\\.\\d)?"],
	 "url": "https://dataforseo.com/dataforseo-bot"
	},
	{"id": "hunter",
	 "n": "Hunter (Velen) Crawler",
	 "r": ["VelenPublicWebCrawler"],
	 "rx": ["VelenPublicWebCrawler\\/(\\d\\.\\d)"],
	 "url": "https://velen.io"
	},
	{"id": "serpstat",
	 "n": "Serpstat Bot",
	 "r": ["serpstatbot"],
	 "rx": ["serpstatbot\\/(\\d\\.\\d)\\s"],
	 "url": "https://serpstatbot.com/"
	},
	{"id": "netestate",
	 "n": "netEstate NE Crawler (DE)",
	 "r": ["netEstate NE Crawler"],
	 "rx": ["netEstate NE Crawler\\s"],
	 "url": "http://www.website-datenbank.de/"
	},
	{"id": "imagesift",
	 "n": "Hive Imagesift Bot (AI Scraper)",
	 "r": ["ImagesiftBot"],
	 "rx": ["ImagesiftBot[;\\s\\/]"],
	 "url": "https://darkvisitors.com/agents/imagesiftbot"
	},
	{"id": "moz",
	 "n": "Moz.com Bots",
	 "r": ["dotbot", "rogerbot"],
	 "rx": ["DotBot\\/(\\d\\.\\d)[\\.;\\s]", "RogerBot\\/(\\d\\.\\d)[\\.;\\s]"],
	 "url": "https://moz.com/help/moz-procedures/crawlers/dotbot"
	},
	{"id": "maui",
	 "n": "MauiBot (unknown developer)",
	 "r": ["MauiBot"],
	 "rx": ["MauiBot\\s"],
	 "url": "https://darkvisitors.com/agents/mauibot"
	},
	{"id": "plagaware",
	 "n": "PlagAwareBot (DE)",
	 "r": ["PlagAwareBot"],
	 "rx": ["PlagAwareBot\\/(\\d\\.\\d)"],
	 "url": "https://www.plagaware.com/bot"
	},
	{"id": "blexbot",
	 "n": "BLEXBot",
	 "r": ["BLEXBot"],
	 "rx": ["BLEXBot\\/(\\d\\.\\d);"],
	 "url": "https://help.seranking.com/en/blex-crawler"
	},
	{"id": "sogou",
	 "n": "Sogou web spider",
	 "r": ["Sogou web spider"],
	 "rx": ["Sogou web spider\\/(\\d+\\.\\d+)[;\\s\\(\\.]"],
	 "url": "http://www.sogou.com/docs/help/webmasters.htm#07"
	},
	{"id": "amazon",
	 "n": "Amazonbot",
	 "r": ["Amazonbot"],
	 "rx": ["Amazonbot\\/(\\d+\\.\\d+)[;\\s\\(\\.]"],
	 "url": "https://developer.amazon.com/amazonbot"
	},
	{"id": "sop",
	 "n": "SitesOverPagesBot",
	 "r": ["SitesOverPagesBot"],
	 "rx": ["SitesOverPagesBot\\/(\\d+\\.\\d+)[;\\s\\(\\.]"],
	 "url": "https://www.sitesoverpages.com/bot"
	},
	{"id": "startme",
	 "n": "StartmeBot",
	 "r": ["StartmeBot"],
	 "rx": ["StartmeBot\\/(\\d+\\.\\d+)[;\\s\\(\\.]"],
	 "url": "https://support.start.me/en/articles/9182933-what-is-startmebot"
	},
	{"id": "bitsight",
	 "n": "BitSightBot",
	 "r": ["BitSightBot"],
	 "rx": ["BitSightBot\\/(\\d+\\.\\d+)"],
	 "url": "https://darkvisitors.com/agents/bitsightbot"
	}
]