56 lines
2.0 KiB
JSON
56 lines
2.0 KiB
JSON
[
|
|
{"id": "bingbot",
|
|
"n": "Bingbot",
|
|
"r": ["bingbot"],
|
|
"rx": ["\\sbingbot\\/(\\d+\\.\\d+);"],
|
|
"url": "http://www.bing.com/bingbot.htm"
|
|
},
|
|
{"id": "googlebot",
|
|
"n": "GoogleBot",
|
|
"r": ["Googlebot"],
|
|
"rx": ["\\sGooglebot\\/(\\d+\\.\\d+);"],
|
|
"url": "http://www.google.com/bot.html"
|
|
},
|
|
{"id": "googleads",
|
|
"n": "Google AdsBot",
|
|
"r": ["AdsBot-Google", "AdsBot-Google-Mobile", "Mediapartners-Google"],
|
|
"rx": ["AdsBot-Google;","AdsBot-Google-Mobile;", "Mediapartners-Google\\/(\\d+\\.\\d+);"],
|
|
"url": "http://www.google.com/mobile/adsbot.html"
|
|
},
|
|
{"id": "googleapi",
|
|
"n": "Google APIs",
|
|
"r": ["APIs-Google"],
|
|
"rx": ["APIs-Google"],
|
|
"url": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers"
|
|
},
|
|
{"id": "applebot",
|
|
"n": "Applebot",
|
|
"r": ["Applebot"],
|
|
"rx": ["Applebot\\/(\\d+\\.\\d+);"],
|
|
"url": "http://www.apple.com/go/applebot"
|
|
},
|
|
{"id": "openai",
|
|
"n": "OpenAI/ChatGPT",
|
|
"r": ["OAI-SearchBot", "ChatGPT-User", "GPTBot"],
|
|
"rx": ["OAI-SearchBot\\/(\\d+\\.\\d+);", "ChatGPT-User\\/(\\d+\\.\\d+);", "GPTBot\\/(\\d+\\.\\d+);"],
|
|
"url": "https://platform.openai.com/docs/bots/"
|
|
},
|
|
{"id": "metabots",
|
|
"n": "Meta/Facebook",
|
|
"r": ["facebookexternalhit", "facebookcatalog","meta-webindexer","meta-externalads","meta-externalagent","meta-externalfetcher"],
|
|
"rx": ["facebook\\w+\\/(\\d+\\.\\d+)", "meta-\\w+\\/(\\d+\\.\\d+)"],
|
|
"url": "https://developers.facebook.com/docs/sharing/webmasters/crawler"
|
|
},
|
|
{"id": "yandexbots",
|
|
"n": "Yandex (Russia)",
|
|
"r": ["YandexBot", "YandexAdNet", "YandexBlogs", "YandexImages", "YandexImageResizer", "YandexMarket", "YandexMedia", "YandexOntoDB", "YandexSitelinks","YandexSpravBot", "YandexVertis", "YandexVerticals", "YandexVideo", "YandexWebmaster", "YandexComBot"],
|
|
"rx": ["Yandex\\w+\\/(\\d+\\.\\d+);"],
|
|
"url": "http://yandex.com/bots"
|
|
},
|
|
{"id": "seznambot",
|
|
"n": "SeznamBot (Czech)",
|
|
"r": ["SeznamBot"],
|
|
"rx": ["SeznamBot\\/(\\d+\\.\\d+);"],
|
|
"url": "https://o-seznam.cz/napoveda/vyhledavani/en/seznambot-crawler/"
|
|
}
|
|
] |