diff --git a/admin.css b/admin.css index 85de5ac..734ee97 100644 --- a/admin.css +++ b/admin.css @@ -46,8 +46,8 @@ &.bot_googleother::before, &.bot_googinspct::before { background-position-y: -20px } &.bot_bingbot::before { background-position-y: -40px } &.bot_applebot::before { background-position-y: -60px } - &.bot_openai::before { background-position-y: -80px } - &.bot_metabots::before { background-position-y: -100px } + &.bot_openai::before, &.bot_chatgpt::before { background-position-y: -80px } + &.bot_metabots::before, &.bot_metauser::before { background-position-y: -100px } &.bot_ahrefs::before { background-position-y: -120px } &.bot_seznambot::before { background-position-y: -140px } &.bot_claude::before { background-position-y: -160px } @@ -58,6 +58,12 @@ &.bot_ibou::before { background-position-y: -260px } &.bot_discord::before { background-position-y: -280px } &.bot_twitter::before { background-position-y: -300px } + &.bot_goognblm::before { background-position-y: -320px } + &.bot_whatsapp::before { background-position-y: -340px } + &.bot_perplexity::before, &.bot_perplexityusr::before { background-position-y: -360px } + &.bot_tiktok::before { background-position-y: -380px } + &.bot_claudeusr::before { background-position-y: -400px } + &.bot_yahoolp::before { background-position-y: -420px } &.bot_other::before { background-image: url('img/more.svg') } /* platform icons */ @@ -100,15 +106,6 @@ &.cl_ecosia::before { background-position-y: -340px } &.cl_webkit::before { background-position-y: -360px } &.cl_operaold::before { background-position-y: -380px } - &.cl_whatsapp::before { background-position-y: -400px } - &.cl_applemsgs::before { background-position-y: -420px } - &.cl_goognblm::before { background-position-y: -440px } - &.cl_chatgpt::before { background-position-y: -460px } - &.cl_meta::before { background-position-y: -480px } - &.cl_yahoolp::before { background-position-y: -500px } - &.cl_tiktok::before { background-position-y: -520px } - &.cl_claude::before { background-position-y: -540px } - &.cl_perplexity::before { background-position-y: -560px } &.cl_other::before { background-image: url('img/more.svg') } /* Country flags */ diff --git a/config/default-config.json b/config/default-config.json index 9b48a26..dbe7b86 100644 --- a/config/default-config.json +++ b/config/default-config.json @@ -59,7 +59,7 @@ }, {"func": "loadSpeed", "params": [3, 20], "id": "speedRun", "desc": "Average time between page loads is less than 20 seconds", - "bot": 80 + "bot": 60 }, {"func": "noAcceptLang", "id": "noAcc", "desc": "No “Accept-Language” header", diff --git a/config/known-bots.json b/config/known-bots.json index 42fc100..0688a75 100644 --- a/config/known-bots.json +++ b/config/known-bots.json @@ -34,14 +34,20 @@ "rx": ["\\sGoogle-InspectionTool(\\-\\w+)?[\\)\\/]"], "url": "https://support.google.com/webmasters/answer/9012289" }, + {"id": "goognblm", + "n": "Google NotebookLM User", + "r": ["Google-NotebookLM"], + "rx": ["Google-NotebookLM"], + "url": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers" + }, {"id": "applebot", "n": "Applebot", - "r": ["Applebot"], - "rx": ["Applebot\\/(\\d+\\.\\d+);"], + "r": ["Applebot", "Applebot-Extended"], + "rx": ["Applebot\\/(\\d+\\.\\d+);", "Applebot\\-Extended\\/(\\d+\\.\\d+);"], "url": "http://www.apple.com/go/applebot" }, {"id": "baidu", - "n": "Baidu Spiders", + "n": "Baidu Bots", "r": ["Baiduspider", "Baiduspider-image", "Baiduspider-video", "Baiduspider-news", "Baiduspider-favo", "Baiduspider-cpro", "Baiduspider-ads", "Baiduspider-render"], "rx": ["Baiduspider"], "url": "http://www.baidu.com/search/spider.html" @@ -58,6 +64,12 @@ "rx": ["Slurp[\\s;\\)]"], "url": "http://help.yahoo.com/help/us/ysearch/slurp" }, + {"id": "yahoolp", + "n": "Yahoo! Link Preview (User)", + "r": ["Yahoo Link Preview"], + "rx": ["Yahoo Link Preview;"], + "url": "https://help.yahoo.com/kb/SLN23615.html" + }, {"id": "ddg", "n": "DuckDuck Bot", "r": ["DuckDuckBot","DuckAssistBot","DuckDuckGo-Favicons-Bot"], @@ -76,24 +88,48 @@ "rx": ["OAI-SearchBot\\/(\\d+\\.\\d+);", "GPTBot\\/(\\d+\\.\\d+);"], "url": "https://platform.openai.com/docs/bots/" }, + {"id": "chatgpt", + "n": "ChatGPT User", + "r": ["ChatGPT-User"], + "rx": ["ChatGPT\\-User\\/(\\d\\.\\d);"], + "url": "https://platform.openai.com/docs/bots/" + }, {"id": "claude", "n": "Anthropic Claude", - "r": ["ClaudeBot", "Claude-User", "Claude-SearchBot"], + "r": ["ClaudeBot", "Claude-SearchBot"], "rx": ["ClaudeBot\\/(\\d+\\.\\d+);"], - "url": "https://darkvisitors.com/agents/claudebot" + "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler" + }, + {"id": "claudeusr", + "n": "ClaudeAI User", + "r": ["Claude-User"], + "rx": ["Claude-User\\/(\\d+\\.\\d+);"], + "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler" }, {"id": "perplexity", - "n": "Perplexity", - "r": ["PerplexityBot", "Perplexity‑User"], - "rx": ["PerplexityBot\\/(\\d+\\.\\d+);", "Perplexity‑User\\/(\\d+\\.\\d+);"], + "n": "Perplexity Bot", + "r": ["PerplexityBot"], + "rx": ["PerplexityBot\\/(\\d+\\.\\d+);"], + "url": "https://perplexity.ai/perplexitybot" + }, + {"id": "perplexityusr", + "n": "Perplexity User", + "r": ["Perplexity‑User"], + "rx": ["Perplexity\\-User\\/(\\d+\\.\\d+);"], "url": "https://perplexity.ai/perplexitybot" }, {"id": "metabots", "n": "Meta/Facebook", - "r": ["facebookcatalog","meta-webindexer","meta-externalads","meta-externalagent"], + "r": ["meta-webindexer","meta-externalads","meta-externalagent"], "rx": ["facebook[cw]\\w+\\/(\\d+\\.\\d+)", "meta-externala\\w+\\/(\\d+\\.\\d+)"], "url": "https://developers.facebook.com/docs/sharing/webmasters/crawler" }, + {"id": "metauser", + "n": "Meta/Facebook User", + "r": ["facebookexternalhit","facebookcatalog"], + "rx": ["facebookexternalhit\\/(\\d+\\.?\\d*)", "meta-externalfetcher\\/(\\d\\.\\d)", "facebookcatalog\\/(\\d+\\.?\\d*)"], + "url": "https://developers.facebook.com/docs/sharing/webmasters/crawler" + }, {"id": "qwant", "n": "Qwant Web crawler", "r": ["Qwantbot", "Qwantbot-news"], @@ -203,17 +239,29 @@ "url": "https://www.plagaware.com/bot" }, {"id": "discord", - "n": "Discord", + "n": "Discord User", "r": ["Discordbot"], "rx": ["Discordbot\\/(\\d\\.\\d);"], "url": "https://datadome.co/bots/discordbot/" }, {"id": "twitter", - "n": "Twitterbot", + "n": "Twitter User", "r": ["Twitterbot"], - "rx": ["Twitterbot\\/(\\d+\\.\\d+)"], + "rx": ["^Twitterbot\\/(\\d\\.?\\d*)"], "url": "https://datadome.co/bots/twitterbot/" }, + {"id": "whatsapp", + "n": "WhatsApp User", + "r": ["WhatsApp"], + "rx": ["WhatsApp/(\\d+\\.?\\d*)"], + "url": "https://darkvisitors.com/agents/whatsapp/" + }, + {"id": "tiktok", + "n": "TikTok User", + "r": ["TikTokSpider"], + "rx": ["TikTokSpider;"], + "url": "https://darkvisitors.com/agents/tiktokspider" + }, {"id": "blexbot", "n": "BLEXBot", "r": ["BLEXBot"], diff --git a/config/known-clients.json b/config/known-clients.json index 1bd2701..d0141ee 100644 --- a/config/known-clients.json +++ b/config/known-clients.json @@ -83,42 +83,6 @@ "id": "firefox", "rx": [ "\\sFirefox\\/(\\d\\d\\d)\\." ] }, - {"n": "WhatsApp", - "id": "whatsapp", - "rx": [ "WhatsApp/(\\d+\\.?\\d*)" ] - }, - {"n": "Apple Messages", - "id": "applemsgs", - "rx": ["Facebot Twitterbot\\/(\\d\\.\\d)"] - }, - {"n": "TikTok", - "id": "tiktok", - "rx": ["TikTokSpider;"] - }, - {"n": "Google NotebookLM", - "id": "goognblm", - "rx": ["Google-NotebookLM"] - }, - {"n": "ChatGPT (User)", - "id": "chatgpt", - "rx": ["ChatGPT\\-User\\/(\\d\\.\\d);"] - }, - {"n": "Claude (User)", - "id": "claude", - "rx": ["Claude\\-User\\/(\\d\\.\\d);"] - }, - {"n": "Perplexity (User)", - "id": "perplexity", - "rx": ["Perplexity‑User\\/(\\d+\\.\\d+);"] - }, - {"n": "Meta Apps (User)", - "id": "meta", - "rx": ["facebookexternalhit\\/(\\d\\.\\d)", "meta-externalfetcher\\/(\\d\\.\\d)"] - }, - {"n": "Yahoo! Link Preview", - "id": "yahoolp", - "rx": ["Yahoo Link Preview;"] - }, {"n": "Undici-based crawler", "id": "undici", "rx": ["undici"] diff --git a/config/known-platforms.json b/config/known-platforms.json index 9b78c52..c4e0d14 100644 --- a/config/known-platforms.json +++ b/config/known-platforms.json @@ -57,6 +57,6 @@ }, {"n": "App", "id": "app", - "rx": [ "WhatsApp/(\\d+\\.\\d+)", "Facebot Twitterbot\\/(\\d\\.\\d)", "Google-NotebookLM", "ChatGPT-User", "meta-externalfetcher"] + "rx": [ "WhatsApp/(\\d+\\.\\d+)", "Facebot Twitterbot\\/(\\d\\.\\d)"] } ] \ No newline at end of file diff --git a/img/bots.png b/img/bots.png index 3d099fb..89911a6 100644 Binary files a/img/bots.png and b/img/bots.png differ diff --git a/img/clients.png b/img/clients.png index 3016fc7..6e59da9 100644 Binary files a/img/clients.png and b/img/clients.png differ