Bots/Clients reorg

admin.css

@@ -46,8 +46,8 @@
 			&.bot_googleother::before, &.bot_googinspct::before { background-position-y: -20px }
 		&.bot_bingbot::before { background-position-y: -40px }
 		&.bot_applebot::before { background-position-y: -60px }
-		&.bot_openai::before { background-position-y: -80px }
-		&.bot_metabots::before { background-position-y: -100px }
+		&.bot_openai::before, &.bot_chatgpt::before { background-position-y: -80px }
+		&.bot_metabots::before, &.bot_metauser::before { background-position-y: -100px }
 		&.bot_ahrefs::before { background-position-y: -120px }
 		&.bot_seznambot::before { background-position-y: -140px }
 		&.bot_claude::before { background-position-y: -160px }
@@ -58,6 +58,12 @@
 		&.bot_ibou::before { background-position-y: -260px }
 		&.bot_discord::before { background-position-y: -280px }
 		&.bot_twitter::before { background-position-y: -300px }
+		&.bot_goognblm::before { background-position-y: -320px }
+		&.bot_whatsapp::before { background-position-y: -340px }
+		&.bot_perplexity::before, &.bot_perplexityusr::before { background-position-y: -360px }
+		&.bot_tiktok::before { background-position-y: -380px }
+		&.bot_claudeusr::before { background-position-y: -400px }
+		&.bot_yahoolp::before { background-position-y: -420px }
 		&.bot_other::before { background-image: url('img/more.svg') }
 
 		/* platform icons */	
@@ -100,15 +106,6 @@
 		&.cl_ecosia::before { background-position-y: -340px }
 		&.cl_webkit::before { background-position-y: -360px }
 		&.cl_operaold::before { background-position-y: -380px }
-		&.cl_whatsapp::before { background-position-y: -400px }
-		&.cl_applemsgs::before { background-position-y: -420px }
-		&.cl_goognblm::before { background-position-y: -440px }
-		&.cl_chatgpt::before { background-position-y: -460px }
-		&.cl_meta::before { background-position-y: -480px }
-		&.cl_yahoolp::before { background-position-y: -500px }
-		&.cl_tiktok::before { background-position-y: -520px }
-		&.cl_claude::before { background-position-y: -540px }
-		&.cl_perplexity::before { background-position-y: -560px }
 		&.cl_other::before { background-image: url('img/more.svg') }
 
 		/* Country flags */

config/default-config.json

@@ -59,7 +59,7 @@
 		},
 		{"func": "loadSpeed", "params": [3, 20],
 		 "id": "speedRun", "desc": "Average time between page loads is less than 20 seconds",
-		 "bot": 80
+		 "bot": 60
 		},
 		{"func": "noAcceptLang",
 		 "id": "noAcc", "desc": "No “Accept-Language” header",

config/known-bots.json

@@ -34,14 +34,20 @@
 	 "rx": ["\\sGoogle-InspectionTool(\\-\\w+)?[\\)\\/]"],
 	 "url": "https://support.google.com/webmasters/answer/9012289"
 	},
+	{"id": "goognblm",
+	 "n": "Google NotebookLM User",
+	 "r": ["Google-NotebookLM"],
+	 "rx": ["Google-NotebookLM"],
+	 "url": "https://developers.google.com/search/docs/crawling-indexing/google-user-triggered-fetchers"
+	},
 	{"id": "applebot",
 	 "n": "Applebot",
-	 "r": ["Applebot"],
-	 "rx": ["Applebot\\/(\\d+\\.\\d+);"],
+	 "r": ["Applebot", "Applebot-Extended"],
+	 "rx": ["Applebot\\/(\\d+\\.\\d+);", "Applebot\\-Extended\\/(\\d+\\.\\d+);"],
 	 "url": "http://www.apple.com/go/applebot"
 	},
 	{"id": "baidu",
-	 "n": "Baidu Spiders",
+	 "n": "Baidu Bots",
 	 "r": ["Baiduspider", "Baiduspider-image", "Baiduspider-video", "Baiduspider-news", "Baiduspider-favo", "Baiduspider-cpro", "Baiduspider-ads", "Baiduspider-render"],
 	 "rx": ["Baiduspider"],
 	 "url": "http://www.baidu.com/search/spider.html"
@@ -58,6 +64,12 @@
 	 "rx": ["Slurp[\\s;\\)]"],
 	 "url": "http://help.yahoo.com/help/us/ysearch/slurp"
 	},
+	{"id": "yahoolp",
+	 "n": "Yahoo! Link Preview (User)",
+	 "r": ["Yahoo Link Preview"],
+	 "rx": ["Yahoo Link Preview;"],
+	 "url": "https://help.yahoo.com/kb/SLN23615.html"
+	},
 	{"id": "ddg",
 	 "n": "DuckDuck Bot",
 	 "r": ["DuckDuckBot","DuckAssistBot","DuckDuckGo-Favicons-Bot"],
@@ -76,24 +88,48 @@
 	 "rx": ["OAI-SearchBot\\/(\\d+\\.\\d+);", "GPTBot\\/(\\d+\\.\\d+);"],
 	 "url": "https://platform.openai.com/docs/bots/"
 	},
+	{"id": "chatgpt",
+	 "n": "ChatGPT User",
+	 "r": ["ChatGPT-User"],
+	 "rx": ["ChatGPT\\-User\\/(\\d\\.\\d);"],
+	 "url": "https://platform.openai.com/docs/bots/"
+	},
 	{"id": "claude",
 	 "n": "Anthropic Claude",
-	 "r": ["ClaudeBot", "Claude-User", "Claude-SearchBot"],
+	 "r": ["ClaudeBot", "Claude-SearchBot"],
 	 "rx": ["ClaudeBot\\/(\\d+\\.\\d+);"],
-	 "url": "https://darkvisitors.com/agents/claudebot"
+	 "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
+	},
+	{"id": "claudeusr",
+	 "n": "ClaudeAI User",
+	 "r": ["Claude-User"],
+	 "rx": ["Claude-User\\/(\\d+\\.\\d+);"],
+	 "url": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler"
 	},
 	{"id": "perplexity",
-	 "n": "Perplexity",
-	 "r": ["PerplexityBot", "Perplexity‑User"],
-	 "rx": ["PerplexityBot\\/(\\d+\\.\\d+);", "Perplexity‑User\\/(\\d+\\.\\d+);"],
+	 "n": "Perplexity Bot",
+	 "r": ["PerplexityBot"],
+	 "rx": ["PerplexityBot\\/(\\d+\\.\\d+);"],
+	 "url": "https://perplexity.ai/perplexitybot"
+	},
+	{"id": "perplexityusr",
+	 "n": "Perplexity User",
+	 "r": ["Perplexity‑User"],
+	 "rx": ["Perplexity\\-User\\/(\\d+\\.\\d+);"],
 	 "url": "https://perplexity.ai/perplexitybot"
 	},
 	{"id": "metabots",
 	 "n": "Meta/Facebook",
-	 "r": ["facebookcatalog","meta-webindexer","meta-externalads","meta-externalagent"],
+	 "r": ["meta-webindexer","meta-externalads","meta-externalagent"],
 	 "rx": ["facebook[cw]\\w+\\/(\\d+\\.\\d+)", "meta-externala\\w+\\/(\\d+\\.\\d+)"],
 	 "url": "https://developers.facebook.com/docs/sharing/webmasters/crawler"
 	},
+	{"id": "metauser",
+	 "n": "Meta/Facebook User",
+	 "r": ["facebookexternalhit","facebookcatalog"],
+	 "rx": ["facebookexternalhit\\/(\\d+\\.?\\d*)", "meta-externalfetcher\\/(\\d\\.\\d)", "facebookcatalog\\/(\\d+\\.?\\d*)"],
+	 "url": "https://developers.facebook.com/docs/sharing/webmasters/crawler"
+	},
 	{"id": "qwant",
 	 "n": "Qwant Web crawler",
 	 "r": ["Qwantbot", "Qwantbot-news"],
@@ -203,17 +239,29 @@
 	 "url": "https://www.plagaware.com/bot"
 	},
 	{"id": "discord",
-	 "n": "Discord",
+	 "n": "Discord User",
 	 "r": ["Discordbot"],
 	 "rx": ["Discordbot\\/(\\d\\.\\d);"],
 	 "url": "https://datadome.co/bots/discordbot/"
 	},
 	{"id": "twitter",
-	 "n": "Twitterbot",
+	 "n": "Twitter User",
 	 "r": ["Twitterbot"],
-	 "rx": ["Twitterbot\\/(\\d+\\.\\d+)"],
+	 "rx": ["^Twitterbot\\/(\\d\\.?\\d*)"],
 	 "url": "https://datadome.co/bots/twitterbot/"
 	},
+	{"id": "whatsapp",
+	 "n": "WhatsApp User",
+	 "r": ["WhatsApp"],
+	 "rx": ["WhatsApp/(\\d+\\.?\\d*)"],
+	 "url": "https://darkvisitors.com/agents/whatsapp/"
+	},
+	{"id": "tiktok",
+	 "n": "TikTok User",
+	 "r": ["TikTokSpider"],
+	 "rx": ["TikTokSpider;"],
+	 "url": "https://darkvisitors.com/agents/tiktokspider"
+	},
 	{"id": "blexbot",
 	 "n": "BLEXBot",
 	 "r": ["BLEXBot"],

config/known-clients.json

@@ -83,42 +83,6 @@
 	 "id": "firefox",
 	 "rx": [ "\\sFirefox\\/(\\d\\d\\d)\\." ]
 	},
-	{"n": "WhatsApp",
-	 "id": "whatsapp",
-	 "rx": [ "WhatsApp/(\\d+\\.?\\d*)" ]
-	},
-	{"n": "Apple Messages",
-	 "id": "applemsgs",
-	 "rx": ["Facebot Twitterbot\\/(\\d\\.\\d)"]
-	},
-	{"n": "TikTok",
-	 "id": "tiktok",
-	 "rx": ["TikTokSpider;"]
-	},
-	{"n": "Google NotebookLM",
-	 "id": "goognblm",
-	 "rx": ["Google-NotebookLM"]
-	},
-	{"n": "ChatGPT (User)",
-	 "id": "chatgpt",
-	 "rx": ["ChatGPT\\-User\\/(\\d\\.\\d);"]
-	},
-	{"n": "Claude (User)",
-	 "id": "claude",
-	 "rx": ["Claude\\-User\\/(\\d\\.\\d);"]
-	},
-	{"n": "Perplexity (User)",
-	 "id": "perplexity",
-	 "rx": ["Perplexity‑User\\/(\\d+\\.\\d+);"]
-	},
-	{"n": "Meta Apps (User)",
-	 "id": "meta",
-	 "rx": ["facebookexternalhit\\/(\\d\\.\\d)", "meta-externalfetcher\\/(\\d\\.\\d)"]
-	},
-	{"n": "Yahoo! Link Preview",
-	 "id": "yahoolp",
-	 "rx": ["Yahoo Link Preview;"]
-	},
 	{"n": "Undici-based crawler",
 	 "id": "undici",
 	 "rx": ["undici"]

config/known-platforms.json

@@ -57,6 +57,6 @@
 	},
 	{"n": "App",
 	 "id": "app",
-	 "rx": [ "WhatsApp/(\\d+\\.\\d+)", "Facebot Twitterbot\\/(\\d\\.\\d)", "Google-NotebookLM", "ChatGPT-User", "meta-externalfetcher"]
+	 "rx": [ "WhatsApp/(\\d+\\.\\d+)", "Facebot Twitterbot\\/(\\d\\.\\d)"]
 	}
 ]