diff --git a/.gitignore b/.gitignore index f17cee9..9e5648a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,4 @@ .DS_Store -*.log -*.srv -*.tck logs/*.log.txt logs/*.srv.txt logs/*.tck.txt diff --git a/data/known-bots.json b/data/known-bots.json index 50f2b81..e3d9ec0 100644 --- a/data/known-bots.json +++ b/data/known-bots.json @@ -29,14 +29,14 @@ "rx": ["Applebot\\/(\\d+\\.\\d+);"], "url": "http://www.apple.com/go/applebot" }, - {"id": "openaibots", - "n": "OpenAI/ChatGPT Bots", + {"id": "openai", + "n": "OpenAI/ChatGPT", "r": ["OAI-SearchBot", "ChatGPT-User", "GPTBot"], "rx": ["OAI-SearchBot\\/(\\d+\\.\\d+);", "ChatGPT-User\\/(\\d+\\.\\d+);", "GPTBot\\/(\\d+\\.\\d+);"], "url": "https://platform.openai.com/docs/bots/" }, {"id": "metabots", - "n": "Meta Web Crawlers", + "n": "Meta/Facebook", "r": ["facebookexternalhit", "facebookcatalog","meta-webindexer","meta-externalads","meta-externalagent","meta-externalfetcher"], "rx": ["facebook\\w+\\/(\\d+\\.\\d+)", "meta-\\w+\\/(\\d+\\.\\d+)"], "url": "https://developers.facebook.com/docs/sharing/webmasters/crawler" diff --git a/data/known-clients.json b/data/known-clients.json index aa93358..dbabeee 100644 --- a/data/known-clients.json +++ b/data/known-clients.json @@ -15,6 +15,14 @@ "id": "huawei", "rx": [ "\\sHuaweiBrowser\\/(\\d+\\.\\d+)[\\s\\.]", "\\/harmony360Browser\\/(\\d+\\.\\d+)[\\s\\.]"] }, + {"n": "DuckDuckGo", + "id": "ddg", + "rx": [ "\\sDdg\\/(\\S+)" ] + }, + {"n": "Vivaldi", + "id": "vivaldi", + "rx": [ "\\sVivaldi\\/(\\d+\\.\\d+)[\\s\\.]" ] + }, {"n": "Internet Explorer", "id": "msie", "rx": [ "\\sMSIE\\s(\\d+\\.\\d+b?);" ], @@ -34,16 +42,12 @@ }, {"n": "Chrome", "id": "chrome", - "rx": [ "\\sChrome\\/(1\\d\\d\\.\\d+)[\\.\\s;]" ] + "rx": [ "\\sChrome\\/(1\\d\\d)\\.\\d+" ] }, {"n": "Safari", "id": "safari", "rx": [ "\\sSafari\\/(\\S+)" ] }, - {"n": "DuckDuckGo", - "id": "ddg", - "rx": [ "\\sDdg\\/(\\S+)" ] - }, {"n": "Firefox", "id": "firefox", "rx": [ "\\sFirefox\\/(\\S+)" ] diff --git a/data/known-platforms.json b/data/known-platforms.json index 3148353..51c4f60 100644 --- a/data/known-platforms.json +++ b/data/known-platforms.json @@ -1,15 +1,19 @@ [ {"n": "Win10/11", "id": "win10", - "rx": [ "\\(Windows NT (1\\d\\.\\d);" ] + "rx": [ "[\\(\\s]Windows\\sNT\\s(1\\d\\.\\d)[\\)\\s\\.;]" ] }, {"n": "Linux", "id": "linux", "rx": [ "\\sLinux\\s" ] }, + {"n": "BSD", + "id": "bsd", + "rx": [ "\\sNetBSD[\\);\\s]", "\\sOpenBSD[\\);\\s]", "\\sFreeBSD[\\);\\s]" ] + }, {"n": "iOS/iPadOS", "id": "ios", - "rx": [ "\\sFxiOS\\/(\\d+\\.\\d+)\\s", "\\sCriOS\\/(\\d+\\.\\d+)\\." ] + "rx": [ "\\sFxiOS\\/(\\d+\\.\\d+)\\s", "\\siPhone\\sOS\\s([\\d\\._]+)\\s", "\\siPadOS\\s([\\d\\._]+)\\s", "\\sCriOS\\/(\\d+\\.\\d+)\\." ] }, {"n": "Android", "id": "android", diff --git a/img/ahrefs.png b/img/ahrefs.png new file mode 100644 index 0000000..1802d9f Binary files /dev/null and b/img/ahrefs.png differ diff --git a/img/anthropic.png b/img/anthropic.png new file mode 100644 index 0000000..18ea8a0 Binary files /dev/null and b/img/anthropic.png differ diff --git a/img/babbar.png b/img/babbar.png new file mode 100644 index 0000000..ffede22 Binary files /dev/null and b/img/babbar.png differ diff --git a/img/bounce.svg b/img/bounce.svg new file mode 100644 index 0000000..1f4d43a --- /dev/null +++ b/img/bounce.svg @@ -0,0 +1,56 @@ + + + + + arrow-down-left + + + + arrow-down-left + + + + + + diff --git a/img/bytedance.svg b/img/bytedance.svg new file mode 100644 index 0000000..7b7cef5 --- /dev/null +++ b/img/bytedance.svg @@ -0,0 +1,65 @@ + + + + + + + + + + + + diff --git a/img/ccbot.svg b/img/ccbot.svg new file mode 100644 index 0000000..3bf9da5 --- /dev/null +++ b/img/ccbot.svg @@ -0,0 +1,70 @@ + + + + + + + + + + + + diff --git a/img/dataforseo.png b/img/dataforseo.png new file mode 100644 index 0000000..c4da2c3 Binary files /dev/null and b/img/dataforseo.png differ diff --git a/img/freebsd.png b/img/freebsd.png new file mode 100644 index 0000000..6305f58 Binary files /dev/null and b/img/freebsd.png differ diff --git a/img/hive.svg b/img/hive.svg new file mode 100644 index 0000000..98c9c18 --- /dev/null +++ b/img/hive.svg @@ -0,0 +1,105 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/img/hunter.png b/img/hunter.png new file mode 100644 index 0000000..4180226 Binary files /dev/null and b/img/hunter.png differ diff --git a/img/majestic.png b/img/majestic.png new file mode 100644 index 0000000..4a1498d Binary files /dev/null and b/img/majestic.png differ diff --git a/img/netestate.png b/img/netestate.png new file mode 100644 index 0000000..8bcabaa Binary files /dev/null and b/img/netestate.png differ diff --git a/img/page.svg b/img/page.svg new file mode 100644 index 0000000..91f925b --- /dev/null +++ b/img/page.svg @@ -0,0 +1 @@ +page \ No newline at end of file diff --git a/img/perplexity.svg b/img/perplexity.svg new file mode 100644 index 0000000..4bf985e --- /dev/null +++ b/img/perplexity.svg @@ -0,0 +1,49 @@ + + + + + + + diff --git a/img/petal.svg b/img/petal.svg new file mode 100644 index 0000000..41e7619 --- /dev/null +++ b/img/petal.svg @@ -0,0 +1,110 @@ + + + + + Petal Search logo + + + + + + + + + + + + + Petal Search logo + 16 agt 2021 + + es + + + + + + + + + + + + + + + diff --git a/img/semrush.png b/img/semrush.png new file mode 100644 index 0000000..5e15d0d Binary files /dev/null and b/img/semrush.png differ diff --git a/img/serpstat.svg b/img/serpstat.svg new file mode 100644 index 0000000..90d4891 --- /dev/null +++ b/img/serpstat.svg @@ -0,0 +1,56 @@ + + + + + + + + diff --git a/img/vivaldi.svg b/img/vivaldi.svg new file mode 100644 index 0000000..c3c211e --- /dev/null +++ b/img/vivaldi.svg @@ -0,0 +1,59 @@ + + + + + + + + + + + + diff --git a/php_errors.log b/php_errors.log new file mode 100644 index 0000000..3705f86 --- /dev/null +++ b/php_errors.log @@ -0,0 +1,8 @@ +[03-Sep-2025 15:22:41 UTC] PHP Warning: Undefined variable $file in D:\Webroot\development\lib\plugins\botmon\cleanup.php on line 14 +[03-Sep-2025 15:22:54 UTC] PHP Warning: Undefined variable $file in D:\Webroot\development\lib\plugins\botmon\cleanup.php on line 14 +[03-Sep-2025 15:25:36 UTC] PHP Parse error: syntax error, unexpected end of file, expecting "," or ";" in D:\Webroot\development\lib\plugins\botmon\cleanup.php on line 15 +[03-Sep-2025 15:35:57 UTC] PHP Deprecated: Using ${var} in strings is deprecated, use {$var} instead in D:\Webroot\development\lib\plugins\botmon\cleanup.php on line 14 +[03-Sep-2025 15:37:51 UTC] PHP Warning: unlink(logs/.): Is a directory in D:\Webroot\development\lib\plugins\botmon\cleanup.php on line 15 +[03-Sep-2025 15:37:51 UTC] PHP Warning: unlink(logs/..): Resource temporarily unavailable in D:\Webroot\development\lib\plugins\botmon\cleanup.php on line 15 +[03-Sep-2025 15:38:10 UTC] PHP Warning: unlink(logs/.): Is a directory in D:\Webroot\development\lib\plugins\botmon\cleanup.php on line 17 +[03-Sep-2025 15:38:10 UTC] PHP Warning: unlink(logs/..): Resource temporarily unavailable in D:\Webroot\development\lib\plugins\botmon\cleanup.php on line 17 diff --git a/script.js b/script.js index a1c723a..8437043 100644 --- a/script.js +++ b/script.js @@ -3,6 +3,16 @@ /* 04.09.2025 - 0.1.8 - pre-release */ /* Authors: Sascha Leib */ +// enumeration of user types: +const BM_USERTYPE = Object.freeze({ + 'UNKNOWN': 'unknown', + 'KNOWN_USER': 'user', + 'HUMAN': 'human', + 'LIKELY_BOT': 'likely_bot', + 'KNOWN_BOT': 'known_bot' +}); + +/* BotMon root object */ const BotMon = { init: function() { @@ -179,7 +189,9 @@ BotMon.live = { _visitors: [], // find an already existing visitor record: - findVisitor: function(id) { + findVisitor: function(visitor) { + //console.info('BotMon.live.data.model.findVisitor()'); + //console.log(visitor); // shortcut to make code more readable: const model = BotMon.live.data.model; @@ -187,7 +199,29 @@ BotMon.live = { // loop over all visitors already registered: for (let i=0; i= 1.0) { // known bots + if (v._type == BM_USERTYPE.KNOWN_BOT) { // known bots this.data.bots.known += 1; this.groups.knownBots.push(v); - } if (v.usr && v.usr != '') { // known users */ + } else if (v._type == BM_USERTYPE.KNOWN_USER) { // known users */ + this.groups.users.push(v); this.data.bots.users += 1; - /*} else { - // not a known bot, nor a known user; check other aspects: + + } else { - // no referrer at all: - if (!v._hasReferrer) botScore += 0.2; - - // no js client logging: - if (!v._jsClient) botScore += 0.2; - - // average time between page views less than 30s: - if (v._pageViews.length > 1) { - botScore -= 0.2; // more than one view: good! - let totalDiff = 0; - for (let i=1; i= 0.5) { - this.data.bots.suspected += 1; - this.groups.suspectedBots.push(v); - } else { - this.data.bots.human += 1; - this.groups.humans.push(v); - } - }*/ + // TODO: find suspected bots + this.data.bots.suspected += 1; + this.groups.suspectedBots.push(v); + + } }); - console.log(this.data); - console.log(this.groups); + //console.log(this.data); + //console.log(this.groups); } }, @@ -736,8 +736,13 @@ BotMon.live = { overview: { make: function() { + const data = BotMon.live.data.analytics.data; const parent = document.getElementById('botmon__today__content'); + + // shortcut for neater code: + const makeElement = BotMon.t._makeElement; + if (parent) { const bounceRate = Math.round(data.totalVisits / data.totalPageViews * 1000) / 10; @@ -760,15 +765,29 @@ BotMon.live = {
Probably humans:${data.bots.human}
Registered users:${data.bots.users}
-
-
Known bots
-
+
`)); + + // update known bots list: + const block = document.getElementById('botmon__botslist'); + block.innerHTML = "
Top known bots
"; + + let bots = BotMon.live.data.analytics.groups.knownBots.toSorted( (a, b) => { + return b._pageViews.length - a._pageViews.length; + }); + + for (let i=0; i < Math.min(bots.length, 4); i++) { + const dd = makeElement('dd'); + dd.appendChild(makeElement('span', {'class': 'bot bot_' + bots[i]._bot.id}, bots[i]._bot.n)); + dd.appendChild(makeElement('span', undefined, bots[i]._pageViews.length)); + block.appendChild(dd); + } } } }, + status: { setText: function(txt) { const el = document.getElementById('botmon__today__status'); @@ -864,7 +883,7 @@ BotMon.live = { }, _onDetailsToggle: function(e) { - console.info('BotMon.live.gui.lists._onDetailsToggle()'); + //console.info('BotMon.live.gui.lists._onDetailsToggle()'); const target = e.target; @@ -904,17 +923,20 @@ BotMon.live = { const span1 = make('span'); /* left-hand group */ - if (data._type == 'bot') { /* Bot only */ + const platformName = (data._platform ? data._platform.n : 'Unknown'); + const clientName = (data._client ? data._client.n: 'Unknown'); + + if (data._type == BM_USERTYPE.KNOWN_BOT) { /* Bot only */ span1.appendChild(make('span', { /* Bot */ 'class': 'bot bot_' + (data._bot ? data._bot.id : 'unknown'), 'title': "Bot: " + (data._bot ? data._bot.n : 'Unknown') }, (data._bot ? data._bot.n : 'Unknown'))); - } else if (data._type == 'usr') { /* User only */ + } else if (data._type == BM_USERTYPE.KNOWN_USER) { /* User only */ span1.appendChild(make('span', { /* User */ - 'class': 'user' + (data._user ? data._user.id : 'unknown'), + 'class': 'user_known', 'title': "User: " + data.usr }, data.usr)); @@ -928,27 +950,24 @@ BotMon.live = { } - const platformName = (data._platform ? data._platform.n : 'Unknown'); - span1.appendChild(make('span', { /* Platform */ - 'class': 'icon platform platform_' + (data._platform ? data._platform.id : 'unknown'), - 'title': "Platform: " + platformName - }, platformName)); + if (data._type !== BM_USERTYPE.KNOWN_BOT) { /* Not for bots */ + span1.appendChild(make('span', { /* Platform */ + 'class': 'icon platform platform_' + (data._platform ? data._platform.id : 'unknown'), + 'title': "Platform: " + platformName + }, platformName)); - const clientName = (data._client ? data._client.n: 'Unknown'); - span1.appendChild(make('span', { /* Client */ - 'class': 'icon client client_' + (data._client ? data._client.id : 'unknown'), - 'title': "Client: " + clientName - }, clientName)); - - + span1.appendChild(make('span', { /* Client */ + 'class': 'icon client client_' + (data._client ? data._client.id : 'unknown'), + 'title': "Client: " + clientName + }, clientName)); + } summary.appendChild(span1); const span2 = make('span'); /* right-hand group */ - span2.appendChild(make('time', { /* Last seen */ - 'data-field': 'last-seen', - 'datetime': (data._lastSeen ? data._lastSeen : 'unknown') - }, (data._lastSeen ? data._lastSeen.getHours() + ':' + data._lastSeen.getMinutes() + ':' + data._lastSeen.getSeconds() : 'Unknown'))); + span2.appendChild(make('span', { /* page views */ + 'class': 'pageviews' + }, data._pageViews.length)); summary.appendChild(span2); @@ -956,20 +975,33 @@ BotMon.live = { const dl = make('dl', {'class': 'visitor_details'}); - if (data._bot) { - dl.appendChild(make('dt', {}, "Bot:")); /* bot info */ + if (data._type == BM_USERTYPE.KNOWN_BOT) { + + dl.appendChild(make('dt', {}, "Bot name:")); /* bot info */ dl.appendChild(make('dd', {'class': 'has_icon bot bot_' + (data._bot ? data._bot.id : 'unknown')}, (data._bot ? data._bot.n : 'Unknown'))); + + if (data._bot && data._bot.url) { + dl.appendChild(make('dt', {}, "Bot info:")); /* bot info */ + const botInfoDd = dl.appendChild(make('dd')); + botInfoDd.appendChild(make('a', { + 'href': data._bot.url, + 'target': '_blank' + }, data._bot.url)); /* bot info link*/ + + } + + } else { /* not for bots */ + + dl.appendChild(make('dt', {}, "Client:")); /* client */ + dl.appendChild(make('dd', {'class': 'has_icon client_' + (data._client ? data._client.id : 'unknown')}, + clientName + ( data._client.v > 0 ? ' (' + data._client.v + ')' : '' ) )); + + dl.appendChild(make('dt', {}, "Platform:")); /* platform */ + dl.appendChild(make('dd', {'class': 'has_icon platform_' + (data._platform ? data._platform.id : 'unknown')}, + platformName + ( data._platform.v > 0 ? ' (' + data._platform.v + ')' : '' ) )); } - dl.appendChild(make('dt', {}, "Client:")); /* client */ - dl.appendChild(make('dd', {'class': 'has_icon client_' + (data._client ? data._client.id : 'unknown')}, - clientName + ( data._client.v > 0 ? ' (' + data._client.v + ')' : '' ) )); - - dl.appendChild(make('dt', {}, "Platform:")); /* platform */ - dl.appendChild(make('dd', {'class': 'has_icon platform_' + (data._platform ? data._platform.id : 'unknown')}, - platformName + ( data._platform.v > 0 ? ' (' + data._platform.v + ')' : '' ) )); - dl.appendChild(make('dt', {}, "IP-Address:")); dl.appendChild(make('dd', {'class': 'has_icon ip' + ipType}, data.ip)); @@ -986,6 +1018,12 @@ BotMon.live = { dl.appendChild(make('dt', {}, "User-Agent:")); dl.appendChild(make('dd', {'class': 'agent' + ipType}, data.agent)); + dl.appendChild(make('dt', {}, "Visitor Type:")); + dl.appendChild(make('dd', undefined, data._type )); + + dl.appendChild(make('dt', {}, "Seen by:")); + dl.appendChild(make('dd', undefined, data._seenBy.join(', ') )); + dl.appendChild(make('dt', {}, "Visited pages:")); const pagesDd = make('dd', {'class': 'pages'}); const pageList = make('ul'); @@ -1013,6 +1051,7 @@ BotMon.live = { li.appendChild(details); return li; } + } } }; diff --git a/style.less b/style.less index 85e73e5..cbc3b89 100644 --- a/style.less +++ b/style.less @@ -129,7 +129,8 @@ color: #000; } details ul > li > details { - border: red dotted 1px; + border: #ccc solid 1px; + border-radius: .5em; } details ul > li > details > summary { display: flex; @@ -139,24 +140,30 @@ font-weight: normal; font-size: 1rem; line-height: 1.5; - border: blue dashed 1px; + background-color: #F0F0F0; + border-bottom: #CCC solid 1px; + border-radius: .5em; } details ul > li > details > summary > span { display: flex; + align-items: center; column-gap: .25em; } details ul > li > details > summary > span:first-child { flex-grow: 1; } details ul > li > details > summary > span > span { + display: flex; + align-items: center; + column-gap: .25em; height: 1.5em; overflow: hidden; } details ul > li > details > summary > span > span::before { content: ''; display: inline-block; - width: 1.25em; height: 1em; + min-width: 1.25em; height: 1em; text-align: center; background: transparent url('img/placeholder.svg') center no-repeat; background-size: 1em; @@ -171,9 +178,9 @@ dl.visitor_details { & { - border: green dotted 1px; display: grid; grid-template-columns: min-content auto; + border-left: transparent none 0; } dt { grid-column: 1; @@ -184,6 +191,19 @@ display: inline-block; background-color: transparent; } + dd.pages { + & { + } + ul { + li { + & { + display: flex; + justify-content: space-between; + align-items: center; + } + } + } + } } dd.has_icon::before { content: ''; @@ -200,12 +220,13 @@ span.bot_googleads::before, dd.bot_googleads::before, span.bot_googleapi::before, dd.bot_googleapi::before { background-image: url('img/google.svg') } span.bot_applebot::before, dd.bot_applebot::before { background-image: url('img/apple.svg') } + span.bot_openai::before, dd.bot_openai::before { background-image: url('img/openai.svg') } span.bot_metabots::before, dd.bot_metabots::before { background-image: url('img/meta.svg') } span.bot_yandexbots::before, dd.bot_yandexbots::before { background-image: url('img/yandex.svg') } span.bot_seznambot::before, dd.bot_seznambot::before { background-image: url('img/seznam.svg') } /* user info */ - span.user::before { background-image: url('img/user.svg') } + span.user_known::before { background-image: url('img/user.svg') } /* platform icons */ span.platform_macos::before, dd.platform_macos::before { background-image: url('img/apple.svg') } @@ -217,6 +238,7 @@ span.platform_tizen::before, dd.platform_tizen::before { background-image: url('img/tizen.png') } span.platform_hmos::before, dd.platform_hmos::before { background-image: url('img/hmos.svg') } span.platform_chromium::before, dd.platform_chromium::before { background-image: url('img/chromium.svg') } + span.platform_bsd::before, dd.platform_bsd::before { background-image: url('img/freebsd.png') } /* browser icons */ span.client_opera::before, dd.client_opera::before { background-image: url('img/opera.svg') } @@ -231,6 +253,7 @@ span.client_samsung::before, dd.client_samsung::before { background-image: url('img/samsung.svg') } span.client_uc::before, dd.client_uc::before { background-image: url('img/uc.svg') } span.client_huawei::before, dd.client_huawei::before { background-image: url('img/huawei.png') } + span.client_vivaldi::before, dd.client_vivaldi::before { background-image: url('img/vivaldi.png') } /* ip address type */ span.ip6::before, dd.ip6::before { background-image: url('img/ip6.svg') } @@ -240,6 +263,15 @@ /* user agent */ span.agent::before { background-image: url('img/info.svg') } + /* pageviews */ + span.pageviews { + border: #999 solid 1px; + padding: 0 2px; + font-size: smaller; + border-radius: .25em; + } + span.pageviews::before { background-image: url('img/page.svg') } + } /* item footer */