diff --git a/.gitignore b/.gitignore index d0f2f3e..cbf6a5e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ logs/*.log.txt logs/*.srv.txt logs/*.tck.txt +config/user-config.json php_errors.log diff --git a/config/default-config.json b/config/default-config.json index f220192..a3c1e72 100644 --- a/config/default-config.json +++ b/config/default-config.json @@ -23,11 +23,11 @@ }, {"func": "smallPageCount", "params": [1], "id": "onePage", "desc": "Visiter viewed only a single page", - "bot": 40 + "bot": 30 }, {"func": "noRecord", "params": ["log"], "id": "noClient", "desc": "No client-side JS log was recorded", - "bot": 50 + "bot": 40 }, {"func": "noRecord", "params": ["tck"], "id": "noTicks", "desc": "No client ticks were recorded", @@ -45,17 +45,13 @@ "id": "susClient", "desc": "Client identifier that is popular with bot networks", "bot": 10 }, - {"func": "combinationTest", "params": [["macos", "chrome"]], - "id": "unusualPC", "desc": "Unusual combination of platform and client", - "bot": 10 - }, - {"func": "combinationTest", "params": [["macos", "chromeold"],["macosold", "brave"],["winold", "edge"],["winold", "brave"]], + {"func": "combinationTest", "params": [["macos","chromeold"],["macos","msie"],["winold","edge"],["winold","brave"]], "id": "suspPC", "desc": "Suspicious combination of platform and client", "bot": 30 }, - {"func": "combinationTest", "params": [["macos", "msie"], ["win10", "safari"]], + {"func": "combinationTest", "params": [["macos","msie"],["win10","safari"],["macosold","brave"]], "id": "impPC", "desc": "Impossible combination of platform and client", - "bot": 80 + "bot": 70 }, {"func": "loadSpeed", "params": [3, 20], "id": "speedRun", "desc": "Average time between page loads is less than 20 seconds", @@ -65,31 +61,32 @@ "id": "noAcc", "desc": "No “Accept-Language” header", "bot": 40 }, + {"func": "clientAccepts", "params": ["zh"], + "id": "zhLang", "desc": "Client accepts Chinese language", + "bot": 60 + }, {"func": "matchesCountry", "params": ["BR", "CN", "RU", "US", "MX", "SG", "IN", "UY"], "id": "isFrom", "desc": "Location is in a known bot-spamming country.", "bot": 50 - }, - {"func": "matchesCountry", "params": ["ZZ"], - "id": "zzCtry", "desc": "Location could not be determined", - "bot": 20 } ], "ipRanges": [ - {"from": "3.0.0.0", "to": "3.255.255.254", "label": "Amazon Data Services [US]", "g": "US"}, + {"from": "3.0.0.0", "to": "3.255.255.254", "label": "Amazon Data Services [US]"}, {"from": "8.127.0.0", "to": "8.223.255.254", "label": "Alibaba [CN]"}, {"from": "24.240.0.0", "to": "24.247.255.255", "m": 13, "label": "Charter [US]"}, {"from": "27.106.0.0", "to": "27.106.127.254", "label": "Huawei [US]"}, {"from": "34.0.0.0", "to": "34.191.255.254", "label": "Google LLC"}, {"from": "45.0.0.0", "to": "45.255.255.254", "label": "Various small ISPs, mostly BR"}, {"from": "46.250.160.0", "to": "46.250.191.254", "label": "Huawei [MX]"}, + {"from": "47.200.0.0", "to": "47.203.255.255", "m": 14, "label": "Frontier Communications [US]"}, {"from": "49.0.200.0", "to": "49.0.255.254", "label": "Huawei [SG]"}, {"from": "66.249.64.0", "to": "66.249.95.255", "m": 19, "label": "Google LLC [US]"}, {"from": "84.37.35.0", "to": "84.37.255.254", "label": "GTT.net [US]"}, {"from": "94.74.64.0", "to": "94.74.127.254", "label": "Huawei [HK]"}, {"from": "101.0.0.0", "to": "101.255.255.254", "label": "ChinaNet [CN]"}, - {"from": "110.238.96.0", "to": "110.238.127.254", "label": "Huawei [SG]"}, + {"from": "110.238.80.0", "to": "110.238.127.254", "label": "Huawei [SG]"}, {"from": "111.119.192.0", "to": "111.119.255.254", "label": "Huawei [SG]"}, - {"from": "119.0.0.0", "to": "101.207.255.254", "label": "Unicom [CN]"}, + {"from": "119.0.0.0", "to": "119.207.255.254", "label": "Unicom [CN]"}, {"from": "121.91.168.", "to": "121.91.175.254", "label": "Huawei [HK]"}, {"from": "122.8.0.0", "to": "122.8.255.254", "label": "CN-ISP [CN]"}, {"from": "122.9.0.0", "to": "122.9.255.254", "label": "Huawei [CN]"}, @@ -109,8 +106,9 @@ {"from": "190.0.0.0", "to": "190.255.255.254", "label": "South-American ISPs (190.x)"}, {"from": "192.124.170.0", "to": "192.124.182.254", "label": "Relcom [CZ]"}, {"from": "195.37.0.0", "to": "195.37.255.255", "label": "DFN [DE]"}, - {"from": "2001:4800::::::", "to": "2001:4fff:ffff:ffff:ffff:ffff:ffff:ffff", "label": "Rackspace/Google [US]"}, + {"from": "2001:4860::::::", "to": "2001:4860:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Google LLC [US]"}, {"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "VNPT [VN]"}, + {"from": "2408:8210::::::", "to": "2408:8210:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "China Unicom [CN]"}, {"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "m": "Amazon Cloud [US]"}, {"from": "2603:6010::::::", "to": "2603:6010:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Charter [US]"}, {"from": "2603:8000::::::", "to": "2603:80ff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "label": "Charter [US]"}, diff --git a/config/known-bots.json b/config/known-bots.json index ededd49..646823c 100644 --- a/config/known-bots.json +++ b/config/known-bots.json @@ -22,6 +22,12 @@ "rx": ["APIs-Google"], "url": "https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers" }, + {"id": "googleother", + "n": "GoogleOther", "geo": "US", + "r": ["GoogleOther"], + "rx": ["\\sGoogleOther(\\-\\w+)?[\\)\\/]"], + "url": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers#googleother" + }, {"id": "applebot", "n": "Applebot", "geo": "US", "r": ["Applebot"], diff --git a/config/known-clients.json b/config/known-clients.json index d650d9f..f5bca20 100644 --- a/config/known-clients.json +++ b/config/known-clients.json @@ -23,6 +23,10 @@ "id": "huawei", "rx": [ "\\sHuaweiBrowser\\/(\\d+\\.\\d+)[\\s\\.]", "\\/harmony360Browser\\/(\\d+\\.\\d+)[\\s\\.]"] }, + {"n": "Ecosia Browser", + "id": "ecosia", + "rx": [ "\\(Ecosia ios@(\\d+)\\." ] + }, {"n": "Silk", "id": "silk", "rx": [ "\\Silk\\/(\\d+)\\." ] diff --git a/config/known-platforms.json b/config/known-platforms.json index a4a9ed5..817432e 100644 --- a/config/known-platforms.json +++ b/config/known-platforms.json @@ -25,11 +25,11 @@ }, {"n": "Android", "id": "android", - "rx": [ " Android[\\s;\\/](\\d\\d)\\.;\\s" ] + "rx": [ "Android[\\s;\\/](\\d\\d)[\\.;\\s]" ] }, {"n": "Old MacOS", "id": "macosold", - "rx": [ "\\sMac OS X 10[\\._](\\d|1[0-3])[\\._;\\s\\)]", "\\sMac OS X (1[123])[\\._]" ] + "rx": [ "\\sMac OS X 10[\\._](\\d|1[0-3])[\\._;\\s\\)]", "\\sMac OS X (1[123])[\\.;_\\s]" ] }, {"n": "MacOS", "id": "macos", diff --git a/config/user-config.json b/config/user-config.json index 739dc64..3f87f02 100644 --- a/config/user-config.json +++ b/config/user-config.json @@ -9,7 +9,7 @@ "id": "oldClient", "desc": "Obsolete browser version", "bot": 40 }, - {"func": "matchesPlatform", "params": ["winold", "macosold"], + {"func": "matchesPlatform", "params": ["winold", "macosold", "androidold"], "id": "oldOS", "desc": "Obsolete platform version", "bot": 40 }, @@ -17,6 +17,10 @@ "id": "serverOS", "desc": "Server OS", "bot": 40 }, + {"func": "matchesPlatform", "params": ["null"], + "id": "noOS", "desc": "Unknown or missing OS information", + "bot": 40 + }, {"func": "smallPageCount", "params": [1], "id": "onePage", "desc": "Visiter viewed only a single page", "bot": 40 @@ -77,12 +81,13 @@ "ipRanges": [ {"from": "3.0.0.0", "to": "3.255.255.254", "label": "Amazon Data Services [US]"}, {"from": "8.127.0.0", "to": "8.223.255.254", "label": "Alibaba [CN]"}, - {"from": "24.240.0.0", "to": "24.243.255.254", "label": "Charter [US]"}, + {"from": "24.240.0.0", "to": "24.247.255.255", "m": 13, "label": "Charter [US]"}, {"from": "27.106.0.0", "to": "27.106.127.254", "label": "Huawei [US]"}, {"from": "34.0.0.0", "to": "34.191.255.254", "label": "Google LLC"}, {"from": "45.0.0.0", "to": "45.255.255.254", "label": "Various small ISPs, mostly BR"}, {"from": "46.250.160.0", "to": "46.250.191.254", "label": "Huawei [MX]"}, {"from": "49.0.200.0", "to": "49.0.255.254", "label": "Huawei [SG]"}, + {"from": "66.249.64.0", "to": "66.249.95.255", "m": 19, "label": "Google LLC [US]"}, {"from": "84.37.35.0", "to": "84.37.255.254", "label": "GTT.net [US]"}, {"from": "94.74.64.0", "to": "94.74.127.254", "label": "Huawei [HK]"}, {"from": "101.0.0.0", "to": "101.255.255.254", "label": "ChinaNet [CN]"}, @@ -106,12 +111,14 @@ {"from": "188.0.0.0", "to": "188.255.255.254", "label": "South-American ISPs (188.x)"}, {"from": "189.0.0.0", "to": "189.255.255.254", "label": "South-American ISPs (189.x)"}, {"from": "190.0.0.0", "to": "190.255.255.254", "label": "South-American ISPs (190.x)"}, - {"from": "192.124.170.0", "to": "192.124.182.254", "label": "Relcom [CZ]"}, + {"from": "192.124.170.0", "to": "192.124.182.254", "label": "Relcom [CZ]"}, {"from": "195.37.0.0", "to": "195.37.255.255", "label": "DFN [DE]"}, - {"from": "2001:4800::::::", "to": "2001:4fff:ffff:ffff:ffff:ffff:ffff:ffff", "label": "Rackspace/Google [US]"}, - {"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "mask": 30, "label": "VNPT [VN]"}, - {"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "label": "Amazon Cloud [US]"}, - {"from": "2804:::::::", "to": "2804:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "label": "Inspire [BR]"}, + {"from": "2001:4860::::::", "to": "2001:4860:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Google LLC [US]"}, + {"from": "2001:0ee0::::::", "to": "2001:ee3:ffff:ffff:ffff:ffff:ffff:ffff", "m": 30, "label": "VNPT [VN]"}, + {"from": "2600:1f00::::::", "to": "2600:1fff:ffff:ffff:ffff:ffff:ffff:ffff", "m": "Amazon Cloud [US]"}, + {"from": "2603:6010::::::", "to": "2603:6010:ffff:ffff:ffff:ffff:ffff:ffff", "m": 32, "label": "Charter [US]"}, + {"from": "2603:8000::::::", "to": "2603:80ff:ffff:ffff:ffff:ffff:ffff:ffff", "m": 24, "label": "Charter [US]"}, + {"from": "2804:::::::", "to": "2804:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "m": 16, "label": "Inspire [BR]"}, {"from": "2a0a:4cc0::::::", "to": "2a0a:4cc0:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF", "label": "Netcup [DE]"} ] } \ No newline at end of file diff --git a/img/clients.png b/img/clients.png index fd9d06b..b087ff6 100644 Binary files a/img/clients.png and b/img/clients.png differ diff --git a/img/links.png b/img/links.png new file mode 100644 index 0000000..7fd9869 Binary files /dev/null and b/img/links.png differ diff --git a/script.js b/script.js index 1062364..b6e1592 100644 --- a/script.js +++ b/script.js @@ -107,7 +107,7 @@ const BotMon = { _formatTime: function(date) { if (date) { - return ('0'+date.getHours()).slice(-2) + ':' + ('0'+date.getMinutes()).slice(-2) + ':' + ('0'+date.getSeconds()).slice(-2); + return date.getHours() + ':' + ('0'+date.getMinutes()).slice(-2) + ':' + ('0'+date.getSeconds()).slice(-2); } else { return null; } @@ -433,7 +433,7 @@ BotMon.live = { // get the page view info: let pv = model._getPageView(visitor, dat); if (!pv) { - console.warn(`No page view for visit ID “${dat.id}”, page “${dat.pg}”, registering a new one.`); + console.info(`No page view for visit ID “${dat.id}”, page “${dat.pg}”, registering a new one.`); pv = model._makePageView(dat, type); visitor._pageViews.push(pv); } @@ -448,19 +448,21 @@ BotMon.live = { // helper function to create a new "page view" item: _makePageView: function(data, type) { + // console.info('_makePageView', data); // try to parse the referrer: let rUrl = null; try { rUrl = ( data.ref && data.ref !== '' ? new URL(data.ref) : null ); } catch (e) { - console.info(`Invalid referer: “${data.ref}”.`); + console.warn(`Invalid referer: “${data.ref}”.`); } return { _by: type, ip: data.ip, pg: data.pg, + lang: data.lang || '??', _ref: rUrl, _firstSeen: data.ts, _lastSeen: data.ts, @@ -1293,6 +1295,16 @@ BotMon.live = { return false; }, + // the "Accept language" header contains certain entries: + clientAccepts: function(visitor, ...languages) { + //console.info('clientAccepts', visitor.accept, languages); + + if (visitor.accept && languages) {; + return ( visitor.accept.split(',').filter(lang => languages.includes(lang)).length > 0 ); + } + return false; + }, + // Is there an accept-language field defined at all? noAcceptLang: function(visitor) { @@ -1886,7 +1898,21 @@ BotMon.live = { platformName + ( data._platform.v > 0 ? ' (' + data._platform.v + ')' : '' ) )); dl.appendChild(make('dt', {}, "IP-Address:")); - dl.appendChild(make('dd', {'class': 'has_icon ipaddr ip' + ipType}, data.ip)); + const ipItem = make('dd', {'class': 'has_icon ipaddr ip' + ipType}); + ipItem.appendChild(make('span', {'class': 'address'} , data.ip)); + ipItem.appendChild(make('a', { + 'class': 'icon_only extlink dnscheck', + 'href': `https://dnschecker.org/ip-location.php?ip=${encodeURIComponent(data.ip)}`, + 'target': 'dnscheck', + 'title': "View this address on DNSChecker.org" + } , "Check Address")); + ipItem.appendChild(make('a', { + 'class': 'icon_only extlink ipinfo', + 'href': `https://ipinfo.io/${encodeURIComponent(data.ip)}`, + 'target': 'ipinfo', + 'title': "View this address on IPInfo.io" + } , "DNS Info")); + dl.appendChild(ipItem); /*dl.appendChild(make('dt', {}, "ID:")); dl.appendChild(make('dd', {'class': 'has_icon ip' + data.typ}, data.id));*/ @@ -1906,7 +1932,7 @@ BotMon.live = { dl.appendChild(make('dd', {'class': 'agent'}, data.agent)); dl.appendChild(make('dt', {}, "Languages:")); - dl.appendChild(make('dd', {'class': 'langs'}, "Client accepts: [" + data.accept + "]; Page: [" + data.lang + ']')); + dl.appendChild(make('dd', {'class': 'langs'}, ` [${data.accept}]`)); if (data.geo && data.geo !=='') { dl.appendChild(make('dt', {}, "Location:")); @@ -1931,37 +1957,56 @@ BotMon.live = { const pageList = make('ul'); /* list all page views */ + data._pageViews.sort( (a, b) => a._firstSeen - b._firstSeen ); data._pageViews.forEach( (page) => { + //console.log("page:",page); + const pgLi = make('li'); - let visitTimeStr = "Bounce"; - const visitDuration = page._lastSeen.getTime() - page._firstSeen.getTime(); - if (visitDuration > 0) { - visitTimeStr = Math.floor(visitDuration / 1000) + "s"; - } + const lGroup = make('span'); // left group: - pgLi.appendChild(make('span', {}, page.pg)); /* DW Page ID */ - if (page._ref) { - pgLi.appendChild(make('span', { - 'data-ref': page._ref.host, - 'title': "Referrer: " + page._ref.full - }, page._ref.site)); - } else { - pgLi.appendChild(make('span', { - }, "No referer")); - } - pgLi.appendChild(make('span', {}, ( page._seenBy ? page._seenBy.join(', ') : '—') + '; ' + page._tickCount)); - pgLi.appendChild(make('span', {}, BotMon.t._formatTime(page._firstSeen))); + lGroup.appendChild(make('span', { + 'data-lang': page.lang, + 'title': "PageID: " + page.pg + }, page.pg)); /* DW Page ID */ + + pgLi.appendChild(lGroup); // end of left group + + const rGroup = make('span'); // right group: + + let visitTimeStr = "Bounce"; + const visitDuration = page._lastSeen.getTime() - page._firstSeen.getTime(); + if (visitDuration > 0) { + visitTimeStr = Math.floor(visitDuration / 1000) + "s"; + } + + /*if (page._ref) { + rGroup.appendChild(make('span', { + 'data-ref': page._ref.host, + 'title': "Referrer: " + page._ref.full + }, page._ref.site)); + } else { + rGroup.appendChild(make('span', { + }, "No referer")); + }*/ + //rGroup.appendChild(make('span', {}, ( page._seenBy ? page._seenBy.join(', ') : '—') + '; ' + page._tickCount)); + + // get the time difference: + const tDiff = BotMon.t._formatTimeDiff(page._firstSeen, page._lastSeen); + if (tDiff) { + rGroup.appendChild(make('span', {'class': 'visit-length', 'title': 'Last seen: ' + page._lastSeen.toLocaleString()}, tDiff)); + } else { + rGroup.appendChild(make('span', { + 'class': 'bounce', + 'title': "Visitor bounced"}, "Bounce")); + } + rGroup.appendChild(make('span', { + 'class': 'first-seen', + 'title': "First visited: " + page._firstSeen.toLocaleString() + }, BotMon.t._formatTime(page._firstSeen))); + + pgLi.appendChild(rGroup); // end of right group - // get the time difference: - const tDiff = BotMon.t._formatTimeDiff(page._firstSeen, page._lastSeen); - if (tDiff) { - pgLi.appendChild(make('span', {'class': 'visit-length', 'title': 'Last seen: ' + page._lastSeen.toLocaleString()}, tDiff)); - } else { - pgLi.appendChild(make('span', { - 'class': 'bounce', - 'title': "Visitor bounced"}, "Bounce")); - } pageList.appendChild(pgLi); }); diff --git a/style.less b/style.less index 4108677..4f5fede 100644 --- a/style.less +++ b/style.less @@ -40,7 +40,7 @@ /* Bot icons */ &.bot::before { background-image: url('img/bots.png') } - &.bot_googlebot::before, &.bot_googleads::before, &.bot_googleapi::before { background-position-y: -20px } + &.bot_googlebot::before, &.bot_googleads::before, &.bot_googleapi::before, &.bot_googleother::before { background-position-y: -20px } &.bot_bingbot::before { background-position-y: -40px } &.bot_applebot::before { background-position-y: -60px } &.bot_openai::before { background-position-y: -80px } @@ -84,6 +84,7 @@ &.cl_silk::before { background-position-y: -280px } &.cl_ffold::before { background-position-y: -300px } &.cl_chromeold::before { background-position-y: -320px } + &.cl_ecosia::before { background-position-y: -340px } /* Country flags */ /* Note: flag images and CSS adapted from: https://github.com/lafeber/world-flags-sprite/ */ @@ -332,6 +333,11 @@ &.typ_php::before { background-position-y: -40px } &.typ_ip::before { background-position-y: -60px } &.typ_usr::before { background-position-y: -80px } + + /* External link icons */ + &.extlink::before { background-image: url('img/links.png') } + &.extlink.dnscheck::before { background-position-y: -20px } + &.extlink.ipinfo::before { background-position-y: -40px } } /* grid layout for the overview: */ @@ -524,26 +530,49 @@ & { display: flex; justify-content: space-between; - align-items: center; + align-items: baseline; + white-space: nowrap; + line-height: 1.2rem; + margin: 0; + padding: 0 .25em; + } + &:nth-child(odd) { + background-color: #DFDFDF; } span { - &.visit-length { - min-width: min-content; - } - &.bounce { - width: 1.25em; height: 1.25em; - overflow: hidden; - } - &.bounce::before { - display: inline-block; - content: ''; - width: 1.25em; height: 1.25em; - background: transparent url('img/bounce.svg') center no-repeat; - background-size: 1.25em; - } + display: inline-block; } } } + span[data-lang] { + overflow: hidden; + text-overflow: ellipsis; + } + span[data-lang]::after { + content: attr(data-lang); + font-size: smaller; + color: #555; + border: #555 solid 1px; + line-height: 1.25; + border-radius: 2pt; + padding: 0 1pt; + margin-left: .2em; + } + span.first-seen { + min-width: 4.2em; + text-align: right;; + } + span.bounce { + width: 1.25em; height: 1.25em; + overflow: hidden; + } + span.bounce::before { + display: inline-block; + content: ''; + width: 1.25em; height: 1.25em; + background: transparent url('img/bounce.svg') center no-repeat; + background-size: 1.25em; + } } } @@ -554,7 +583,7 @@ align-items: center; } li:nth-child(odd) { - background-color: #EEE; + background-color: #DFDFDF; } li.total { border-top: #333 solid 1px; diff --git a/tick.php b/tick.php index 1d46b1e..67b4550 100644 --- a/tick.php +++ b/tick.php @@ -2,6 +2,12 @@ // Note: this script is normally called in HEAD mode, therefore it can not return any payload. + // quit out if it is called without athe right parameters: + if (!isset($_GET['id']) || !isset($_GET['p'])) { + http_response_code(400); + die("Parameter error."); + } + // what is the session identifier? $sessionId = preg_replace('/[\x00-\x1F{};\"\']/', "\u{FFFD}", $_GET['id']) /* clean json parameter */ ?? session_id()