add support to read max mind db

2022-07-09 12:20:55 -07:00
parent 83c285c50f
commit 0f9201e8c2
5 changed files with 80 additions and 38 deletions
--- a/README.md
+++ b/README.md
@@ -39,42 +39,47 @@ Also check out [examples](./examples/README.md) for the setup of the full stack.

 ## Usage

-Usage of `./endlessh-go`
+`./endlessh-go --help`

-* -alsologtostderr
-  * log to standard error as well as files
-* -conn_type string
-  * Connection type. Possible values are tcp, tcp4, tcp6 (default "tcp")
-* -enable_prometheus
-  * Enable prometheus
-* -geoip_supplier string
-  * Supplier to obtain Geohash of IPs. Possible values are "off", "ip-api", "freegeoip" (default "off")
-* -host string
-  * Listening address (default "0.0.0.0")
-* -interval_ms int
-  * Message millisecond delay (default 1000)
-* -line_length int
-  * Maximum banner line length (default 32)
-* -log_backtrace_at value
-  * when logging hits line file:N, emit a stack trace
-* -log_dir string
-  * If non-empty, write log files in this directory
-* -logtostderr
-  * log to standard error instead of files
-* -max_clients int
-  * Maximum number of clients (default 4096)
-* -port string
-  * Listening port (default "2222")
-* -prometheus_entry string
-  * Entry point for prometheus (default "metrics")
-* -prometheus_port string
-  * The port for prometheus (default "2112")
-* -stderrthreshold value
-  * logs at or above this threshold go to stderr
-* -v value
-  * log level for V logs
-* -vmodule value
-  * comma-separated list of pattern=N settings for file-filtered logging
+```
+Usage of ./endlessh-go
+  -alsologtostderr
+        log to standard error as well as files
+  -conn_type string
+        Connection type. Possible values are tcp, tcp4, tcp6 (default "tcp")
+  -enable_prometheus
+        Enable prometheus
+  -geoip_supplier string
+        Supplier to obtain Geohash of IPs. Possible values are "off", "ip-api", "freegeoip", "max-mind-db" (default "off")
+  -host string
+        Listening address (default "0.0.0.0")
+  -interval_ms int
+        Message millisecond delay (default 1000)
+  -line_length int
+        Maximum banner line length (default 32)
+  -log_backtrace_at value
+        when logging hits line file:N, emit a stack trace
+  -log_dir string
+        If non-empty, write log files in this directory
+  -logtostderr
+        log to standard error instead of files
+  -max_clients int
+        Maximum number of clients (default 4096)
+  -max_mind_db string
+        Path to the MaxMind DB file.
+  -port string
+        Listening port (default "2222")
+  -prometheus_entry string
+        Entry point for prometheus (default "metrics")
+  -prometheus_port string
+        The port for prometheus (default "2112")
+  -stderrthreshold value
+        logs at or above this threshold go to stderr
+  -v value
+        log level for V logs
+  -vmodule value
+        comma-separated list of pattern=N settings for file-filtered logging
+```

 ## Metrics

@@ -95,6 +100,8 @@ It listens to port `2112` and entry point is `/metrics` by default. The port and

 The endlessh-go server stores the geohash of attackers as a label on `endlessh_client_open_count`, which is also off by default. You can turn it on via the CLI argument `-geoip_supplier`. The endlessh-go uses service from either [ip-api](https://ip-api.com/) or [freegeoip](https://freegeoip.live/), which may enforce a query rate and limit commercial use. Visit their website for their terms and policies.

+You could also use an offline GeoIP database from [MaxMind](https://www.maxmind.com) by setting `-geoip_supplier` to *max-mind-db* and `-max_mind_db` to the path of the database file.
+
 ## Dashboard

 The dashboard requires Grafana 8.2.
--- a/geoip.go
+++ b/geoip.go
@@ -20,12 +20,18 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"strings"

+	"github.com/oschwald/geoip2-golang"
 	"github.com/pierrre/geohash"
 )

+var (
+	maxMindDbFileName *string
+)
+
 type freegeoip struct {
 	Ip          string  `json:"ip"`
 	CountryCode string  `json:"country_code"`
@@ -132,6 +138,24 @@ func geohashAndLocationFromIpapi(address string) (string, string, string, error)
 	return gh, country, location, nil
 }

+func geohashAndLocationFromMaxMindDb(address string) (string, string, string, error) {
+	db, err := geoip2.Open(*maxMindDbFileName)
+	if err != nil {
+		return "s000", "Unknown", "Unknown", err
+	}
+	defer db.Close()
+	// If you are using strings that may be invalid, check that ip is not nil
+	ip := net.ParseIP(address)
+	record, err := db.City(ip)
+	if err != nil {
+		return "s000", "Unknown", "Unknown", err
+	}
+	gh := geohash.EncodeAuto(record.Location.Latitude, record.Location.Longitude)
+	country := record.Country.Names["en"]
+	location := record.City.Names["en"]
+	return gh, country, location, nil
+}
+
 func geohashAndLocation(address string, geoipSupplier string) (string, string, string, error) {
 	switch geoipSupplier {
 	case "off":
@@ -140,6 +164,8 @@ func geohashAndLocation(address string, geoipSupplier string) (string, string, s
 		return geohashAndLocationFromIpapi(address)
 	case "freegeoip":
 		return geohashAndLocationFromFreegeoip(address)
+	case "max-mind-db":
+		return geohashAndLocationFromMaxMindDb(address)
 	default:
 		return "s000", "Unknown", "Unknown", fmt.Errorf("unknown geoipSupplier %v.", geoipSupplier)
 	}
--- a/go.mod
+++ b/go.mod
@@ -1,9 +1,10 @@
 module endlessh-go

-go 1.17
+go 1.18

 require (
 	github.com/golang/glog v1.0.0
+	github.com/oschwald/geoip2-golang v1.7.0
 	github.com/pierrre/geohash v1.0.0
 	github.com/prometheus/client_golang v1.12.2
 )
@@ -13,9 +14,10 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/oschwald/maxminddb-golang v1.9.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
-	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
+	golang.org/x/sys v0.0.0-20220325203850-36772127a21f // indirect
 	google.golang.org/protobuf v1.26.0 // indirect
 )
--- a/go.sum
+++ b/go.sum
@@ -161,6 +161,10 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/oschwald/geoip2-golang v1.7.0 h1:JW1r5AKi+vv2ujSxjKthySK3jo8w8oKWPyXsw+Qs/S8=
+github.com/oschwald/geoip2-golang v1.7.0/go.mod h1:mdI/C7iK7NVMcIDDtf4bCKMJ7r0o7UwGeCo9eiitCMQ=
+github.com/oschwald/maxminddb-golang v1.9.0 h1:tIk4nv6VT9OiPyrnDAfJS1s1xKDQMZOsGojab6EjC1Y=
+github.com/oschwald/maxminddb-golang v1.9.0/go.mod h1:TK+s/Z2oZq0rSl4PSeAEoP0bgm82Cp5HyvYbt8K3zLY=
 github.com/pierrre/compare v1.0.2 h1:k4IUsHgh+dbcAOIWCfxVa/7G6STjADH2qmhomv+1quc=
 github.com/pierrre/compare v1.0.2/go.mod h1:8UvyRHH+9HS8Pczdd2z5x/wvv67krDwVxoOndaIIDVU=
 github.com/pierrre/geohash v1.0.0 h1:f/zfjdV4rVofTCz1FhP07T+EMQAvcMM2ioGZVt+zqjI=
@@ -327,6 +331,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220325203850-36772127a21f h1:TrmogKRsSOxRMJbLYGrB4SBbW+LJcEllYBLME5Zk5pU=
+golang.org/x/sys v0.0.0-20220325203850-36772127a21f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
--- a/main.go
+++ b/main.go
@@ -114,7 +114,8 @@ func main() {
 	prometheusEnabled := flag.Bool("enable_prometheus", false, "Enable prometheus")
 	prometheusPort := flag.String("prometheus_port", "2112", "The port for prometheus")
 	prometheusEntry := flag.String("prometheus_entry", "metrics", "Entry point for prometheus")
-	geoipSupplier := flag.String("geoip_supplier", "off", "Supplier to obtain Geohash of IPs. Possible values are \"off\", \"ip-api\", \"freegeoip\"")
+	geoipSupplier := flag.String("geoip_supplier", "off", "Supplier to obtain Geohash of IPs. Possible values are \"off\", \"ip-api\", \"freegeoip\", \"max-mind-db\"")
+	maxMindDbFileName = flag.String("max_mind_db", "", "Path to the MaxMind DB file.")

 	flag.Usage = func() {
 		fmt.Fprintf(flag.CommandLine.Output(), "Usage of %v \n", os.Args[0])