Merge pull request #24 from shizunge/offline

set location to unknown when city is not available.

.github/workflows/docker-hub-build-push-on-release.yml

@@ -24,26 +24,26 @@ jobs:
           echo "Action image_tag=${RELEASE_VERSION}"
       
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
         
       - name: Install buildx
-        uses: docker/setup-buildx-action@v1.6.0
+        uses: docker/setup-buildx-action@v2.0.0
         
       - name: Login to docker hub
-        uses: docker/login-action@v1.14.1
+        uses: docker/login-action@v2.0.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1.14.1
+        uses: docker/login-action@v2.0.0
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ github.token }}
         
       - name: Build and push ${{ github.repository }}:${{ steps.git.outputs.image_tag }}
-        uses: docker/build-push-action@v2.9.0
+        uses: docker/build-push-action@v3.0.0
         with:
           platforms: ${{ env.PLATFORMS }}
           push: true
@@ -54,7 +54,7 @@ jobs:
             ghcr.io/${{ github.repository }}:latest
 
       - name: Update dockerhub description
-        uses: peter-evans/dockerhub-description@v2.4.3
+        uses: peter-evans/dockerhub-description@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}

README.md

@@ -39,42 +39,47 @@ Also check out [examples](./examples/README.md) for the setup of the full stack.
 
 ## Usage
 
-Usage of `./endlessh-go`
+`./endlessh-go --help`
 
-* -alsologtostderr
-  * log to standard error as well as files
-* -conn_type string
-  * Connection type. Possible values are tcp, tcp4, tcp6 (default "tcp")
-* -enable_prometheus
-  * Enable prometheus
-* -geoip_supplier string
-  * Supplier to obtain Geohash of IPs. Possible values are "off", "ip-api", "freegeoip" (default "off")
-* -host string
-  * Listening address (default "0.0.0.0")
-* -interval_ms int
-  * Message millisecond delay (default 1000)
-* -line_length int
-  * Maximum banner line length (default 32)
-* -log_backtrace_at value
-  * when logging hits line file:N, emit a stack trace
-* -log_dir string
-  * If non-empty, write log files in this directory
-* -logtostderr
-  * log to standard error instead of files
-* -max_clients int
-  * Maximum number of clients (default 4096)
-* -port string
-  * Listening port (default "2222")
-* -prometheus_entry string
-  * Entry point for prometheus (default "metrics")
-* -prometheus_port string
-  * The port for prometheus (default "2112")
-* -stderrthreshold value
-  * logs at or above this threshold go to stderr
-* -v value
-  * log level for V logs
-* -vmodule value
-  * comma-separated list of pattern=N settings for file-filtered logging
+```
+Usage of ./endlessh-go
+  -alsologtostderr
+        log to standard error as well as files
+  -conn_type string
+        Connection type. Possible values are tcp, tcp4, tcp6 (default "tcp")
+  -enable_prometheus
+        Enable prometheus
+  -geoip_supplier string
+        Supplier to obtain Geohash of IPs. Possible values are "off", "ip-api", "freegeoip", "max-mind-db" (default "off")
+  -host string
+        Listening address (default "0.0.0.0")
+  -interval_ms int
+        Message millisecond delay (default 1000)
+  -line_length int
+        Maximum banner line length (default 32)
+  -log_backtrace_at value
+        when logging hits line file:N, emit a stack trace
+  -log_dir string
+        If non-empty, write log files in this directory
+  -logtostderr
+        log to standard error instead of files
+  -max_clients int
+        Maximum number of clients (default 4096)
+  -max_mind_db string
+        Path to the MaxMind DB file.
+  -port string
+        Listening port (default "2222")
+  -prometheus_entry string
+        Entry point for prometheus (default "metrics")
+  -prometheus_port string
+        The port for prometheus (default "2112")
+  -stderrthreshold value
+        logs at or above this threshold go to stderr
+  -v value
+        log level for V logs
+  -vmodule value
+        comma-separated list of pattern=N settings for file-filtered logging
+```
 
 ## Metrics
 
@@ -95,6 +100,8 @@ It listens to port `2112` and entry point is `/metrics` by default. The port and
 
 The endlessh-go server stores the geohash of attackers as a label on `endlessh_client_open_count`, which is also off by default. You can turn it on via the CLI argument `-geoip_supplier`. The endlessh-go uses service from either [ip-api](https://ip-api.com/) or [freegeoip](https://freegeoip.live/), which may enforce a query rate and limit commercial use. Visit their website for their terms and policies.
 
+You could also use an offline GeoIP database from [MaxMind](https://www.maxmind.com) by setting `-geoip_supplier` to *max-mind-db* and `-max_mind_db` to the path of the database file.
+
 ## Dashboard
 
 The dashboard requires Grafana 8.2.

geoip.go

@@ -20,12 +20,39 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"strings"
 
+	"github.com/oschwald/geoip2-golang"
 	"github.com/pierrre/geohash"
 )
 
+var (
+	maxMindDbFileName *string
+)
+
+func composeLocation(country string, region string, city string) string {
+	var locations []string
+	for _, s := range []string{country, region, city} {
+		if strings.TrimSpace(s) != "" {
+			locations = append(locations, s)
+		}
+	}
+	location := strings.Join(locations, ", ")
+	if location == "" {
+		return "Unknown"
+	}
+	return location
+}
+
+func composeCountry(country string) string {
+	if country == "" {
+		return "Unknown"
+	}
+	return country
+}
+
 type freegeoip struct {
 	Ip          string  `json:"ip"`
 	CountryCode string  `json:"country_code"`
@@ -58,21 +85,9 @@ func geohashAndLocationFromFreegeoip(address string) (string, string, string, er
 		return "s000", "Unknown", "Unknown", err
 	}
 
-	var locations []string
-	for _, s := range []string{geo.CountryName, geo.RegionName, geo.City} {
-		if strings.TrimSpace(s) != "" {
-			locations = append(locations, s)
-		}
-	}
-	location := strings.Join(locations, ", ")
-	if location == "" {
-		location = "Unknown"
-	}
-	country := geo.CountryName
-	if country == "" {
-		country = "Unknown"
-	}
 	gh := geohash.EncodeAuto(geo.Latitude, geo.Longitude)
+	country := composeCountry(geo.CountryName)
+	location := composeLocation(geo.CountryName, geo.RegionName, geo.City)
 
 	return gh, country, location, nil
 }
@@ -113,21 +128,28 @@ func geohashAndLocationFromIpapi(address string) (string, string, string, error)
 		return "s000", "Unknown", "Unknown", fmt.Errorf("failed to query %v via ip-api: status: %v, message: %v", address, geo.Status, geo.Message)
 	}
 
-	var locations []string
-	for _, s := range []string{geo.CountryName, geo.RegionName, geo.City} {
-		if strings.TrimSpace(s) != "" {
-			locations = append(locations, s)
-		}
-	}
-	location := strings.Join(locations, ", ")
-	if location == "" {
-		location = "Unknown"
-	}
-	country := geo.CountryName
-	if country == "" {
-		country = "Unknown"
-	}
 	gh := geohash.EncodeAuto(geo.Latitude, geo.Longitude)
+	country := composeCountry(geo.CountryName)
+	location := composeLocation(geo.CountryName, geo.RegionName, geo.City)
+
+	return gh, country, location, nil
+}
+
+func geohashAndLocationFromMaxMindDb(address string) (string, string, string, error) {
+	db, err := geoip2.Open(*maxMindDbFileName)
+	if err != nil {
+		return "s000", "Unknown", "Unknown", err
+	}
+	defer db.Close()
+	// If you are using strings that may be invalid, check that ip is not nil
+	ip := net.ParseIP(address)
+	record, err := db.City(ip)
+	if err != nil {
+		return "s000", "Unknown", "Unknown", err
+	}
+	gh := geohash.EncodeAuto(record.Location.Latitude, record.Location.Longitude)
+	country := composeCountry(record.Country.Names["en"])
+	location := composeLocation(record.Country.Names["en"], "", record.City.Names["en"])
 
 	return gh, country, location, nil
 }
@@ -140,6 +162,8 @@ func geohashAndLocation(address string, geoipSupplier string) (string, string, s
 		return geohashAndLocationFromIpapi(address)
 	case "freegeoip":
 		return geohashAndLocationFromFreegeoip(address)
+	case "max-mind-db":
+		return geohashAndLocationFromMaxMindDb(address)
 	default:
 		return "s000", "Unknown", "Unknown", fmt.Errorf("unknown geoipSupplier %v.", geoipSupplier)
 	}

go.mod

@@ -1,11 +1,12 @@
 module endlessh-go
 
-go 1.17
+go 1.18
 
 require (
 	github.com/golang/glog v1.0.0
+	github.com/oschwald/geoip2-golang v1.7.0
 	github.com/pierrre/geohash v1.0.0
-	github.com/prometheus/client_golang v1.12.1
+	github.com/prometheus/client_golang v1.12.2
 )
 
 require (
@@ -13,9 +14,10 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/oschwald/maxminddb-golang v1.9.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
-	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
+	golang.org/x/sys v0.0.0-20220325203850-36772127a21f // indirect
 	google.golang.org/protobuf v1.26.0 // indirect
 )

go.sum

@@ -161,6 +161,10 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/oschwald/geoip2-golang v1.7.0 h1:JW1r5AKi+vv2ujSxjKthySK3jo8w8oKWPyXsw+Qs/S8=
+github.com/oschwald/geoip2-golang v1.7.0/go.mod h1:mdI/C7iK7NVMcIDDtf4bCKMJ7r0o7UwGeCo9eiitCMQ=
+github.com/oschwald/maxminddb-golang v1.9.0 h1:tIk4nv6VT9OiPyrnDAfJS1s1xKDQMZOsGojab6EjC1Y=
+github.com/oschwald/maxminddb-golang v1.9.0/go.mod h1:TK+s/Z2oZq0rSl4PSeAEoP0bgm82Cp5HyvYbt8K3zLY=
 github.com/pierrre/compare v1.0.2 h1:k4IUsHgh+dbcAOIWCfxVa/7G6STjADH2qmhomv+1quc=
 github.com/pierrre/compare v1.0.2/go.mod h1:8UvyRHH+9HS8Pczdd2z5x/wvv67krDwVxoOndaIIDVU=
 github.com/pierrre/geohash v1.0.0 h1:f/zfjdV4rVofTCz1FhP07T+EMQAvcMM2ioGZVt+zqjI=
@@ -173,8 +177,8 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
-github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
+github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
+github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -327,6 +331,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220325203850-36772127a21f h1:TrmogKRsSOxRMJbLYGrB4SBbW+LJcEllYBLME5Zk5pU=
+golang.org/x/sys v0.0.0-20220325203850-36772127a21f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

main.go

@@ -114,7 +114,8 @@ func main() {
 	prometheusEnabled := flag.Bool("enable_prometheus", false, "Enable prometheus")
 	prometheusPort := flag.String("prometheus_port", "2112", "The port for prometheus")
 	prometheusEntry := flag.String("prometheus_entry", "metrics", "Entry point for prometheus")
-	geoipSupplier := flag.String("geoip_supplier", "off", "Supplier to obtain Geohash of IPs. Possible values are \"off\", \"ip-api\", \"freegeoip\"")
+	geoipSupplier := flag.String("geoip_supplier", "off", "Supplier to obtain Geohash of IPs. Possible values are \"off\", \"ip-api\", \"freegeoip\", \"max-mind-db\"")
+	maxMindDbFileName = flag.String("max_mind_db", "", "Path to the MaxMind DB file.")
 
 	flag.Usage = func() {
 		fmt.Fprintf(flag.CommandLine.Output(), "Usage of %v \n", os.Args[0])