diff --git a/core/endpoint.go b/core/endpoint.go index 6fe3c90e..d28a1381 100644 --- a/core/endpoint.go +++ b/core/endpoint.go @@ -66,6 +66,8 @@ var ( // This is because the free whois service we are using should not be abused, especially considering the fact that // the data takes a while to be updated. ErrInvalidEndpointIntervalForDomainExpirationPlaceholder = errors.New("the minimum interval for an endpoint with a condition using the " + DomainExpirationPlaceholder + " placeholder is 300s (5m)") + + whoisClient = whois.NewClient().WithReferralCache(true) ) // Endpoint is the configuration of a monitored @@ -255,11 +257,6 @@ func (endpoint *Endpoint) EvaluateHealth() *Result { if endpoint.needsToRetrieveIP() { endpoint.getIP(result) } - // Retrieve domain expiration if necessary - if endpoint.needsToRetrieveDomainExpiration() && len(result.Hostname) > 0 { - endpoint.getDomainExpiration(result) - } - // if len(result.Errors) == 0 { endpoint.call(result) } else { @@ -272,6 +269,10 @@ func (endpoint *Endpoint) EvaluateHealth() *Result { } } result.Timestamp = time.Now() + // Retrieve domain expiration if necessary + if endpoint.needsToRetrieveDomainExpiration() && len(result.Hostname) > 0 { + endpoint.getDomainExpiration(result) + } // No need to keep the body after the endpoint has been evaluated result.body = nil // Clean up parameters that we don't need to keep in the results @@ -299,7 +300,6 @@ func (endpoint *Endpoint) getIP(result *Result) { } func (endpoint *Endpoint) getDomainExpiration(result *Result) { - whoisClient := whois.NewClient() if whoisResponse, err := whoisClient.QueryAndParse(result.Hostname); err != nil { result.AddError("error querying and parsing hostname using whois client: " + err.Error()) } else { diff --git a/go.mod b/go.mod index dab25be7..2ae1c65f 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/TwiN/g8 v1.4.0 github.com/TwiN/gocache/v2 v2.1.1 github.com/TwiN/health v1.5.0 - github.com/TwiN/whois v1.0.0 + github.com/TwiN/whois v1.1.0 github.com/coreos/go-oidc/v3 v3.1.0 github.com/go-ping/ping v0.0.0-20210911151512-381826476871 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index 26b707cd..e0ef3089 100644 --- a/go.sum +++ b/go.sum @@ -39,8 +39,8 @@ github.com/TwiN/gocache/v2 v2.1.1 h1:W/GLImqa+pZVIH9pcWEn1cBgy1KU66fUcBjOnPhjuno github.com/TwiN/gocache/v2 v2.1.1/go.mod h1:SnUuBsrwGQeNcDG6vhkOMJnqErZM0JGjgIkuKryokYA= github.com/TwiN/health v1.5.0 h1:ETTtbQfUbiiIiVTSpAiNzesHQvm8qarV/8ctlZsVhwA= github.com/TwiN/health v1.5.0/go.mod h1:Z6TszwQPMvtSiVx1QMidVRgvVr4KZGfiwqcD7/Z+3iw= -github.com/TwiN/whois v1.0.0 h1:I+aQzXLPmhWovkFUzlPV2DdfLZUWDLrkMDlM6QwCv+Q= -github.com/TwiN/whois v1.0.0/go.mod h1:9WbCzYlR+r5eq9vbgJVh7A4H2uR2ct4wKEB0/QITJ/c= +github.com/TwiN/whois v1.1.0 h1:lhyrC/9yIXntEnbJ+0IBy9Z5NBcreieYyamlvniwq88= +github.com/TwiN/whois v1.1.0/go.mod h1:9WbCzYlR+r5eq9vbgJVh7A4H2uR2ct4wKEB0/QITJ/c= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -238,7 +238,6 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -284,7 +283,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -386,7 +384,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/vendor/github.com/TwiN/whois/README.md b/vendor/github.com/TwiN/whois/README.md index fe3c61f1..59859c11 100644 --- a/vendor/github.com/TwiN/whois/README.md +++ b/vendor/github.com/TwiN/whois/README.md @@ -63,3 +63,19 @@ Currently, the only fields parsed are: - `NameServers`: The nameservers currently tied to the domain If you'd like one or more other fields to be parsed, please don't be shy and create an issue or a pull request. + +#### Caching referral WHOIS servers +The way that WHOIS scales is by having one "main" WHOIS server, namely `whois.iana.org:43`, refer to other WHOIS server +on a per-TLD basis. + +In other word, let's say that you wanted to have the WHOIS information for `example.com`. +The first step would be to query `whois.iana.org:43` with `com`, which would return `whois.verisign-grs.com`. +Then, you would query `whois.verisign-grs.com:43` for the WHOIS information on `example.com`. + +If you're querying a lot of servers, making two queries instead of one can be a little wasteful, hence `WithReferralCache(true)`: +```go +client := whois.NewClient().WithReferralCache(true) +``` +The above will cache the referral WHOIS server for each TLD, so that you can directly query the appropriate WHOIS server +instead of first querying `whois.iana.org:43` for the referral. + diff --git a/vendor/github.com/TwiN/whois/whois.go b/vendor/github.com/TwiN/whois/whois.go index 2f7b8ee9..90d8f3b4 100644 --- a/vendor/github.com/TwiN/whois/whois.go +++ b/vendor/github.com/TwiN/whois/whois.go @@ -13,16 +13,49 @@ const ( type Client struct { whoisServerAddress string + + isCachingReferralWHOISServers bool + referralWHOISServersCache map[string]string } func NewClient() *Client { return &Client{ - whoisServerAddress: ianaWHOISServerAddress, + whoisServerAddress: ianaWHOISServerAddress, + referralWHOISServersCache: make(map[string]string), } } -func (c Client) Query(domain string) (string, error) { +// WithReferralCache allows you to enable or disable the referral WHOIS server cache. +// While ianaWHOISServerAddress is the "entry point" for WHOIS queries, it sometimes has +// availability issues. One way to mitigate this is to cache the referral WHOIS server. +// +// This is disabled by default +func (c *Client) WithReferralCache(enabled bool) *Client { + c.isCachingReferralWHOISServers = enabled + if enabled { + // We'll set a couple of common ones right away to avoid unnecessary queries + c.referralWHOISServersCache = map[string]string{ + "com": "whois.verisign-grs.com", + "black": "whois.nic.black", + "dev": "whois.nic.google", + "green": "whois.nic.green", + "io": "whois.nic.io", + "net": "whois.verisign-grs.com", + "org": "whois.publicinterestregistry.org", + "red": "whois.nic.red", + "sh": "whois.nic.sh", + } + } + return c +} + +func (c *Client) Query(domain string) (string, error) { parts := strings.Split(domain, ".") + if c.isCachingReferralWHOISServers { + if cachedWHOISServer, ok := c.referralWHOISServersCache[domain]; ok { + return c.query(cachedWHOISServer, domain) + } + } output, err := c.query(c.whoisServerAddress, parts[len(parts)-1]) if err != nil { return "", err @@ -32,6 +65,9 @@ func (c Client) Query(domain string) (string, error) { endIndex := strings.Index(output[startIndex:], "\n") + startIndex whois := strings.TrimSpace(output[startIndex:endIndex]) if referOutput, err := c.query(whois+":43", domain); err == nil { + if c.isCachingReferralWHOISServers { + c.referralWHOISServersCache[domain] = whois + ":43" + } return referOutput, nil } return "", err @@ -45,7 +81,7 @@ func (c Client) query(whoisServerAddress, domain string) (string, error) { return "", err } defer connection.Close() - connection.SetDeadline(time.Now().Add(5 * time.Second)) + _ = connection.SetDeadline(time.Now().Add(5 * time.Second)) _, err = connection.Write([]byte(domain + "\r\n")) if err != nil { return "", err diff --git a/vendor/modules.txt b/vendor/modules.txt index a7747509..7f255f6e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -7,7 +7,7 @@ github.com/TwiN/gocache/v2 # github.com/TwiN/health v1.5.0 ## explicit; go 1.19 github.com/TwiN/health -# github.com/TwiN/whois v1.0.0 +# github.com/TwiN/whois v1.1.0 ## explicit; go 1.19 github.com/TwiN/whois # github.com/beorn7/perks v1.0.1