extern/wttr.in
1
0
Fork 0
mirror of https://github.com/chubin/wttr.in.git synced 2024-10-05 01:42:07 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add geoloc cache converter

Browse Source
This commit is contained in:
Igor Chubin 2022-12-11 09:36:11 +01:00
parent 1510ce6a88
commit 1bcdd45f34
9 changed files with 77 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -15,6 +15,7 @@ require (
github.com/smartystreets/assertions v1.2.0 // indirect
github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/stretchr/testify v1.8.1 // indirect
github.com/zsefvlol/timezonemapper v1.0.0 // indirect
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

2
go.sum
View File

@ -38,6 +38,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/zsefvlol/timezonemapper v1.0.0 h1:HXqkOzf01gXYh2nDQcDSROikFgMaximnhE8BY9SyF6E=
github.com/zsefvlol/timezonemapper v1.0.0/go.mod h1:cVUCOLEmc/VvOMusEhpd2G/UBtadL26ZVz2syODXDoQ=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=

19
internal/config/config.go
View File

@ -77,7 +77,15 @@ type Geo struct {
// IPCacheDB contains the path to the SQLite DB with the IP Geodata cache.
IPCacheDB string `yaml:"ipCacheDB,omitempty"`
CacheType types.CacheType `yaml:"cacheType,omitempty"`
IPCacheType types.CacheType `yaml:"cacheType,omitempty"`
// LocationCache contains the path to the Location Geodata cache.
LocationCache string `yaml:"locationCache,omitempty"`
// LocationCacheDB contains the path to the SQLite DB with the Location Geodata cache.
LocationCacheDB string `yaml:"locationCacheDB,omitempty"`
LocationCacheType types.CacheType `yaml:"locationCacheType,omitempty"`
Nominatim []Nominatim
}
@ -97,9 +105,12 @@ func Default() *Config {
Size: 12800,
},
Geo{
IPCache: "/wttr.in/cache/ip2l",
IPCacheDB: "/wttr.in/cache/geoip.db",
CacheType: types.CacheTypeDB,
IPCache: "/wttr.in/cache/ip2l",
IPCacheDB: "/wttr.in/cache/geoip.db",
IPCacheType: types.CacheTypeDB,
LocationCache: "/wttr.in/cache/loc",
LocationCacheDB: "/wttr.in/cache/geoloc.db",
LocationCacheType: types.CacheTypeFiles,
Nominatim: []Nominatim{
{
Name: "locationiq",

31
internal/geo/ip/convert.go
View File

@ -3,17 +3,18 @@ package ip
import (
"fmt"
"log"
"os"
"path/filepath"
"github.com/samonzeweb/godb"
"github.com/samonzeweb/godb/adapters/sqlite"
"github.com/chubin/wttr.in/internal/util"
)
func (c *Cache) ConvertCache() error {
dbfile := c.config.Geo.IPCacheDB
err := removeDBIfExists(dbfile)
err := util.RemoveFileIfExists(dbfile)
if err != nil {
return err
}
@ -29,7 +30,7 @@ func (c *Cache) ConvertCache() error {
}
log.Println("listing cache entries...")
files, err := filepath.Glob(filepath.Join(c.config.Geo.IPCache, "*"))
files, err := filepath.Glob(filepath.Join(c.config.Geo.LocationCache, "*"))
if err != nil {
return err
}
@ -72,28 +73,12 @@ func (c *Cache) ConvertCache() error {
func createTable(db *godb.DB, tableName string) error {
createTable := fmt.Sprintf(
`create table %s (
ip text not null primary key,
countryCode text not null,
country text not null,
region text not null,
city text not null,
latitude text not null,
longitude text not null);
name text not null primary key,
fullName text not null,
lat text not null,
long text not null);
`, tableName)
_, err := db.CurrentDB().Exec(createTable)
return err
}
func removeDBIfExists(filename string) error {
_, err := os.Stat(filename)
if err != nil {
if !os.IsNotExist(err) {
return err
}
// no db file
return nil
}
return os.Remove(filename)
}

22
internal/geo/ip/ip.go
View File

@ -1,7 +1,6 @@
package ip
import (
"errors"
"fmt"
"log"
"net/http"
@ -19,11 +18,6 @@ import (
"github.com/samonzeweb/godb/adapters/sqlite"
)
var (
ErrNotFound = errors.New("cache entry not found")
ErrInvalidCacheEntry = errors.New("invalid cache entry format")
)
// Location information.
type Location struct {
IP string `db:"ip,key"`
@ -69,8 +63,8 @@ func NewCache(config *config.Config) (*Cache, error) {
}
// Read returns location information from the cache, if found,
// or ErrNotFound if not found. If the entry is found, but its format
// is invalid, ErrInvalidCacheEntry is returned.
// or types.ErrNotFound if not found. If the entry is found, but its format
// is invalid, types.ErrInvalidCacheEntry is returned.
//
// Format:
//
@ -81,7 +75,7 @@ func NewCache(config *config.Config) (*Cache, error) {
// DE;Germany;Free and Hanseatic City of Hamburg;Hamburg;53.5736;9.9782
//
func (c *Cache) Read(addr string) (*Location, error) {
if c.config.Geo.CacheType == types.CacheTypeDB {
if c.config.Geo.IPCacheType == types.CacheTypeDB {
return c.readFromCacheDB(addr)
}
return c.readFromCacheFile(addr)
@ -90,7 +84,7 @@ func (c *Cache) Read(addr string) (*Location, error) {
func (c *Cache) readFromCacheFile(addr string) (*Location, error) {
bytes, err := os.ReadFile(c.cacheFile(addr))
if err != nil {
return nil, ErrNotFound
return nil, types.ErrNotFound
}
return NewLocationFromString(addr, string(bytes))
}
@ -107,7 +101,7 @@ func (c *Cache) readFromCacheDB(addr string) (*Location, error) {
}
func (c *Cache) Put(addr string, loc *Location) error {
if c.config.Geo.CacheType == types.CacheTypeDB {
if c.config.Geo.IPCacheType == types.CacheTypeDB {
return c.putToCacheDB(addr, loc)
}
return c.putToCacheFile(addr, loc)
@ -150,18 +144,18 @@ func NewLocationFromString(addr, s string) (*Location, error) {
parts := strings.Split(s, ";")
if len(parts) < 4 {
return nil, ErrInvalidCacheEntry
return nil, types.ErrInvalidCacheEntry
}
if len(parts) >= 6 {
lat, err = strconv.ParseFloat(parts[4], 64)
if err != nil {
return nil, ErrInvalidCacheEntry
return nil, types.ErrInvalidCacheEntry
}
long, err = strconv.ParseFloat(parts[5], 64)
if err != nil {
return nil, ErrInvalidCacheEntry
return nil, types.ErrInvalidCacheEntry
}
}

4
internal/geo/ip/ip_test.go
View File

@ -4,6 +4,8 @@ import (
"testing"
"github.com/stretchr/testify/require"
"github.com/chubin/wttr.in/internal/types"
)
func TestParseCacheEntry(t *testing.T) {
@ -63,7 +65,7 @@ func TestParseCacheEntry(t *testing.T) {
"1.2.3.4",
"DE;Germany;Free and Hanseatic City of Hamburg;Hamburg;53.5736;XXX",
Location{},
ErrInvalidCacheEntry,
types.ErrInvalidCacheEntry,
},
}

27
internal/geo/location/location.go
View File

@ -1,12 +1,29 @@
package location
import "github.com/chubin/wttr.in/internal/config"
import (
"encoding/json"
"log"
"github.com/chubin/wttr.in/internal/config"
)
type Location struct {
Name string
Fullname string `json:"display_name"`
Lat string
Lon string
Name string `db:"name,key"`
Fullname string `db:"displayName" json:"display_name"`
Lat string `db:"lat"`
Lon string `db:"lon"`
Timezone string `db:"timezone"`
}
// String returns string represenation of location
func (l *Location) String() string {
bytes, err := json.Marshal(l)
if err != nil {
// should never happen
log.Fatalln(err)
}
return string(bytes)
}
type Provider interface {

7
internal/types/types.go
View File

@ -1,8 +1,15 @@
package types
import "errors"
type CacheType string
const (
CacheTypeDB = "db"
CacheTypeFiles = "files"
)
var (
ErrNotFound = errors.New("cache entry not found")
ErrInvalidCacheEntry = errors.New("invalid cache entry format")
)

12
srv.go
View File

@ -25,7 +25,8 @@ var cli struct {
ConfigFile string `name:"config-file" arg:"" optional:"" help:"Name of configuration file"`
ConvertGeoIPCache bool `name:"convert-geo-ip-cache" help:"Convert Geo IP data cache to SQlite"`
ConvertGeoIPCache bool `name:"convert-geo-ip-cache" help:"Convert Geo IP data cache to SQlite"`
ConvertGeoLocationCache bool `name:"convert-geo-location-cache" help:"Convert Geo Location data cache to SQlite"`
}
const logLineStart = "LOG_LINE_START "
@ -188,6 +189,15 @@ func main() {
return
}
if cli.ConvertGeoLocationCache {
geoLocCache, err := geoloc.NewCache(conf)
if err != nil {
ctx.FatalIfErrorf(err)
}
ctx.FatalIfErrorf(geoLocCache.ConvertCache())
return
}
if cli.GeoResolve != "" {
sr := geoloc.NewSearcher(conf)
loc, err := sr.Search(cli.GeoResolve)