From cf7016752942f980a2a239641bcc22b11512a6e3 Mon Sep 17 00:00:00 2001 From: Igor Chubin Date: Fri, 14 Jul 2023 21:08:02 +0200 Subject: [PATCH] Switch geolocation client to the internal service --- lib/location.py | 51 ++++++++++++++++++++++++++++++++++-------------- lib/proxy_log.py | 13 +++++++----- lib/view/line.py | 3 ++- lib/view/v2.py | 9 ++++++--- 4 files changed, 52 insertions(+), 24 deletions(-) diff --git a/lib/location.py b/lib/location.py index 7b36e79..f457c3f 100644 --- a/lib/location.py +++ b/lib/location.py @@ -40,6 +40,7 @@ import json import os import socket import sys +import random import geoip2.database import pycountry @@ -48,7 +49,6 @@ import requests from globals import GEOLITE, GEOLOCATOR_SERVICE, IP2LCACHE, IP2LOCATION_KEY, NOT_FOUND_LOCATION, \ ALIASES, BLACKLIST, IATA_CODES_FILE, IPLOCATION_ORDER, IPINFO_TOKEN - GEOIP_READER = geoip2.database.Reader(GEOLITE) COUNTRY_MAP = {"Russian Federation": "Russia"} @@ -99,7 +99,10 @@ def _geolocator(location): """ try: - geo = requests.get('%s/%s' % (GEOLOCATOR_SERVICE, location)).text + if random.random() < 0: + geo = requests.get('%s/%s' % (GEOLOCATOR_SERVICE, location)).text + else: + geo = requests.get("http://127.0.0.1:8083/:geo-location?location=%s" % location).text except requests.exceptions.ConnectionError as exception: print("ERROR: %s" % exception) return None @@ -109,6 +112,8 @@ def _geolocator(location): try: answer = json.loads(geo.encode('utf-8')) + if "error" in answer: + return None return answer except ValueError as exception: print("ERROR: %s" % exception) @@ -129,6 +134,7 @@ def _ipcachewrite(ip_addr, location): The latitude and longitude are optional elements. """ + return cachefile = os.path.join(IP2LCACHE, ip_addr) if not os.path.exists(IP2LCACHE): os.makedirs(IP2LCACHE) @@ -144,20 +150,29 @@ def _ipcache(ip_addr): Returns a triple of (CITY, REGION, COUNTRY) or None TODO: When cache becomes more robust, transition to using latlong """ - cachefile = os.path.join(IP2LCACHE, ip_addr) - if os.path.exists(cachefile): - try: - _, country, region, city, *_ = open(cachefile, 'r').read().split(';') - return city, region, country - except ValueError: - # cache entry is malformed: should be - # [ccode];country;region;city;[lat];[long];... - return None - else: - _debug_log("[_ipcache] %s not found" % ip_addr) + ## Use Geo IP service when available + r = requests.get("http://127.0.0.1:8083/:geo-ip-get?ip=%s" % ip_addr) + if r.status_code == 200 and ";" in r.text: + _, country, region, city, *_ = r.text.split(';') + return city, region, country + return None + # cachefile = os.path.join(IP2LCACHE, ip_addr) + # + # if os.path.exists(cachefile): + # try: + # _, country, region, city, *_ = open(cachefile, 'r').read().split(';') + # return city, region, country + # except ValueError: + # # cache entry is malformed: should be + # # [ccode];country;region;city;[lat];[long];... + # return None + # else: + # _debug_log("[_ipcache] %s not found" % ip_addr) + # return None + def _ip2location(ip_addr): """ Convert IP address `ip_addr` to a location name using ip2location. @@ -340,7 +355,7 @@ def _get_hemisphere(location): geolocation = _geolocator(location_string) if geolocation is None: return True - return geolocation["latitude"] > 0 + return float(geolocation["latitude"]) > 0 def _fully_qualified_location(location, region, country): @@ -477,7 +492,13 @@ def _main_(): print(city) shutil.move(filename, os.path.join("/wttr.in/cache/ip2l-broken-format", ip_address)) +def _trace_ip(): + + print(_geoip("108.5.186.108")) + print(_get_location("108.5.186.108")) + print(location_processing("", "108.5.186.108")) if __name__ == "__main__": - _main_() + _trace_ip() + #_main_() #print(_geoip("173.216.90.56")) diff --git a/lib/proxy_log.py b/lib/proxy_log.py index 5673569..0dc88bc 100644 --- a/lib/proxy_log.py +++ b/lib/proxy_log.py @@ -14,10 +14,12 @@ class Logger: For specific loggers, _shorten_query() should be rewritten. """ - def __init__(self, filename): + def __init__(self, filename_access, filename_errors): - self._filename = filename - self._file = open(filename, "a", encoding="utf-8") + self._filename_access = filename_access + self._filename_errors = filename_errors + self._log_access = open(filename_access, "a", encoding="utf-8") + self._log_errors = open(filename_errors, "a", encoding="utf-8") def _shorten_query(self, query): return query @@ -31,10 +33,11 @@ class Logger: query = self._shorten_query(query) if error != "": message += " ERR " + query + " " + error + self._log_errors.write(message+"\n") else: - message = " OK " + query + message += " OK " + query + self._log_access.write(message+"\n") - self._file.write(message+"\n") class LoggerWWO(Logger): """ diff --git a/lib/view/line.py b/lib/view/line.py index cc0ae26..1198a5d 100644 --- a/lib/view/line.py +++ b/lib/view/line.py @@ -238,7 +238,8 @@ def render_moonday(_, query): # this is just a temporary solution def get_geodata(location): - text = requests.get("http://localhost:8004/%s" % location).text + # text = requests.get("http://localhost:8004/%s" % location).text + text = requests.get("http://127.0.0.1:8083/:geo-location?location=%s" % location).text return json.loads(text) diff --git a/lib/view/v2.py b/lib/view/v2.py index 047dbf6..ac401e7 100644 --- a/lib/view/v2.py +++ b/lib/view/v2.py @@ -573,13 +573,16 @@ def textual_information(data_parsed, geo_data, config, html_output=False): city_only = True suffix = ", Крым" + latitude = float(geo_data["latitude"]) + longitude = float(geo_data["longitude"]) + if config["full_address"]: output.append('Location: %s%s [%5.4f,%5.4f]' \ % ( _shorten_full_location(config["full_address"], city_only=city_only), suffix, - geo_data["latitude"], - geo_data["longitude"], + latitude, + longitude, )) output = [ @@ -593,7 +596,7 @@ def textual_information(data_parsed, geo_data, config, html_output=False): # }}} # get_geodata {{{ def get_geodata(location): - text = requests.get("http://localhost:8004/%s" % location).text + text = requests.get("http://127.0.0.1:8083/:geo-location?location=%s" % location).text return json.loads(text) # }}}