webadmin/bpfire
1
0
Fork 0
mirror of https://github.com/vincentmli/bpfire.git synced 2026-04-09 18:45:54 +02:00
Code Issues Packages Projects Releases Wiki Activity

libloc: Update to 0.9.2

Browse Source 
Signed-off-by: Stefan Schantl <stefan.schantl@ipfire.org>
This commit is contained in:
Stefan Schantl
2020-06-15 19:47:07 +02:00
parent f1d982cce6
commit 45f4de2bbc
17 changed files with 3 additions and 2037 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
lfs/libloc
View File

@@ -24,7 +24,7 @@
include Config
VER = 0.9.1
VER = 0.9.2
DB_DATE = 2020-06-10
THISAPP = libloc-$(VER)
@@ -40,10 +40,10 @@ TARGET = $(DIR_INFO)/$(THISAPP)
objects = $(DL_FILE) \
location-$(DB_DATE).db.xz
$(DL_FILE) = $(DL_FROM)/$(DL_FILE)
$(DL_FILE) = https://source.ipfire.org/releases/libloc//$(DL_FILE)
location-$(DB_DATE).db.xz = https://location.ipfire.org/databases/1/archive/location-$(DB_DATE).db.xz
$(DL_FILE)_MD5 = b62331e7a5bc5299bdd35f340342fc51
$(DL_FILE)_MD5 = c6ed4fcbdb2ce4ca7e6df8abbd985411
location-$(DB_DATE).db.xz_MD5 = 268b6d58a26c6d36081ed1e899b89020
install : $(TARGET)
@@ -78,24 +78,6 @@ $(TARGET) : $(patsubst %,$(DIR_DL)/%,$(objects))
@$(PREBUILD)
@rm -rf $(DIR_APP) && cd $(DIR_SRC) && tar xvf $(DIR_DL)/$(DL_FILE)
# Add upstream patches.
cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-downloader-do-not-change-content-of-open-database.patch
cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-remove-python-path-overrides-for-debian.patch
cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-query-require-at-least-one-flag.patch
cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-exporter-do-not-mistake-country-as-for-an-as-number.patch
cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-location-exporter-warn-but-do-not-fail-on-invalid-input.patch
cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-move-location-downloader-functionality-into-location-query.patch
cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-merge-location-downloader-manpage-into-location-query.patch
cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-downloader-rename-user-agent-to-location.patch
cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-rename-location-query-to-location.patch
cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-merge-location-exporter-into-location.patch
cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-remove-accidently-commited-hacks-for-debian.patch
cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-add-option-to-iterate-over-all-contries-and-print-them.patch
cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-export-all-countries-by-default.patch
cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-export-flagged-networks-with-their-faked-country-names-too.patch
cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-database-fix-brocken-search-for-networks-with-flags.patch
cd $(DIR_APP) && patch -Np1 -i $(DIR_SRC)/src/patches/libloc-0.9.1-adjust-format-to-print-ASes.patch
cd $(DIR_APP) && ./autogen.sh
cd $(DIR_APP) && ./configure \
--prefix=/usr \

214
src/patches/libloc-0.9.1-add-option-to-iterate-over-all-contries-and-print-them.patch
View File

@@ -1,214 +0,0 @@
commit fa9a3663cb2dfb2490da43f6967f1a3a2948fc8a
Author: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri Jun 5 09:41:28 2020 +0000
Add option to iterate over all countries and print them to the console
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
diff --git a/man/location.txt b/man/location.txt
index 672c2b2..0d70e0b 100644
--- a/man/location.txt
+++ b/man/location.txt
@@ -13,6 +13,7 @@ location - Query the location database
`location list-networks-by-as ASN`
`location list-networks-by-cc COUNTRY_CODE`
`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]`
+`location list-countries [--show-name] [--show-continent]`
== DESCRIPTION
`location` retrieves information from the location database.
@@ -86,6 +87,12 @@ or countries.
+
See above for usage of the '--family' and '--output-format' parameters.
+'list-countries [--show-name] [--show-continent]'::
+ Lists all countries known to the database.
+ +
+ With the optional parameters '--show-name' and '--show-continent', the name and
+ continent code will be printed, too.
+
'--help'::
Shows a short help text on using this program.
diff --git a/src/database.c b/src/database.c
index d919278..8e6c5ab 100644
--- a/src/database.c
+++ b/src/database.c
@@ -107,6 +107,9 @@ struct loc_database_enumerator {
// Index of the AS we are looking at
unsigned int as_index;
+ // Index of the country we are looking at
+ unsigned int country_index;
+
// Network state
struct in6_addr network_address;
struct loc_node_stack network_stack[MAX_STACK_DEPTH];
@@ -1219,3 +1222,30 @@ LOC_EXPORT int loc_database_enumerator_next_network(
return 0;
}
+
+LOC_EXPORT int loc_database_enumerator_next_country(
+ struct loc_database_enumerator* enumerator, struct loc_country** country) {
+ *country = NULL;
+
+ // Do not do anything if not in country mode
+ if (enumerator->mode != LOC_DB_ENUMERATE_COUNTRIES)
+ return 0;
+
+ struct loc_database* db = enumerator->db;
+
+ while (enumerator->country_index < db->countries_count) {
+ // Fetch the next country
+ int r = loc_database_fetch_country(db, country, enumerator->country_index++);
+ if (r)
+ return r;
+
+ // We do not filter here, so it always is a match
+ return 0;
+ }
+
+ // Reset the index
+ enumerator->country_index = 0;
+
+ // We have searched through all of them
+ return 0;
+}
diff --git a/src/libloc.sym b/src/libloc.sym
index e9e8549..9a1e6f0 100644
--- a/src/libloc.sym
+++ b/src/libloc.sym
@@ -68,6 +68,7 @@ global:
# Database Enumerator
loc_database_enumerator_new;
loc_database_enumerator_next_as;
+ loc_database_enumerator_next_country;
loc_database_enumerator_next_network;
loc_database_enumerator_ref;
loc_database_enumerator_set_asn;
diff --git a/src/loc/database.h b/src/loc/database.h
index ab9ef72..43173dd 100644
--- a/src/loc/database.h
+++ b/src/loc/database.h
@@ -50,8 +50,9 @@ int loc_database_get_country(struct loc_database* db,
struct loc_country** country, const char* code);
enum loc_database_enumerator_mode {
- LOC_DB_ENUMERATE_NETWORKS = 1,
- LOC_DB_ENUMERATE_ASES = 2,
+ LOC_DB_ENUMERATE_NETWORKS = 1,
+ LOC_DB_ENUMERATE_ASES = 2,
+ LOC_DB_ENUMERATE_COUNTRIES = 3,
};
struct loc_database_enumerator;
@@ -69,5 +70,7 @@ int loc_database_enumerator_next_as(
struct loc_database_enumerator* enumerator, struct loc_as** as);
int loc_database_enumerator_next_network(
struct loc_database_enumerator* enumerator, struct loc_network** network);
+int loc_database_enumerator_next_country(
+ struct loc_database_enumerator* enumerator, struct loc_country** country);
#endif
diff --git a/src/python/database.c b/src/python/database.c
index 581ed5b..1013a58 100644
--- a/src/python/database.c
+++ b/src/python/database.c
@@ -316,6 +316,10 @@ static PyObject* Database_search_networks(DatabaseObject* self, PyObject* args,
return obj;
}
+static PyObject* Database_countries(DatabaseObject* self) {
+ return Database_iterate_all(self, LOC_DB_ENUMERATE_COUNTRIES);
+}
+
static struct PyMethodDef Database_methods[] = {
{
"get_as",
@@ -364,6 +368,13 @@ static struct PyGetSetDef Database_getsetters[] = {
NULL,
NULL,
},
+ {
+ "countries",
+ (getter)Database_countries,
+ NULL,
+ NULL,
+ NULL,
+ },
{
"created_at",
(getter)Database_get_created_at,
@@ -462,6 +473,22 @@ static PyObject* DatabaseEnumerator_next(DatabaseEnumeratorObject* self) {
return obj;
}
+ // Enumerate all countries
+ struct loc_country* country = NULL;
+
+ r = loc_database_enumerator_next_country(self->enumerator, &country);
+ if (r) {
+ PyErr_SetFromErrno(PyExc_ValueError);
+ return NULL;
+ }
+
+ if (country) {
+ PyObject* obj = new_country(&CountryType, country);
+ loc_country_unref(country);
+
+ return obj;
+ }
+
// Nothing found, that means the end
PyErr_SetNone(PyExc_StopIteration);
return NULL;
diff --git a/src/python/location.in b/src/python/location.in
index 7614cae..5c1effd 100644
--- a/src/python/location.in
+++ b/src/python/location.in
@@ -147,6 +147,18 @@ class CLI(object):
choices=location.export.formats.keys(), default="list")
list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
+ # List countries
+ list_countries = subparsers.add_parser("list-countries",
+ help=_("Lists all countries"),
+ )
+ list_countries.add_argument("--show-name",
+ action="store_true", help=_("Show the name of the country"),
+ )
+ list_countries.add_argument("--show-continent",
+ action="store_true", help=_("Show the continent"),
+ )
+ list_countries.set_defaults(func=self.handle_list_countries)
+
# Export
export = subparsers.add_parser("export",
help=_("Exports data in many formats to load it into packet filters"),
@@ -435,6 +447,24 @@ class CLI(object):
return cls
+ def handle_list_countries(self, db, ns):
+ for country in db.countries:
+ line = [
+ country.code,
+ ]
+
+ if ns.show_continent:
+ line.append(country.continent_code)
+
+ if ns.show_name:
+ line.append(country.name)
+
+ # Format the output
+ line = " ".join(line)
+
+ # Print the output
+ print(line)
+
def handle_list_networks_by_as(self, db, ns):
writer = self.__get_output_formatter(ns)

0
src/patches/libloc-0.9.1-adjust-format-to-print-ASes.patch
View File

24
src/patches/libloc-0.9.1-database-fix-brocken-search-for-networks-with-flags.patch
View File

@@ -1,24 +0,0 @@
commit 864dd22e17f7487a90e165274cf3f7898966028d
Author: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri Jun 5 10:01:47 2020 +0000
database: Fix broken search for networks with flags
The search was ended after the first network. No matter if
it matched, or not.
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
diff --git a/src/database.c b/src/database.c
index 8e6c5ab..fa1dad0 100644
--- a/src/database.c
+++ b/src/database.c
@@ -1208,6 +1208,8 @@ LOC_EXPORT int loc_database_enumerator_next_network(
!loc_network_match_flag(*network, enumerator->flags)) {
loc_network_unref(*network);
*network = NULL;
+
+ continue;
}
return 0;

21
src/patches/libloc-0.9.1-downloader-rename-user-agent-to-location.patch
View File

@@ -1,21 +0,0 @@
commit dc1df0f469668ef3dc4e1a9a7623a0ebba2b051e
Author: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed Jun 3 17:15:27 2020 +0000
downloader: Change user-agent to location
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
diff --git a/src/python/downloader.py b/src/python/downloader.py
index c9e6e00..eb28007 100644
--- a/src/python/downloader.py
+++ b/src/python/downloader.py
@@ -71,7 +71,7 @@ class Downloader(object):
# Update headers
headers.update({
- "User-Agent" : "location-downloader/@VERSION@",
+ "User-Agent" : "location/@VERSION@",
})
# Set headers

33
src/patches/libloc-0.9.1-export-all-countries-by-default.patch
View File

@@ -1,33 +0,0 @@
commit 10fa313b392a269e15bdaf316218a114d9b23b55
Author: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri Jun 5 09:47:36 2020 +0000
location(8): Export all countries by default
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
diff --git a/src/python/location.in b/src/python/location.in
index 5c1effd..6ced5f5 100644
--- a/src/python/location.in
+++ b/src/python/location.in
@@ -169,7 +169,7 @@ class CLI(object):
export.add_argument("--family",
help=_("Specify address family"), choices=("ipv6", "ipv4"),
)
- export.add_argument("objects", nargs="+", help=_("List country codes or ASNs to export"))
+ export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export"))
export.set_defaults(func=self.handle_export)
args = parser.parse_args()
@@ -539,9 +539,9 @@ class CLI(object):
log.warning("Invalid argument: %s" % object)
continue
+ # Default to exporting all countries
if not countries and not asns:
- log.error("Nothing to export")
- return 2
+ countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
# Select the output format
writer = self.__get_output_formatter(ns)

54
src/patches/libloc-0.9.1-export-flagged-networks-with-their-faked-country-names-too.patch
View File

@@ -1,54 +0,0 @@
commit fae36e81a32717ac43c0ce48702f6ff05b7cd029
Author: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri Jun 5 09:57:41 2020 +0000
export flagged networks with their faked country names, too
This will lead to some networks showing up twice. Once with
their real country and once with their faked one.
It is likely that the first one will match.
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
diff --git a/src/python/export.py b/src/python/export.py
index 69fe964..d0bbe77 100644
--- a/src/python/export.py
+++ b/src/python/export.py
@@ -23,10 +23,18 @@ import logging
import os
import socket
+import _location
+
# Initialise logging
log = logging.getLogger("location.export")
log.propagate = 1
+flags = {
+ _location.NETWORK_FLAG_ANONYMOUS_PROXY : "A1",
+ _location.NETWORK_FLAG_SATELLITE_PROVIDER : "A2",
+ _location.NETWORK_FLAG_ANYCAST : "A3",
+}
+
class OutputWriter(object):
suffix = "networks"
mode = "w"
@@ -173,6 +181,17 @@ class Exporter(object):
except KeyError:
pass
+ # Handle flags
+ for flag in flags:
+ if network.has_flag(flag):
+ # Fetch the "fake" country code
+ country = flags[flag]
+
+ try:
+ writers[country].write(network)
+ except KeyError:
+ pass
+
# Write everything to the filesystem
for writer in writers.values():
writer.finish()

81
src/patches/libloc-0.9.1-location-downloader-do-not-change-content-of-open-database.patch
View File

@@ -1,81 +0,0 @@
commit 679e5ae2e45b98e254c651cb3102a4331c2c22eb
Author: Michael Tremer <michael.tremer@ipfire.org>
Date: Mon Jun 1 13:47:44 2020 +0000
location-downloader: Do not change content of open database files
The database might be opened by another process. When modified,
it will return random results.
Fixes: #12420
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
diff --git a/src/python/location-downloader.in b/src/python/location-downloader.in
index 7d06030..bf0d682 100644
--- a/src/python/location-downloader.in
+++ b/src/python/location-downloader.in
@@ -24,6 +24,7 @@ import lzma
import os
import random
import shutil
+import stat
import sys
import tempfile
import time
@@ -116,7 +117,7 @@ class Downloader(object):
return res
- def download(self, url, public_key, timestamp=None, **kwargs):
+ def download(self, url, public_key, timestamp=None, tmpdir=None, **kwargs):
headers = {}
if timestamp:
@@ -124,7 +125,7 @@ class Downloader(object):
"%a, %d %b %Y %H:%M:%S GMT",
)
- t = tempfile.NamedTemporaryFile(delete=False)
+ t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False)
with t:
# Try all mirrors
for mirror in self.mirrors:
@@ -175,6 +176,9 @@ class Downloader(object):
t.truncate()
continue
+ # Make the file readable for everyone
+ os.chmod(t.name, stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH)
+
# Return temporary file
return t
@@ -296,10 +300,13 @@ class CLI(object):
except FileNotFoundError as e:
db = None
+ # Download the database into the correct directory
+ tmpdir = os.path.dirname(ns.database)
+
# Try downloading a new database
try:
t = self.downloader.download("%s/%s" % (self.version, DATABASE_FILENAME),
- public_key=ns.public_key, timestamp=timestamp)
+ public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
# If no file could be downloaded, log a message
except FileNotFoundError as e:
@@ -310,11 +317,8 @@ class CLI(object):
if not t:
return 3
- # Write temporary file to destination
- shutil.copyfile(t.name, ns.database)
-
- # Remove temporary file
- os.unlink(t.name)
+ # Move temporary file to destination
+ shutil.move(t.name, ns.database)
return 0

36
src/patches/libloc-0.9.1-location-exporter-do-not-mistake-country-as-for-an-as-number.patch
View File

@@ -1,36 +0,0 @@
commit 141b10999b280b2563580c705d5d23dc4c442deb
Author: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed Jun 3 16:31:44 2020 +0000
location-exporter: Do not mistake country AS for an AS number
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
diff --git a/src/python/location-exporter.in b/src/python/location-exporter.in
index 894bb44..5454561 100644
--- a/src/python/location-exporter.in
+++ b/src/python/location-exporter.in
@@ -22,6 +22,7 @@ import io
import ipaddress
import logging
import os.path
+import re
import socket
import sys
@@ -258,12 +259,9 @@ class CLI(object):
families = [ socket.AF_INET6, socket.AF_INET ]
for object in ns.objects:
- if object.startswith("AS"):
- try:
- object = int(object[2:])
- except ValueError:
- log.error("Invalid argument: %s" % object)
- return 2
+ m = re.match("^AS(\d+)$", object)
+ if m:
+ object = int(m.group(1))
asns.append(object)

27
src/patches/libloc-0.9.1-location-exporter-warn-but-do-not-fail-on-invalid-input.patch
View File

@@ -1,27 +0,0 @@
commit 92af07adfb1e06fe1b055fbcf5ba61159637cd73
Author: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed Jun 3 16:33:44 2020 +0000
location-exporter: Warn, but do not fail on invalid input
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
diff --git a/src/python/location-exporter.in b/src/python/location-exporter.in
index 5454561..d82f1d3 100644
--- a/src/python/location-exporter.in
+++ b/src/python/location-exporter.in
@@ -270,8 +270,12 @@ class CLI(object):
countries.append(object)
else:
- log.error("Invalid argument: %s" % object)
- return 2
+ log.warning("Invalid argument: %s" % object)
+ continue
+
+ if not countries and not asns:
+ log.error("Nothing to export")
+ return 2
# Open the database
try:

37
src/patches/libloc-0.9.1-location-query-require-at-least-one-flag.patch
View File

@@ -1,37 +0,0 @@
commit 228d0e74ec47c9954d3a0e1da2e1c0fc6c1b518f
Author: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed Jun 3 16:15:24 2020 +0000
location-query: Require at least one flag
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
diff --git a/src/python/location-query.in b/src/python/location-query.in
index 5f05b5c..dfdff8c 100644
--- a/src/python/location-query.in
+++ b/src/python/location-query.in
@@ -246,7 +246,13 @@ class CLI(object):
args.family = 0
# Call function
- ret = args.func(db, args)
+ try:
+ ret = args.func(db, args)
+
+ # Catch invalid inputs
+ except ValueError as e:
+ sys.stderr.write("%s\n" % e)
+ ret = 2
# Return with exit code
if ret:
@@ -451,6 +457,9 @@ class CLI(object):
if ns.anycast:
flags |= location.NETWORK_FLAG_ANYCAST
+ if not flags:
+ raise ValueError(_("You must at least pass one flag"))
+
with self.__get_output_formatter(ns) as f:
for n in db.search_networks(flags=flags, family=ns.family):
f.network(n)

140
src/patches/libloc-0.9.1-merge-location-downloader-manpage-into-location-query.patch
View File

@@ -1,140 +0,0 @@
commit 889b932aa6172c96872be545af37d351f7c1c705
Author: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed Jun 3 17:10:35 2020 +0000
location-downloader: Merge man page into location-query
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
diff --git a/Makefile.am b/Makefile.am
index c0b1300..91f0436 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -37,6 +37,9 @@ LIBLOC_CURRENT=0
LIBLOC_REVISION=0
LIBLOC_AGE=0
+pythondir = $(prefix)/lib/python3/dist-packages
+pyexecdir = $(prefix)/lib/python$(PYTHON_VERSION)/lib-dynload
+
DISTCHECK_CONFIGURE_FLAGS = \
--with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir)
@@ -371,7 +374,6 @@ src_test_signature_LDADD = \
# ------------------------------------------------------------------------------
MANPAGES = \
- man/location-downloader.8 \
man/location-query.8
MANPAGES_TXT = $(patsubst %.8,%.txt,$(MANPAGES))
diff --git a/man/location-downloader.txt b/man/location-downloader.txt
deleted file mode 100644
index d733923..0000000
--- a/man/location-downloader.txt
+++ /dev/null
@@ -1,61 +0,0 @@
-= location-downloader(8)
-
-== NAME
-location-downloader - Download a location database
-
-== SYNOPSIS
-[verse]
-`location-downloader update`
-
-== DESCRIPTION
-The `location-downloader` command updates the local version of the
-location database.
-
-== OPTIONS
-
---database FILE::
--d FILE::
- The path of the database which is being updated.
- +
- If this option is omitted, the system's database will be opened.
-
---quiet::
- Enable quiet mode
-
---debug::
- Enable debugging mode
-
-== COMMANDS
-
-'update'::
- This command will try to update the local database.
- +
- It will terminate with a return code of zero if the database has been
- successfully updated. 1 on error, 2 on invalid call and 3 if the
- database was already the latest version.
-
-'verify'::
- Verifies the downloaded database.
-
-'--help'::
- Shows a short help text on using this program.
-
-'--version'::
- Shows the program's version and exists.
-
-== EXIT CODES
-The 'location-downloader' command will normally exit with code zero.
-If there has been a problem and the requested action could not be performed,
-the exit code is unequal to zero.
-
-== HOW IT WORKS
-The downloader checks a DNS record for the latest version of the database.
-It will then try to download a file with that version from a mirror server.
-If the downloaded file is outdated, the next mirror will be tried until we
-have found a file that is recent enough.
-
-== BUGS
-Please report all bugs to the bugtracker at https://bugzilla.ipfire.org/.
-
-== AUTHORS
-Michael Tremer
diff --git a/man/location-query.txt b/man/location-query.txt
index b91e8e1..acb43cd 100644
--- a/man/location-query.txt
+++ b/man/location-query.txt
@@ -8,6 +8,8 @@ location-query - Query the location database
`location-query lookup ADDRESS [ADDRESS...]`
`location-query get-as ASN [ASN...]`
`location-query search-as STRING`
+`location-query update`
+`location-query verify`
`location-query list-networks-by-as ASN`
`location-query list-networks-by-cc COUNTRY_CODE`
`location-query list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]`
@@ -47,6 +49,16 @@ or countries.
+
The search will be performed case-insensitively.
+'update'::
+ This command will try to update the local database.
+ +
+ It will terminate with a return code of zero if the database has been
+ successfully updated. 1 on error, 2 on invalid call and 3 if the
+ database was already the latest version.
+
+'verify'::
+ Verifies the downloaded database.
+
'list-networks-by-as [--family=[ipv6|ipv4]] [--output-format FORMAT] ASN'::
Lists all networks which belong to this Autonomous System.
+
@@ -85,6 +97,12 @@ The 'location-query' command will normally exit with code zero.
If there has been a problem and the requested action could not be performed,
the exit code is unequal to zero.
+== HOW IT WORKS
+The downloader checks a DNS record for the latest version of the database.
+It will then try to download a file with that version from a mirror server.
+If the downloaded file is outdated, the next mirror will be tried until we
+have found a file that is recent enough.
+
== BUGS
Please report all bugs to the bugtracker at https://bugzilla.ipfire.org/.

796
src/patches/libloc-0.9.1-merge-location-exporter-into-location.patch
View File

@@ -1,796 +0,0 @@
commit 88ef7e9cd4b3a1a5662c7dc071bd7a44e1242cba
Author: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed Jun 3 18:36:28 2020 +0000
Merge location-exporter(8) into location(8)
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
diff --git a/Makefile.am b/Makefile.am
index 59870b1..9f520cc 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -150,6 +150,7 @@ dist_pkgpython_PYTHON = \
src/python/__init__.py \
src/python/database.py \
src/python/downloader.py \
+ src/python/export.py \
src/python/i18n.py \
src/python/importer.py \
src/python/logger.py
@@ -239,17 +240,14 @@ uninstall-perl:
bin_SCRIPTS = \
src/python/location \
- src/python/location-exporter \
src/python/location-importer
EXTRA_DIST += \
src/python/location.in \
- src/python/location-exporter.in \
src/python/location-importer.in
CLEANFILES += \
src/python/location \
- src/python/location-exporter \
src/python/location-importer
# ------------------------------------------------------------------------------
diff --git a/src/python/export.py b/src/python/export.py
new file mode 100644
index 0000000..69fe964
--- /dev/null
+++ b/src/python/export.py
@@ -0,0 +1,185 @@
+#!/usr/bin/python3
+###############################################################################
+# #
+# libloc - A library to determine the location of someone on the Internet #
+# #
+# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
+# #
+# This library is free software; you can redistribute it and/or #
+# modify it under the terms of the GNU Lesser General Public #
+# License as published by the Free Software Foundation; either #
+# version 2.1 of the License, or (at your option) any later version. #
+# #
+# This library is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
+# Lesser General Public License for more details. #
+# #
+###############################################################################
+
+import io
+import ipaddress
+import logging
+import os
+import socket
+
+# Initialise logging
+log = logging.getLogger("location.export")
+log.propagate = 1
+
+class OutputWriter(object):
+ suffix = "networks"
+ mode = "w"
+
+ def __init__(self, f, prefix=None):
+ self.f, self.prefix = f, prefix
+
+ # Immediately write the header
+ self._write_header()
+
+ @classmethod
+ def open(cls, filename, **kwargs):
+ """
+ Convenience function to open a file
+ """
+ f = open(filename, cls.mode)
+
+ return cls(f, **kwargs)
+
+ def __repr__(self):
+ return "<%s f=%s>" % (self.__class__.__name__, self.f)
+
+ def _write_header(self):
+ """
+ The header of the file
+ """
+ pass
+
+ def _write_footer(self):
+ """
+ The footer of the file
+ """
+ pass
+
+ def write(self, network):
+ self.f.write("%s\n" % network)
+
+ def finish(self):
+ """
+ Called when all data has been written
+ """
+ self._write_footer()
+
+ # Close the file
+ self.f.close()
+
+
+class IpsetOutputWriter(OutputWriter):
+ """
+ For ipset
+ """
+ suffix = "ipset"
+
+ def _write_header(self):
+ self.f.write("create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.prefix)
+
+ def write(self, network):
+ self.f.write("add %s %s\n" % (self.prefix, network))
+
+
+class NftablesOutputWriter(OutputWriter):
+ """
+ For nftables
+ """
+ suffix = "set"
+
+ def _write_header(self):
+ self.f.write("define %s = {\n" % self.prefix)
+
+ def _write_footer(self):
+ self.f.write("}\n")
+
+ def write(self, network):
+ self.f.write(" %s,\n" % network)
+
+
+class XTGeoIPOutputWriter(OutputWriter):
+ """
+ Formats the output in that way, that it can be loaded by
+ the xt_geoip kernel module from xtables-addons.
+ """
+ suffix = "iv"
+ mode = "wb"
+
+ def write(self, network):
+ n = ipaddress.ip_network("%s" % network)
+
+ for address in (n.network_address, n.broadcast_address):
+ bytes = socket.inet_pton(
+ socket.AF_INET6 if address.version == 6 else socket.AF_INET,
+ "%s" % address,
+ )
+
+ self.f.write(bytes)
+
+
+formats = {
+ "ipset" : IpsetOutputWriter,
+ "list" : OutputWriter,
+ "nftables" : NftablesOutputWriter,
+ "xt_geoip" : XTGeoIPOutputWriter,
+}
+
+class Exporter(object):
+ def __init__(self, db, writer):
+ self.db, self.writer = db, writer
+
+ def export(self, directory, families, countries, asns):
+ for family in families:
+ log.debug("Exporting family %s" % family)
+
+ writers = {}
+
+ # Create writers for countries
+ for country_code in countries:
+ filename = self._make_filename(
+ directory, prefix=country_code, suffix=self.writer.suffix, family=family,
+ )
+
+ writers[country_code] = self.writer.open(filename, prefix="CC_%s" % country_code)
+
+ # Create writers for ASNs
+ for asn in asns:
+ filename = self._make_filename(
+ directory, "AS%s" % asn, suffix=self.writer.suffix, family=family,
+ )
+
+ writers[asn] = self.writer.open(filename, prefix="AS%s" % asn)
+
+ # Get all networks that match the family
+ networks = self.db.search_networks(family=family)
+
+ # Walk through all networks
+ for network in networks:
+ # Write matching countries
+ try:
+ writers[network.country_code].write(network)
+ except KeyError:
+ pass
+
+ # Write matching ASNs
+ try:
+ writers[network.asn].write(network)
+ except KeyError:
+ pass
+
+ # Write everything to the filesystem
+ for writer in writers.values():
+ writer.finish()
+
+ def _make_filename(self, directory, prefix, suffix, family):
+ filename = "%s.%s%s" % (
+ prefix, suffix, "6" if family == socket.AF_INET6 else "4"
+ )
+
+ return os.path.join(directory, filename)
diff --git a/src/python/location-exporter.in b/src/python/location-exporter.in
deleted file mode 100644
index d82f1d3..0000000
--- a/src/python/location-exporter.in
+++ /dev/null
@@ -1,300 +0,0 @@
-#!/usr/bin/python3
-###############################################################################
-# #
-# libloc - A library to determine the location of someone on the Internet #
-# #
-# Copyright (C) 2019 IPFire Development Team <info@ipfire.org> #
-# #
-# This library is free software; you can redistribute it and/or #
-# modify it under the terms of the GNU Lesser General Public #
-# License as published by the Free Software Foundation; either #
-# version 2.1 of the License, or (at your option) any later version. #
-# #
-# This library is distributed in the hope that it will be useful, #
-# but WITHOUT ANY WARRANTY; without even the implied warranty of #
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU #
-# Lesser General Public License for more details. #
-# #
-###############################################################################
-
-import argparse
-import io
-import ipaddress
-import logging
-import os.path
-import re
-import socket
-import sys
-
-# Load our location module
-import location
-from location.i18n import _
-
-# Initialise logging
-log = logging.getLogger("location.exporter")
-log.propagate = 1
-
-class OutputWriter(object):
- suffix = "networks"
-
- def __init__(self, family, country_code=None, asn=None):
- self.family, self.country_code, self.asn = family, country_code, asn
-
- self.f = io.BytesIO()
-
- def write_out(self, directory):
- # Make the output filename
- filename = os.path.join(
- directory, self._make_filename(),
- )
-
- with open(filename, "wb") as f:
- self._write_header(f)
-
- # Copy all data into the file
- f.write(self.f.getbuffer())
-
- self._write_footer(f)
-
- def _make_filename(self):
- return "%s.%s%s" % (
- self.country_code or "AS%s" % self.asn,
- self.suffix,
- "6" if self.family == socket.AF_INET6 else "4"
- )
-
- @property
- def name(self):
- if self.country_code:
- return "CC_%s" % self.country_code
-
- if self.asn:
- return "AS%s" % self.asn
-
- def _write_header(self, f):
- """
- The header of the file
- """
- pass
-
- def _write_footer(self, f):
- """
- The footer of the file
- """
- pass
-
- def write(self, network):
- s = "%s\n" % network
-
- self.f.write(s.encode("ascii"))
-
-
-class IpsetOutputWriter(OutputWriter):
- """
- For ipset
- """
- suffix = "ipset"
-
- def _write_header(self, f):
- h = "create %s hash:net family inet hashsize 1024 maxelem 65536\n" % self.name
-
- f.write(h.encode("ascii"))
-
- def write(self, network):
- s = "add %s %s\n" % (self.name, network)
-
- self.f.write(s.encode("ascii"))
-
-
-class NftablesOutputWriter(OutputWriter):
- """
- For nftables
- """
- suffix = "set"
-
- def _write_header(self, f):
- h = "define %s = {\n" % self.name
-
- f.write(h.encode("ascii"))
-
- def _write_footer(self, f):
- f.write(b"}")
-
- def write(self, network):
- s = " %s,\n" % network
-
- self.f.write(s.encode("ascii"))
-
-
-class XTGeoIPOutputWriter(OutputWriter):
- """
- Formats the output in that way, that it can be loaded by
- the xt_geoip kernel module from xtables-addons.
- """
- suffix = "iv"
-
- def write(self, network):
- n = ipaddress.ip_network("%s" % network)
-
- for address in (n.network_address, n.broadcast_address):
- bytes = socket.inet_pton(
- socket.AF_INET6 if address.version == 6 else socket.AF_INET,
- "%s" % address,
- )
-
- self.f.write(bytes)
-
-
-class Exporter(object):
- def __init__(self, db, writer):
- self.db = db
- self.writer = writer
-
- def export(self, directory, families, countries, asns):
- for family in families:
- log.debug("Exporting family %s" % family)
-
- writers = {}
-
- # Create writers for countries
- for country_code in countries:
- writers[country_code] = self.writer(family, country_code=country_code)
-
- # Create writers for ASNs
- for asn in asns:
- writers[asn] = self.writer(family, asn=asn)
-
- # Get all networks that match the family
- networks = self.db.search_networks(family=family)
-
- # Walk through all networks
- for network in networks:
- # Write matching countries
- if network.country_code in countries:
- writers[network.country_code].write(network)
-
- # Write matching ASNs
- if network.asn in asns:
- writers[network.asn].write(network)
-
- # Write everything to the filesystem
- for writer in writers.values():
- writer.write_out(directory)
-
-
-class CLI(object):
- output_formats = {
- "ipset" : IpsetOutputWriter,
- "list" : OutputWriter,
- "nftables" : NftablesOutputWriter,
- "xt_geoip" : XTGeoIPOutputWriter,
- }
-
- def parse_cli(self):
- parser = argparse.ArgumentParser(
- description=_("Location Exporter Command Line Interface"),
- )
-
- # Global configuration flags
- parser.add_argument("--debug", action="store_true",
- help=_("Enable debug output"))
- parser.add_argument("--quiet", action="store_true",
- help=_("Enable quiet mode"))
-
- # version
- parser.add_argument("--version", action="version",
- version="%(prog)s @VERSION@")
-
- # database
- parser.add_argument("--database", "-d",
- default="@databasedir@/database.db", help=_("Path to database"),
- )
-
- # format
- parser.add_argument("--format", help=_("Output format"),
- default="list", choices=self.output_formats.keys())
-
- # directory
- parser.add_argument("--directory", help=_("Output directory"), required=True)
-
- # family
- parser.add_argument("--family", help=_("Specify address family"), choices=("ipv6", "ipv4"))
-
- # Countries and Autonomous Systems
- parser.add_argument("objects", nargs="+")
-
- args = parser.parse_args()
-
- # Configure logging
- if args.debug:
- location.logger.set_level(logging.DEBUG)
- elif args.quiet:
- location.logger.set_level(logging.WARNING)
-
- return args
-
- def run(self):
- # Parse command line arguments
- args = self.parse_cli()
-
- # Call function
- ret = self.handle_export(args)
-
- # Return with exit code
- if ret:
- sys.exit(ret)
-
- # Otherwise just exit
- sys.exit(0)
-
- def handle_export(self, ns):
- countries, asns = [], []
-
- # Translate family
- if ns.family == "ipv6":
- families = [ socket.AF_INET6 ]
- elif ns.family == "ipv4":
- families = [ socket.AF_INET ]
- else:
- families = [ socket.AF_INET6, socket.AF_INET ]
-
- for object in ns.objects:
- m = re.match("^AS(\d+)$", object)
- if m:
- object = int(m.group(1))
-
- asns.append(object)
-
- elif location.country_code_is_valid(object) \
- or object in ("A1", "A2", "A3"):
- countries.append(object)
-
- else:
- log.warning("Invalid argument: %s" % object)
- continue
-
- if not countries and not asns:
- log.error("Nothing to export")
- return 2
-
- # Open the database
- try:
- db = location.Database(ns.database)
- except FileNotFoundError as e:
- log.error("Count not open database: %s" % ns.database)
- return 1
-
- # Select the output format
- writer = self.output_formats.get(ns.format)
- assert writer
-
- e = Exporter(db, writer)
- e.export(ns.directory, countries=countries, asns=asns, families=families)
-
-
-def main():
- # Run the command line interface
- c = CLI()
- c.run()
-
-main()
diff --git a/src/python/location.in b/src/python/location.in
index 10618e2..7614cae 100644
--- a/src/python/location.in
+++ b/src/python/location.in
@@ -22,6 +22,7 @@ import datetime
import ipaddress
import logging
import os
+import re
import shutil
import socket
import sys
@@ -30,6 +31,8 @@ import time
# Load our location module
import location
import location.downloader
+import location.export
+
from location.i18n import _
# Setup logging
@@ -37,88 +40,7 @@ log = logging.getLogger("location")
# Output formatters
-class OutputFormatter(object):
- def __init__(self, ns):
- self.ns = ns
-
- def __enter__(self):
- # Open the output
- self.open()
-
- return self
-
- def __exit__(self, type, value, tb):
- if tb is None:
- self.close()
-
- @property
- def name(self):
- if "country_code" in self.ns:
- return "networks_country_%s" % self.ns.country_code[0]
-
- elif "asn" in self.ns:
- return "networks_AS%s" % self.ns.asn[0]
-
- def open(self):
- pass
-
- def close(self):
- pass
-
- def network(self, network):
- print(network)
-
-
-class IpsetOutputFormatter(OutputFormatter):
- """
- For nftables
- """
- def open(self):
- print("create %s hash:net family inet hashsize 1024 maxelem 65536" % self.name)
-
- def network(self, network):
- print("add %s %s" % (self.name, network))
-
-
-class NftablesOutputFormatter(OutputFormatter):
- """
- For nftables
- """
- def open(self):
- print("define %s = {" % self.name)
-
- def close(self):
- print("}")
-
- def network(self, network):
- print(" %s," % network)
-
-
-class XTGeoIPOutputFormatter(OutputFormatter):
- """
- Formats the output in that way, that it can be loaded by
- the xt_geoip kernel module from xtables-addons.
- """
- def network(self, network):
- n = ipaddress.ip_network("%s" % network)
-
- for address in (n.network_address, n.broadcast_address):
- bytes = socket.inet_pton(
- socket.AF_INET6 if address.version == 6 else socket.AF_INET,
- "%s" % address,
- )
-
- os.write(1, bytes)
-
-
class CLI(object):
- output_formats = {
- "ipset" : IpsetOutputFormatter,
- "list" : OutputFormatter,
- "nftables" : NftablesOutputFormatter,
- "xt_geoip" : XTGeoIPOutputFormatter,
- }
-
def parse_cli(self):
parser = argparse.ArgumentParser(
description=_("Location Database Command Line Interface"),
@@ -193,8 +115,8 @@ class CLI(object):
)
list_networks_by_as.add_argument("asn", nargs=1, type=int)
list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4"))
- list_networks_by_as.add_argument("--output-format",
- choices=self.output_formats.keys(), default="list")
+ list_networks_by_as.add_argument("--format",
+ choices=location.export.formats.keys(), default="list")
list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as)
# List all networks in a country
@@ -203,8 +125,8 @@ class CLI(object):
)
list_networks_by_cc.add_argument("country_code", nargs=1)
list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4"))
- list_networks_by_cc.add_argument("--output-format",
- choices=self.output_formats.keys(), default="list")
+ list_networks_by_cc.add_argument("--format",
+ choices=location.export.formats.keys(), default="list")
list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc)
# List all networks with flags
@@ -221,10 +143,23 @@ class CLI(object):
action="store_true", help=_("Anycasts"),
)
list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
- list_networks_by_flags.add_argument("--output-format",
- choices=self.output_formats.keys(), default="list")
+ list_networks_by_flags.add_argument("--format",
+ choices=location.export.formats.keys(), default="list")
list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
+ # Export
+ export = subparsers.add_parser("export",
+ help=_("Exports data in many formats to load it into packet filters"),
+ )
+ export.add_argument("--format", help=_("Output format"),
+ choices=location.export.formats.keys(), default="list")
+ export.add_argument("--directory", help=_("Output directory"), required=True)
+ export.add_argument("--family",
+ help=_("Specify address family"), choices=("ipv6", "ipv4"),
+ )
+ export.add_argument("objects", nargs="+", help=_("List country codes or ASNs to export"))
+ export.set_defaults(func=self.handle_export)
+
args = parser.parse_args()
# Configure logging
@@ -494,25 +429,36 @@ class CLI(object):
def __get_output_formatter(self, ns):
try:
- cls = self.output_formats[ns.output_format]
+ cls = location.export.formats[ns.format]
except KeyError:
- cls = OutputFormatter
+ cls = location.export.OutputFormatter
- return cls(ns)
+ return cls
def handle_list_networks_by_as(self, db, ns):
- with self.__get_output_formatter(ns) as f:
- for asn in ns.asn:
- # Print all matching networks
- for n in db.search_networks(asn=asn, family=ns.family):
- f.network(n)
+ writer = self.__get_output_formatter(ns)
+
+ for asn in ns.asn:
+ f = writer(sys.stdout, prefix="AS%s" % asn)
+
+ # Print all matching networks
+ for n in db.search_networks(asn=asn, family=ns.family):
+ f.write(n)
+
+ f.finish()
def handle_list_networks_by_cc(self, db, ns):
- with self.__get_output_formatter(ns) as f:
- for country_code in ns.country_code:
- # Print all matching networks
- for n in db.search_networks(country_code=country_code, family=ns.family):
- f.network(n)
+ writer = self.__get_output_formatter(ns)
+
+ for country_code in ns.country_code:
+ # Open standard output
+ f = writer(sys.stdout, prefix=country_code)
+
+ # Print all matching networks
+ for n in db.search_networks(country_code=country_code, family=ns.family):
+ f.write(n)
+
+ f.finish()
def handle_list_networks_by_flags(self, db, ns):
flags = 0
@@ -529,9 +475,49 @@ class CLI(object):
if not flags:
raise ValueError(_("You must at least pass one flag"))
- with self.__get_output_formatter(ns) as f:
- for n in db.search_networks(flags=flags, family=ns.family):
- f.network(n)
+ writer = self.__get_output_formatter(ns)
+ f = writer(sys.stdout, prefix="custom")
+
+ for n in db.search_networks(flags=flags, family=ns.family):
+ f.write(n)
+
+ f.finish()
+
+ def handle_export(self, db, ns):
+ countries, asns = [], []
+
+ # Translate family
+ if ns.family == "ipv6":
+ families = [ socket.AF_INET6 ]
+ elif ns.family == "ipv4":
+ families = [ socket.AF_INET ]
+ else:
+ families = [ socket.AF_INET6, socket.AF_INET ]
+
+ for object in ns.objects:
+ m = re.match("^AS(\d+)$", object)
+ if m:
+ object = int(m.group(1))
+
+ asns.append(object)
+
+ elif location.country_code_is_valid(object) \
+ or object in ("A1", "A2", "A3"):
+ countries.append(object)
+
+ else:
+ log.warning("Invalid argument: %s" % object)
+ continue
+
+ if not countries and not asns:
+ log.error("Nothing to export")
+ return 2
+
+ # Select the output format
+ writer = self.__get_output_formatter(ns)
+
+ e = location.export.Exporter(db, writer)
+ e.export(ns.directory, countries=countries, asns=asns, families=families)
def main():

383
src/patches/libloc-0.9.1-move-location-downloader-functionality-into-location-query.patch
View File

@@ -1,383 +0,0 @@
commit a6f1e3463d4c2085c203ad58072d7a154b663904
Author: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed Jun 3 17:06:13 2020 +0000
Move location-downloader functionality into location-query
The commands are very long and confusion. Hence we merge this
all into one command.
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
diff --git a/Makefile.am b/Makefile.am
index 31869e0..c0b1300 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -146,6 +146,7 @@ CLEANFILES += \
dist_pkgpython_PYTHON = \
src/python/__init__.py \
src/python/database.py \
+ src/python/downloader.py \
src/python/i18n.py \
src/python/importer.py \
src/python/logger.py
@@ -234,19 +235,16 @@ uninstall-perl:
$(DESTDIR)/$(prefix)/man/man3/Location.3pm
bin_SCRIPTS = \
- src/python/location-downloader \
src/python/location-exporter \
src/python/location-importer \
src/python/location-query
EXTRA_DIST += \
- src/python/location-downloader.in \
src/python/location-exporter.in \
src/python/location-importer.in \
src/python/location-query.in
CLEANFILES += \
- src/python/location-downloader \
src/python/location-exporter \
src/python/location-importer \
src/python/location-query
diff --git a/src/python/location-downloader.in b/src/python/downloader.py
similarity index 60%
rename from src/python/location-downloader.in
rename to src/python/downloader.py
index bf0d682..c9e6e00 100644
--- a/src/python/location-downloader.in
+++ b/src/python/downloader.py
@@ -3,7 +3,7 @@
# #
# libloc - A library to determine the location of someone on the Internet #
# #
-# Copyright (C) 2019 IPFire Development Team <info@ipfire.org> #
+# Copyright (C) 2020 IPFire Development Team <info@ipfire.org> #
# #
# This library is free software; you can redistribute it and/or #
# modify it under the terms of the GNU Lesser General Public #
@@ -17,24 +17,18 @@
# #
###############################################################################
-import argparse
-import datetime
import logging
import lzma
import os
import random
-import shutil
import stat
-import sys
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request
-# Load our location module
-import location
-from location.i18n import _
+from _location import Database, DATABASE_VERSION_LATEST
DATABASE_FILENAME = "location.db.xz"
MIRRORS = (
@@ -46,9 +40,11 @@ log = logging.getLogger("location.downloader")
log.propagate = 1
class Downloader(object):
- def __init__(self, version, mirrors):
+ def __init__(self, version=DATABASE_VERSION_LATEST, mirrors=None):
self.version = version
- self.mirrors = list(mirrors)
+
+ # Set mirrors or use defaults
+ self.mirrors = list(mirrors or MIRRORS)
# Randomize mirrors
random.shuffle(self.mirrors)
@@ -117,9 +113,10 @@ class Downloader(object):
return res
- def download(self, url, public_key, timestamp=None, tmpdir=None, **kwargs):
- headers = {}
+ def download(self, public_key, timestamp=None, tmpdir=None, **kwargs):
+ url = "%s/%s" % (self.version, DATABASE_FILENAME)
+ headers = {}
if timestamp:
headers["If-Modified-Since"] = timestamp.strftime(
"%a, %d %b %Y %H:%M:%S GMT",
@@ -191,7 +188,7 @@ class Downloader(object):
"""
log.debug("Opening downloaded database at %s" % f.name)
- db = location.Database(f.name)
+ db = Database(f.name)
# Database is not recent
if timestamp and db.created_at < timestamp.timestamp():
@@ -208,141 +205,3 @@ class Downloader(object):
return False
return True
-
-
-class CLI(object):
- def __init__(self):
- # Which version are we downloading?
- self.version = location.DATABASE_VERSION_LATEST
-
- self.downloader = Downloader(version=self.version, mirrors=MIRRORS)
-
- def parse_cli(self):
- parser = argparse.ArgumentParser(
- description=_("Location Downloader Command Line Interface"),
- )
- subparsers = parser.add_subparsers()
-
- # Global configuration flags
- parser.add_argument("--debug", action="store_true",
- help=_("Enable debug output"))
- parser.add_argument("--quiet", action="store_true",
- help=_("Enable quiet mode"))
-
- # version
- parser.add_argument("--version", action="version",
- version="%(prog)s @VERSION@")
-
- # database
- parser.add_argument("--database", "-d",
- default="@databasedir@/database.db", help=_("Path to database"),
- )
-
- # public key
- parser.add_argument("--public-key", "-k",
- default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
- )
-
- # Update
- update = subparsers.add_parser("update", help=_("Update database"))
- update.set_defaults(func=self.handle_update)
-
- # Verify
- verify = subparsers.add_parser("verify",
- help=_("Verify the downloaded database"))
- verify.set_defaults(func=self.handle_verify)
-
- args = parser.parse_args()
-
- # Configure logging
- if args.debug:
- location.logger.set_level(logging.DEBUG)
- elif args.quiet:
- location.logger.set_level(logging.WARNING)
-
- # Print usage if no action was given
- if not "func" in args:
- parser.print_usage()
- sys.exit(2)
-
- return args
-
- def run(self):
- # Parse command line arguments
- args = self.parse_cli()
-
- # Call function
- ret = args.func(args)
-
- # Return with exit code
- if ret:
- sys.exit(ret)
-
- # Otherwise just exit
- sys.exit(0)
-
- def handle_update(self, ns):
- # Fetch the timestamp we need from DNS
- t = location.discover_latest_version(self.version)
-
- # Parse timestamp into datetime format
- timestamp = datetime.datetime.fromtimestamp(t) if t else None
-
- # Open database
- try:
- db = location.Database(ns.database)
-
- # Check if we are already on the latest version
- if timestamp and db.created_at >= timestamp.timestamp():
- log.info("Already on the latest version")
- return
-
- except FileNotFoundError as e:
- db = None
-
- # Download the database into the correct directory
- tmpdir = os.path.dirname(ns.database)
-
- # Try downloading a new database
- try:
- t = self.downloader.download("%s/%s" % (self.version, DATABASE_FILENAME),
- public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
-
- # If no file could be downloaded, log a message
- except FileNotFoundError as e:
- log.error("Could not download a new database")
- return 1
-
- # If we have not received a new file, there is nothing to do
- if not t:
- return 3
-
- # Move temporary file to destination
- shutil.move(t.name, ns.database)
-
- return 0
-
- def handle_verify(self, ns):
- try:
- db = location.Database(ns.database)
- except FileNotFoundError as e:
- log.error("%s: %s" % (ns.database, e))
- return 127
-
- # Verify the database
- with open(ns.public_key, "r") as f:
- if not db.verify(f):
- log.error("Could not verify database")
- return 1
-
- # Success
- log.debug("Database successfully verified")
- return 0
-
-
-def main():
- # Run the command line interface
- c = CLI()
- c.run()
-
-main()
diff --git a/src/python/location-query.in b/src/python/location-query.in
index dfdff8c..0291786 100644
--- a/src/python/location-query.in
+++ b/src/python/location-query.in
@@ -18,16 +18,23 @@
###############################################################################
import argparse
+import datetime
import ipaddress
+import logging
import os
+import shutil
import socket
import sys
import time
# Load our location module
import location
+import location.downloader
from location.i18n import _
+# Setup logging
+log = logging.getLogger("location")
+
# Output formatters
class OutputFormatter(object):
@@ -157,6 +164,15 @@ class CLI(object):
dump.add_argument("output", nargs="?", type=argparse.FileType("w"))
dump.set_defaults(func=self.handle_dump)
+ # Update
+ update = subparsers.add_parser("update", help=_("Update database"))
+ update.set_defaults(func=self.handle_update)
+
+ # Verify
+ verify = subparsers.add_parser("verify",
+ help=_("Verify the downloaded database"))
+ verify.set_defaults(func=self.handle_verify)
+
# Get AS
get_as = subparsers.add_parser("get-as",
help=_("Get information about one or multiple Autonomous Systems"),
@@ -423,6 +439,59 @@ class CLI(object):
for a in db.search_as(query):
print(a)
+ def handle_update(self, db, ns):
+ # Fetch the timestamp we need from DNS
+ t = location.discover_latest_version()
+
+ # Parse timestamp into datetime format
+ timestamp = datetime.datetime.fromtimestamp(t) if t else None
+
+ # Check the version of the local database
+ if db and timestamp and db.created_at >= timestamp.timestamp():
+ log.info("Already on the latest version")
+ return
+
+ # Download the database into the correct directory
+ tmpdir = os.path.dirname(ns.database)
+
+ # Create a downloader
+ d = location.downloader.Downloader()
+
+ # Try downloading a new database
+ try:
+ t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
+
+ # If no file could be downloaded, log a message
+ except FileNotFoundError as e:
+ log.error("Could not download a new database")
+ return 1
+
+ # If we have not received a new file, there is nothing to do
+ if not t:
+ return 3
+
+ # Move temporary file to destination
+ shutil.move(t.name, ns.database)
+
+ return 0
+
+ def handle_verify(self, ns):
+ try:
+ db = location.Database(ns.database)
+ except FileNotFoundError as e:
+ log.error("%s: %s" % (ns.database, e))
+ return 127
+
+ # Verify the database
+ with open(ns.public_key, "r") as f:
+ if not db.verify(f):
+ log.error("Could not verify database")
+ return 1
+
+ # Success
+ log.debug("Database successfully verified")
+ return 0
+
def __get_output_formatter(self, ns):
try:
cls = self.output_formats[ns.output_format]
diff --git a/src/python/locationmodule.c b/src/python/locationmodule.c
index a04cab7..5b72be9 100644
--- a/src/python/locationmodule.c
+++ b/src/python/locationmodule.c
@@ -50,9 +50,9 @@ static PyObject* set_log_level(PyObject* m, PyObject* args) {
}
static PyObject* discover_latest_version(PyObject* m, PyObject* args) {
- unsigned int version = 0;
+ unsigned int version = LOC_DATABASE_VERSION_LATEST;
- if (!PyArg_ParseTuple(args, "i", &version))
+ if (!PyArg_ParseTuple(args, "|i", &version))
return NULL;
time_t t = 0;

22
src/patches/libloc-0.9.1-remove-accidently-commited-hacks-for-debian.patch
View File

@@ -1,22 +0,0 @@
commit 6bfde1447d237d2a345b99677c5b74e54cbd5739
Author: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu Jun 4 10:37:50 2020 +0000
Makefile: Remove accidentially committed hacks for Debian
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
diff --git a/Makefile.am b/Makefile.am
index ef57551..c75839c 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -37,9 +37,6 @@ LIBLOC_CURRENT=0
LIBLOC_REVISION=0
LIBLOC_AGE=0
-pythondir = $(prefix)/lib/python3/dist-packages
-pyexecdir = $(prefix)/lib/python$(PYTHON_VERSION)/lib-dynload
-
DISTCHECK_CONFIGURE_FLAGS = \
--with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir)

37
src/patches/libloc-0.9.1-remove-python-path-overrides-for-debian.patch
View File

@@ -1,37 +0,0 @@
commit 9df8db2ae6268b0901961625fd27b4dde1ed451f
Author: Michael Tremer <michael.tremer@ipfire.org>
Date: Mon Jun 1 18:23:50 2020 +0000
Makefile: Remove Python path overrides for Debian
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
diff --git a/Makefile.am b/Makefile.am
index 570ec3a..31869e0 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -54,10 +54,6 @@ SED_PROCESS = \
databasedir = $(localstatedir)/lib/location
pkgconfigdir = $(libdir)/pkgconfig
-# XXX hardcode path for Debian
-pythondir = $(prefix)/lib/python3/dist-packages
-pyexecdir = $(prefix)/lib/python$(PYTHON_VERSION)/lib-dynload
-
# Overwrite Python path
pkgpythondir = $(pythondir)/location
diff --git a/debian/libloc.install b/debian/libloc.install
index eb3b49a..30bbeca 100644
--- a/debian/libloc.install
+++ b/debian/libloc.install
@@ -2,8 +2,7 @@ usr/bin/location-downloader
usr/bin/location-exporter
usr/bin/location-query
usr/lib/*/libloc.so.*
-usr/lib/python3*/dist-packages
-usr/lib/python3*/lib-dynload/*.so
+usr/lib/python3*/site-packages
src/systemd/*.service /lib/systemd/system/
src/systemd/*.timer /lib/systemd/system/
var/lib/location/signing-key.pem

111
src/patches/libloc-0.9.1-rename-location-query-to-location.patch
View File

@@ -1,111 +0,0 @@
commit 1d237439676e8b9ee10a6dde2c64f5ba3a057210
Author: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed Jun 3 17:21:31 2020 +0000
Rename location-query(8) to location(8)
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
diff --git a/Makefile.am b/Makefile.am
index bf204d8..59870b1 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -238,19 +238,19 @@ uninstall-perl:
$(DESTDIR)/$(prefix)/man/man3/Location.3pm
bin_SCRIPTS = \
+ src/python/location \
src/python/location-exporter \
- src/python/location-importer \
- src/python/location-query
+ src/python/location-importer
EXTRA_DIST += \
+ src/python/location.in \
src/python/location-exporter.in \
- src/python/location-importer.in \
- src/python/location-query.in
+ src/python/location-importer.in
CLEANFILES += \
+ src/python/location \
src/python/location-exporter \
- src/python/location-importer \
- src/python/location-query
+ src/python/location-importer
# ------------------------------------------------------------------------------
@@ -374,7 +374,7 @@ src_test_signature_LDADD = \
# ------------------------------------------------------------------------------
MANPAGES = \
- man/location-query.8
+ man/location.8
MANPAGES_TXT = $(patsubst %.8,%.txt,$(MANPAGES))
MANPAGES_HTML = $(patsubst %.txt,%.html,$(MANPAGES_TXT))
diff --git a/man/location-query.txt b/man/location.txt
similarity index 84%
rename from man/location-query.txt
rename to man/location.txt
index acb43cd..672c2b2 100644
--- a/man/location-query.txt
+++ b/man/location.txt
@@ -1,21 +1,21 @@
-= location-query(8)
+= location(8)
== NAME
-location-query - Query the location database
+location - Query the location database
== SYNOPSIS
[verse]
-`location-query lookup ADDRESS [ADDRESS...]`
-`location-query get-as ASN [ASN...]`
-`location-query search-as STRING`
-`location-query update`
-`location-query verify`
-`location-query list-networks-by-as ASN`
-`location-query list-networks-by-cc COUNTRY_CODE`
-`location-query list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]`
+`location lookup ADDRESS [ADDRESS...]`
+`location get-as ASN [ASN...]`
+`location search-as STRING`
+`location update`
+`location verify`
+`location list-networks-by-as ASN`
+`location list-networks-by-cc COUNTRY_CODE`
+`location list-networks-by-flags [--anonymous-proxy|--satellite-provider|--anycast]`
== DESCRIPTION
-The `location-query` retrieves information from the location database.
+`location` retrieves information from the location database.
This data can be used to determine someone's location on the Internet
and for building firewall rulesets to block access from certain ASes
or countries.
@@ -93,7 +93,7 @@ or countries.
Shows the program's version and exists.
== EXIT CODES
-The 'location-query' command will normally exit with code zero.
+The 'location' command will normally exit with code zero.
If there has been a problem and the requested action could not be performed,
the exit code is unequal to zero.
diff --git a/src/python/location-query.in b/src/python/location.in
similarity index 99%
rename from src/python/location-query.in
rename to src/python/location.in
index 0291786..10618e2 100644
--- a/src/python/location-query.in
+++ b/src/python/location.in
@@ -248,7 +248,7 @@ class CLI(object):
try:
db = location.Database(args.database)
except FileNotFoundError as e:
- sys.stderr.write("location-query: Could not open database %s: %s\n" \
+ sys.stderr.write("location: Could not open database %s: %s\n" \
% (args.database, e))
sys.exit(1)