Skip to content
Snippets Groups Projects
Commit 518d69f8 authored by Rajmund Hruška's avatar Rajmund Hruška
Browse files

Feature: Parse new whois file format. (Redmine issue: #6227)

parent ddadd84c
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
conf/cron/mentat-netmngr-py.cron
+ 1
− 1
View file @ 518d69f8
......@@ -11,4 +11,4 @@
#
# Run every day at 6am.
0 6 * * * mentat /usr/local/bin/mentat-netmngr.py --regular --command update --whois-file /var/mentat/whois-negistry.json
0 6 * * * mentat /usr/local/bin/mentat-netmngr.py --regular --command update --whois-file /var/mentat/whois-negistry.json --source negistry
lib/mentat/datatype/internal.py
+ 15
− 3
View file @ 518d69f8
......@@ -318,8 +318,11 @@ def t_network_record(val, source = None):
record['first'] = record['nrobj'].low()
record['last'] = record['nrobj'].high()
if 'netname' in val:
record['netname'] = val['netname']
if 'netnames' in val:
if isinstance(val['netnames'], list):
record['netname'] = ', '.join(val['netnames'])
else:
record['netname'] = val['netnames']
if 'description' in val:
if isinstance(val['description'], list):
......@@ -339,7 +342,16 @@ def t_network_record(val, source = None):
record['source'] = source
if 'resolved_abuses' in val:
record['resolved_abuses'] = val['resolved_abuses']
if 'low' in val['resolved_abuses']:
record['resolved_abuses'] = val['resolved_abuses']['low']
elif 'medium' in val['resolved_abuses']:
record['resolved_abuses'] = val['resolved_abuses']['medium']
elif 'high' in val['resolved_abuses']:
record['resolved_abuses'] = val['resolved_abuses']['high']
elif 'critical' in val['resolved_abuses']:
record['resolved_abuses'] = val['resolved_abuses']['critical']
else:
record['resolved_abuses'] = []
if 'emails' in val:
record['emails'] = val['emails']
......
lib/mentat/module/netmngr.py
+ 71
− 76
View file @ 518d69f8
......@@ -62,6 +62,11 @@ Custom command line options
*Type:* ``string``, *default:* ``None``
``--source``
Origin of the whois file.
*Type:* ``string``, *default:* ``whois``
Custom config file options
^^^^^^^^^^^^^^^^^^^^^^^^^^
......@@ -86,8 +91,8 @@ Exception file format
The exception file is an ordinary text file containing single IPv(4|6)
address|network|range on each line. Blank lines and lines beginning with ``#``
are ignored. Whois exception files are very easy to be generated and they are meant
for specifiing whois resolving exceptions. For example you may use it to describe
hosts with addresses from the domain of one particular abuse group, but actualy
for specifying whois resolving exceptions. For example you may use it to describe
hosts with addresses from the domain of one particular abuse group, but actually
belonging to different group. This might be the case of routers belonging to service
provider but residing within the network address space of the customer. Another
example may be nodes of some cloud computing service that have addresses from
......@@ -106,59 +111,65 @@ tool, which is an internal custom copy of relevant RIPE whois data. It is JSON b
format. Following example content describes multiple valid syntaxes for describing
network records::
{
"__whois_about__" "Generated by Custom tool(tm) at 2018-01-25 01:00:01.765858.",
"__whois_type__": "custom",
[
# Option 1: Pass IP4 start and end addresses
"78.128.128.0 - 78.128.255.255": {
{
"primary_key": "78.128.128.0 - 78.128.255.255",
"ip4_start": "78.128.128.0",
"ip4_end": "78.128.255.255",
"netname": "CZ-TEN-34-20070410",
"resolved_abuses": [
"netnames": ["CZ-TEN-34-20070410"],
"resolved_abuses": {
"low": [
"abuse@cesnet.cz"
]
]
}
},
# Option 2: Pass network CIDR or range and type
"78.128.212.64 - 78.128.212.127": {
{
"primary_key": "78.128.212.64 - 78.128.212.127",
"network": "78.128.212.64/26",
"type": "ipv4",
"netname": "CESNET-HSM4",
"netnames": ["CESNET-HSM4"],
"descr": [
"CESNET, z.s.p.o.",
"Ostrava"
],
"resolved_abuses": [
"abuse@cesnet.cz"
]
"resolved_abuses": {
"low": [
"abuse@cesnet.cz"
]
}
},
# Option 3: Pass IP6 address and prefix
"2001:718::/29": {
{
"primary_key": "2001:718::/29",
"ip6_addr": "2001:718::",
"ip6_prefix": 29,
"netname": "CZ-TEN-34-20010521",
"description": "Extensive network description",
"resolved_abuses": [
"netnames": ["CZ-TEN-34-20010521"],
"descr": ["Extensive network description"],
"resolved_abuses": {
"low": [
"abuse@cesnet.cz"
]
]
}
},
# Option 4: Pass only IPv(4|6) network|range without type for autodetection (slower)
"2001:718::/29": {
{
"primary_key": "2001:718::/29",
"network": "2001:718::/29",
"netname": "CZ-TEN-34-20010521",
"resolved_abuses": [
"abuse@cesnet.cz"
]
"netnames": ["CZ-TEN-34-20010521"],
"resolved_abuses": {
"low": [
"abuse@cesnet.cz"
]
}
},
...
}
]
The ``__whois_type__`` meta attribute has a deeper meaning. It is used throughout
the database records to mark their origin. It is possible to keep track which
tool provided which whois network records.
The ``netname``, ``descr`` and ``description`` attributes are optional and will
be used/stored into database, if present.
......@@ -166,16 +177,6 @@ be used/stored into database, if present.
The ``resolved_abuses`` attribute is mandatory and must contain list of abuse groups
(abuse contacts) for that particular network record.
For the purposes of cooperation with the *Negistry* tool following header is also
recognized and automatically sets up the ``__whois_type__`` header to value
``negistry``::
{
"__negistry_about__": "Generated by Negistry at 2018-01-25 01:00:01.765858.",
...
}
"""
......@@ -199,10 +200,6 @@ from mentat.datatype.sqldb import GroupModel, NetworkModel, SettingsReportingMod
networkmodel_from_typeddict
WHOIS_KEY_TYPE = '__whois_type__'
WHOIS_KEY_ABOUT_GENERIC = '__whois_about__'
WHOIS_KEY_ABOUT_NEGISTRY = '__negistry_about__'
WHOIS_TYPE_GENERIC = 'whois'
WHOIS_TYPE_NEGISTRY = 'negistry'
......@@ -222,6 +219,7 @@ class MentatNetmngrScript(mentat.script.fetcher.FetcherScript):
# List of configuration keys.
CONFIG_WHOIS_FILE = 'whois_file'
CONFIG_WHOIS_SOURCE = 'source'
CONFIG_EXCEPTION_FILES = 'exception_files'
......@@ -267,6 +265,13 @@ class MentatNetmngrScript(mentat.script.fetcher.FetcherScript):
help = 'path to reference whois file containing network data'
)
arggroup_script.add_argument(
'--source',
type = str,
default = WHOIS_TYPE_GENERIC,
help = 'origin of the whois file'
)
return argparser
def _init_config(self, cfgs, **kwargs):
......@@ -284,8 +289,9 @@ class MentatNetmngrScript(mentat.script.fetcher.FetcherScript):
:rtype: dict
"""
cfgs = (
(self.CONFIG_WHOIS_FILE, None),
(self.CONFIG_EXCEPTION_FILES, None)
(self.CONFIG_WHOIS_FILE, None),
(self.CONFIG_WHOIS_SOURCE, WHOIS_TYPE_GENERIC),
(self.CONFIG_EXCEPTION_FILES, None)
) + cfgs
return super()._init_config(cfgs, **kwargs)
......@@ -384,28 +390,14 @@ class MentatNetmngrScript(mentat.script.fetcher.FetcherScript):
:rtype: dict
"""
try:
whois_file_data = self.json_load(whois_file)
with open(whois_file, 'r') as jsf:
json_data = jsf.read()
whois_file_data = json.loads(json_data)
except:
raise pyzenkit.zenscript.ZenScriptException("Invalid whois file '{}', expected JSON formated file".format(whois_file))
# Each valid whois file must contain either '__negistry_about__' or
# '__whois_about__' header. Attempt to find them
whois_file_about = None
whois_file_type = WHOIS_TYPE_GENERIC
for meta in (WHOIS_KEY_ABOUT_GENERIC, WHOIS_KEY_ABOUT_NEGISTRY):
if meta in whois_file_data:
whois_file_about = whois_file_data[meta]
del whois_file_data[meta]
if meta == WHOIS_KEY_ABOUT_NEGISTRY:
whois_file_type = WHOIS_TYPE_NEGISTRY
elif WHOIS_KEY_TYPE in whois_file_data:
whois_file_type = whois_file_data[WHOIS_KEY_TYPE]
del whois_file_data[WHOIS_KEY_TYPE]
break
if not whois_file_about:
raise pyzenkit.zenscript.ZenScriptException("Invalid whois file '{}', missing meta information header".format(whois_file))
self.logger.info("Loaded reference whois file '%s :: %s' (%s)", whois_file, whois_file_type, whois_file_about)
whois_file_type = self.c(self.CONFIG_WHOIS_SOURCE)
self.logger.info("Loaded reference whois file '%s :: %s'", whois_file, whois_file_type)
return (whois_file_type, whois_file_data)
@staticmethod
......@@ -420,10 +412,10 @@ class MentatNetmngrScript(mentat.script.fetcher.FetcherScript):
:rtype: dict
"""
processed_data = collections.defaultdict(dict)
for network_data in whois_file_data.values():
for network_data in whois_file_data:
nwr = mentat.datatype.internal.t_network_record(network_data, source = whois_file_type)
nwrkey = nwr['network']
for abuse_group in network_data['resolved_abuses']:
for abuse_group in nwr['resolved_abuses']:
processed_data[abuse_group][nwrkey] = nwr
return processed_data
......@@ -602,7 +594,9 @@ class MentatNetmngrScript(mentat.script.fetcher.FetcherScript):
if match:
exc = {
'network': match.group(1),
'resolved_abuses': [abuse]
'resolved_abuses': {
'low': [abuse]
}
}
exc = mentat.datatype.internal.t_network_record(exc, source = 'exception')
self.logger.info("Found whois exception '%s' for abuse group '%s'", exc['network'], abuse)
......@@ -616,16 +610,17 @@ class MentatNetmngrScript(mentat.script.fetcher.FetcherScript):
:param str whois_file: path to target whois file.
:param dict exceptions: Structure containing whois exceptions.
"""
exception_dict = {
WHOIS_KEY_TYPE: 'exceptions',
WHOIS_KEY_ABOUT_GENERIC: 'Generated at {}'.format(str(datetime.datetime.now()))
}
exception_dict = []
for exc in exceptions:
exception_dict[exc['network']] = {
'network': exc['network'],
'type': exc['type'],
'resolved_abuses': list(exc['resolved_abuses'])
}
exception_dict.append(
{
'primary_key': exc['network'],
'type': exc['type'],
'resolved_abuses': {
'low': list(exc['resolved_abuses'])
}
}
)
with open(whois_file, 'w') as excfh:
json.dump(exception_dict, excfh, indent = 4, sort_keys = True)
self.logger.info("Saved '%d' whois exceptions into target file '%s'", len(exceptions), whois_file)
lib/mentat/services/test_whois.py
+ 47
− 30
View file @ 518d69f8
......@@ -28,30 +28,33 @@ import mentat.services.whois
CONTENT_WHOIS_NEGISTRY = """
{
"__negistry_about__": "Dummy Negistry file for unit tests",
"78.128.128.0 - 78.128.255.255": {
[
{
"primary_key": "78.128.128.0 - 78.128.255.255",
"ip4_start": "78.128.128.0",
"ip4_end": "78.128.255.255",
"resolved_abuses": [
"abuse@cesnet.cz"
]
"resolved_abuses": {
"low": [
"abuse@cesnet.cz"
]
}
}
}
]
"""
CONTENT_WHOIS_EXCEPTIONS = """
{
"__whois_about__": "Dummy whois exceptions file for unit tests",
"__whois_type__": "exceptions",
"78.128.214.67": {
[
{
"primary_key": "78.128.214.67",
"network": "78.128.214.67",
"resolved_abuses": [
"resolved_abuses": {
"low": [
"abuse@cuni.cz"
],
]
},
"type": "ipv4"
}
}
]
"""
FILE_WHOIS_NEGISTRY = '/var/tmp/unittest-whois-negistry.json'
......@@ -80,58 +83,72 @@ class TestMentatWhois(unittest.TestCase):
{
"id" : 1,
"source" : "negistry",
"netname" : "MUNI-01",
"netnames" : "MUNI-01",
"network" : "195.178.86.0-195.178.87.255",
"type" : "ipv4",
"resolved_abuses": ["abuse@muni.cz"]
"resolved_abuses": {
"low" : ["abuse@muni.cz"]
}
},
{
"id" : 2,
"source" : "negistry",
"netname" : "MUNI-02",
"netnames" : "MUNI-02",
"network" : "147.251.0.0-147.251.255.255",
"type" : "ipv4",
"resolved_abuses": ["abuse@muni.cz"]
"resolved_abuses": {
"low" : ["abuse@muni.cz"]
}
},
{
"id" : 3,
"source" : "negistry",
"netname" : "MUNI-03",
"netnames" : "MUNI-03",
"network" : "2001:718:800:5::/64",
"type" : "ipv6",
"resolved_abuses": ["abuse@muni.cz"]
"resolved_abuses": {
"low" : ["abuse@muni.cz"]
}
},
{
"id" : 4,
"source" : "negistry",
"netname" : "CESNET-01",
"netnames" : "CESNET-01",
"network" : "195.179.86.0-195.179.87.255",
"type" : "ipv4",
"resolved_abuses": ["abuse@cesnet.cz"]
"resolved_abuses": {
"low" : ["abuse@cesnet.cz"]
}
},
{
"id" : 5,
"source" : "negistry",
"netname" : "CESNET-00",
"netnames" : "CESNET-00",
"network" : "195.179.0.0-195.179.255.255",
"type" : "ipv4",
"resolved_abuses": ["abuse@cesnet.cz"]
"resolved_abuses": {
"low" : ["abuse@cesnet.cz"]
}
},
{
"id" : 6,
"source" : "negistry",
"netname" : "CESNET-02",
"netnames" : "CESNET-02",
"network" : "147.252.0.0-147.252.255.255",
"type" : "ipv4",
"resolved_abuses": ["abuse@cesnet.cz"]
"resolved_abuses": {
"low" : ["abuse@cesnet.cz"]
}
},
{
"id" : 7,
"source" : "negistry",
"netname" : "CESNET-03",
"netnames" : "CESNET-03",
"network" : "2001:718:1:6::/64",
"type" : "ipv6",
"resolved_abuses": ["abuse@cesnet.cz"]
"resolved_abuses": {
"low" : ["abuse@cesnet.cz"]
}
}
]
......@@ -172,9 +189,9 @@ class TestMentatWhois(unittest.TestCase):
for net in self.networks_raw:
obj = mentat.datatype.sqldb.NetworkModel()
obj.source = net['source']
obj.netname = net['netname']
obj.netname = net['netnames']
obj.network = net['network']
for grp in net['resolved_abuses']:
for grp in net['resolved_abuses']['low']:
if grp not in groups:
groups[grp] = mentat.datatype.sqldb.GroupModel(name = grp, source = 'manual', description = grp)
groups[grp].networks.append(obj)
......
lib/mentat/services/whois.py
+ 5
− 16
View file @ 518d69f8
......@@ -208,24 +208,11 @@ class FileWhoisModule(WhoisModule):
whois_file_data = json.loads(contents)
# Perform some amount of preprocessing.
whois_file_about = None
whois_file_type = WHOIS_TYPE_GENERIC
for meta in (WHOIS_KEY_ABOUT_GENERIC, WHOIS_KEY_ABOUT_NEGISTRY):
if meta in whois_file_data:
whois_file_about = whois_file_data[meta]
del whois_file_data[meta]
if meta == WHOIS_KEY_ABOUT_NEGISTRY:
whois_file_type = WHOIS_TYPE_NEGISTRY
elif WHOIS_KEY_TYPE in whois_file_data:
whois_file_type = whois_file_data[WHOIS_KEY_TYPE]
del whois_file_data[WHOIS_KEY_TYPE]
break
if not whois_file_about:
raise Exception("ERROR")
# Generate list of network record objects.
networks = []
for network_data in whois_file_data.values():
for network_data in whois_file_data:
nwr = mentat.datatype.internal.t_network_record(network_data, source = whois_file_type)
networks.append(nwr)
......@@ -262,11 +249,13 @@ class SqldbWhoisModule(WhoisModule):
for rec in records:
netw = mentat.datatype.internal.t_network_record({
'id': rec.id,
'netname': rec.netname,
'netnames': [rec.netname],
'source': rec.source,
'network': rec.network,
'description': rec.description,
'resolved_abuses': [rec.group.name]
'resolved_abuses': {
'low': [rec.group.name]
}
})
networks.append(netw)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment