From 89cf4f04db9ee5dda435553f0661a1d5af5e65a2 Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 8 Jan 2025 11:11:11 +0100 Subject: [PATCH 01/17] DEMO --- octopoes/bits/https_availability/bit.py | 15 ---- .../https_availability/__init__.py | 0 .../https_availability/https_availability.py | 5 +- octopoes/nibbles/https_availability/nibble.py | 50 +++++++++++++ .../octopoes/repositories/ooi_repository.py | 1 + .../test_https_availability_nibble.py | 71 +++++++++++++++++++ 6 files changed, 124 insertions(+), 18 deletions(-) delete mode 100644 octopoes/bits/https_availability/bit.py rename octopoes/{bits => nibbles}/https_availability/__init__.py (100%) rename octopoes/{bits => nibbles}/https_availability/https_availability.py (78%) create mode 100644 octopoes/nibbles/https_availability/nibble.py create mode 100644 octopoes/tests/integration/test_https_availability_nibble.py diff --git a/octopoes/bits/https_availability/bit.py b/octopoes/bits/https_availability/bit.py deleted file mode 100644 index 762dde5e3dc..00000000000 --- a/octopoes/bits/https_availability/bit.py +++ /dev/null @@ -1,15 +0,0 @@ -from bits.definitions import BitDefinition, BitParameterDefinition -from octopoes.models.ooi.network import IPAddress, IPPort -from octopoes.models.ooi.web import Website - -BIT = BitDefinition( - id="https-availability", - consumes=IPAddress, - parameters=[ - BitParameterDefinition(ooi_type=IPPort, relation_path="address"), - BitParameterDefinition( - ooi_type=Website, relation_path="ip_service.ip_port.address" - ), # we place the findings on the http websites - ], - module="bits.https_availability.https_availability", -) diff --git a/octopoes/bits/https_availability/__init__.py b/octopoes/nibbles/https_availability/__init__.py similarity index 100% rename from octopoes/bits/https_availability/__init__.py rename to octopoes/nibbles/https_availability/__init__.py diff --git a/octopoes/bits/https_availability/https_availability.py b/octopoes/nibbles/https_availability/https_availability.py similarity index 78% rename from octopoes/bits/https_availability/https_availability.py rename to octopoes/nibbles/https_availability/https_availability.py index 2abec18b340..b3386c684ff 100644 --- a/octopoes/bits/https_availability/https_availability.py +++ b/octopoes/nibbles/https_availability/https_availability.py @@ -1,13 +1,12 @@ from collections.abc import Iterator -from typing import Any from octopoes.models import OOI from octopoes.models.ooi.findings import Finding, KATFindingType -from octopoes.models.ooi.network import IPAddress, IPPort +from octopoes.models.ooi.network import IPPort from octopoes.models.ooi.web import Website -def run(input_ooi: IPAddress, additional_oois: list[IPPort | Website], config: dict[str, Any]) -> Iterator[OOI]: +def nibble() -> Iterator[OOI]: websites = [website for website in additional_oois if isinstance(website, Website)] open_ports = [port.port for port in additional_oois if isinstance(port, IPPort)] diff --git a/octopoes/nibbles/https_availability/nibble.py b/octopoes/nibbles/https_availability/nibble.py new file mode 100644 index 00000000000..df5c6f40867 --- /dev/null +++ b/octopoes/nibbles/https_availability/nibble.py @@ -0,0 +1,50 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models import Reference +from octopoes.models.ooi.network import IPAddress, IPPort +from octopoes.models.ooi.web import Website + + +def query(targets: list[Reference | None]) -> str: + def pull(statements: list[str]) -> str: + return f""" + {{ + :query {{ + :find [(pull ?ipaddress [*]) (pull ?ipport80 [*]) (pull ?ipport443 [*]) (pull ?website [*])] + :where [ + {" ".join(statements)} + ] + }} + }} + """ + + sgn = "".join(str(int(isinstance(target, Reference))) for target in targets) + if sgn == "1000": + return pull( + [ + f""" + [?ipaddress :object_type "IPAddressV4"] + [?ipaddress :IPAddressV4/primary_key "{str(targets[0])}"] + [?ipport80 :IPPort/address ?ipaddress] + [?ipport80 :IPPort/port 80] + (or + (and [?ipport443 :IPPort/address ?ipaddress][?ipport443 :IPPort/port 443]) + (and [(identity 0) ?ipport443]) + ) + [?ip_service :IPService/ip_port ?ipport80] + [?website :Website/ip_service ?ip_service] + """ + ] + ) + return "potato" + + +NIBBLE = NibbleDefinition( + id="https-availability", + signature=[ + NibbleParameter(object_type=IPAddress, parser="[*][?object_type == 'IPAddressV4'][]"), + NibbleParameter(object_type=IPPort, parser='[*][?"IPPort/port" == "80"][]'), + NibbleParameter(object_type=Website, parser="[*][?object_type == 'Website'][]"), + NibbleParameter(object_type=int, parser='[length([*][?"IPPort/port" == "443"])]'), + ], + query=query, +) diff --git a/octopoes/octopoes/repositories/ooi_repository.py b/octopoes/octopoes/repositories/ooi_repository.py index c171c42c19a..a67707765c9 100644 --- a/octopoes/octopoes/repositories/ooi_repository.py +++ b/octopoes/octopoes/repositories/ooi_repository.py @@ -927,6 +927,7 @@ def nibble_query( arguments = [None for _ in nibble.signature] query = nibble.query if isinstance(nibble.query, str) else nibble.query(arguments) data = self.session.client.query(query, valid_time) + breakpoint() objects = [ {self.parse_as(element.object_type, obj) for obj in search(element.parser, data)} for element in nibble.signature diff --git a/octopoes/tests/integration/test_https_availability_nibble.py b/octopoes/tests/integration/test_https_availability_nibble.py new file mode 100644 index 00000000000..168210d197c --- /dev/null +++ b/octopoes/tests/integration/test_https_availability_nibble.py @@ -0,0 +1,71 @@ +import os +from datetime import datetime +from unittest.mock import Mock + +import pytest +from nibbles.https_availability.nibble import NIBBLE as https_availability +from nibbles.runner import NibblesRunner + +from octopoes.core.service import OctopoesService +from octopoes.models import Reference +from octopoes.models.ooi.dns.zone import Hostname +from octopoes.models.ooi.network import IPAddressV4, IPPort, Network, Protocol +from octopoes.models.ooi.service import IPService, Service +from octopoes.models.ooi.web import HostnameHTTPURL, HTTPHeader, HTTPResource, WebScheme, Website + +if os.environ.get("CI") != "1": + pytest.skip("Needs XTDB multinode container.", allow_module_level=True) + +STATIC_IP = ".".join((4 * "1 ").split()) + + +def test_https_availability_query(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + nibbler = NibblesRunner( + xtdb_octopoes_service.ooi_repository, + xtdb_octopoes_service.origin_repository, + xtdb_octopoes_service.nibbler.nibble_repository, + ) + xtdb_octopoes_service.nibbler.disable() + nibbler.nibbles = {https_availability.id: https_availability} + + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + + hostname = Hostname(name="example.com", network=network.reference) + xtdb_octopoes_service.ooi_repository.save(hostname, valid_time) + + web_url = HostnameHTTPURL( + network=network.reference, netloc=hostname.reference, port=443, path="/", scheme=WebScheme.HTTP + ) + xtdb_octopoes_service.ooi_repository.save(web_url, valid_time) + + service = Service(name="https") + xtdb_octopoes_service.ooi_repository.save(service, valid_time) + + ip_address = IPAddressV4(address=STATIC_IP, network=network.reference) + xtdb_octopoes_service.ooi_repository.save(ip_address, valid_time) + + port = IPPort(port=80, address=ip_address.reference, protocol=Protocol.TCP) + xtdb_octopoes_service.ooi_repository.save(port, valid_time) + + port443 = IPPort(port=443, address=ip_address.reference, protocol=Protocol.TCP) + xtdb_octopoes_service.ooi_repository.save(port443, valid_time) + + ip_service = IPService(ip_port=port.reference, service=service.reference) + xtdb_octopoes_service.ooi_repository.save(ip_service, valid_time) + + website = Website(ip_service=ip_service.reference, hostname=hostname.reference) + xtdb_octopoes_service.ooi_repository.save(website, valid_time) + + resource = HTTPResource(website=website.reference, web_url=web_url.reference) + xtdb_octopoes_service.ooi_repository.save(resource, valid_time) + + header = HTTPHeader( + resource=resource.reference, key="strict-transport-security", value="max-age=21536000; includeSubDomains" + ) + xtdb_octopoes_service.ooi_repository.save(header, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + + args: list[Reference | None] = [ip_address.reference, None, None, None] + print(xtdb_octopoes_service.ooi_repository.nibble_query(ip_address, https_availability, valid_time, args)) From abb845d4c996cbb44fa331821bd1401d35d86e00 Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 8 Jan 2025 13:27:10 +0100 Subject: [PATCH 02/17] DEMO2 --- octopoes/.ci/docker-compose.yml | 2 +- .../https_availability/https_availability.py | 20 ++++++++----------- octopoes/nibbles/https_availability/nibble.py | 16 +++++++-------- .../octopoes/repositories/ooi_repository.py | 1 - .../test_https_availability_nibble.py | 17 ++++++++++------ 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/octopoes/.ci/docker-compose.yml b/octopoes/.ci/docker-compose.yml index d4e33b42e13..ae46e5cd8ae 100644 --- a/octopoes/.ci/docker-compose.yml +++ b/octopoes/.ci/docker-compose.yml @@ -17,7 +17,7 @@ services: args: ENVIRONMENT: dev context: . - command: pytest tests/integration --timeout=300 + command: pytest -s tests/integration/test_https_availability_nibble.py --timeout=300 depends_on: - xtdb - ci_octopoes diff --git a/octopoes/nibbles/https_availability/https_availability.py b/octopoes/nibbles/https_availability/https_availability.py index b3386c684ff..8f917e52cd9 100644 --- a/octopoes/nibbles/https_availability/https_availability.py +++ b/octopoes/nibbles/https_availability/https_availability.py @@ -2,19 +2,15 @@ from octopoes.models import OOI from octopoes.models.ooi.findings import Finding, KATFindingType -from octopoes.models.ooi.network import IPPort +from octopoes.models.ooi.network import IPAddressV4, IPPort from octopoes.models.ooi.web import Website -def nibble() -> Iterator[OOI]: - websites = [website for website in additional_oois if isinstance(website, Website)] - - open_ports = [port.port for port in additional_oois if isinstance(port, IPPort)] - if 80 in open_ports and 443 not in open_ports: +def nibble(ipv4: IPAddressV4, port80: IPPort, website: Website, port443s: int) -> Iterator[OOI]: + if port443s < 2: ft = KATFindingType(id="KAT-HTTPS-NOT-AVAILABLE") - for website in websites: - yield Finding( - ooi=website.reference, - finding_type=ft.reference, - description="HTTP port is open, but HTTPS port is not open", - ) + yield Finding( + ooi=website.reference, + finding_type=ft.reference, + description="HTTP port is open, but HTTPS port is not open", + ) diff --git a/octopoes/nibbles/https_availability/nibble.py b/octopoes/nibbles/https_availability/nibble.py index df5c6f40867..8edd3dd8a1a 100644 --- a/octopoes/nibbles/https_availability/nibble.py +++ b/octopoes/nibbles/https_availability/nibble.py @@ -1,6 +1,6 @@ from nibbles.definitions import NibbleDefinition, NibbleParameter from octopoes.models import Reference -from octopoes.models.ooi.network import IPAddress, IPPort +from octopoes.models.ooi.network import IPAddressV4, IPPort from octopoes.models.ooi.web import Website @@ -9,7 +9,7 @@ def pull(statements: list[str]) -> str: return f""" {{ :query {{ - :find [(pull ?ipaddress [*]) (pull ?ipport80 [*]) (pull ?ipport443 [*]) (pull ?website [*])] + :find [(pull ?ipaddress [*]) (pull ?ipport80 [*]) (pull ?website [*]) (count ?ipport443)] :where [ {" ".join(statements)} ] @@ -26,12 +26,12 @@ def pull(statements: list[str]) -> str: [?ipaddress :IPAddressV4/primary_key "{str(targets[0])}"] [?ipport80 :IPPort/address ?ipaddress] [?ipport80 :IPPort/port 80] + [?ip_service :IPService/ip_port ?ipport80] + [?website :Website/ip_service ?ip_service] (or (and [?ipport443 :IPPort/address ?ipaddress][?ipport443 :IPPort/port 443]) (and [(identity 0) ?ipport443]) ) - [?ip_service :IPService/ip_port ?ipport80] - [?website :Website/ip_service ?ip_service] """ ] ) @@ -39,12 +39,12 @@ def pull(statements: list[str]) -> str: NIBBLE = NibbleDefinition( - id="https-availability", + id="https_availability", signature=[ - NibbleParameter(object_type=IPAddress, parser="[*][?object_type == 'IPAddressV4'][]"), - NibbleParameter(object_type=IPPort, parser='[*][?"IPPort/port" == "80"][]'), + NibbleParameter(object_type=IPAddressV4, parser="[*][?object_type == 'IPAddressV4'][]"), + NibbleParameter(object_type=IPPort, parser="[*][?object_type == 'IPPort'][]"), NibbleParameter(object_type=Website, parser="[*][?object_type == 'Website'][]"), - NibbleParameter(object_type=int, parser='[length([*][?"IPPort/port" == "443"])]'), + NibbleParameter(object_type=int, parser='[*][-1][]'), ], query=query, ) diff --git a/octopoes/octopoes/repositories/ooi_repository.py b/octopoes/octopoes/repositories/ooi_repository.py index a67707765c9..c171c42c19a 100644 --- a/octopoes/octopoes/repositories/ooi_repository.py +++ b/octopoes/octopoes/repositories/ooi_repository.py @@ -927,7 +927,6 @@ def nibble_query( arguments = [None for _ in nibble.signature] query = nibble.query if isinstance(nibble.query, str) else nibble.query(arguments) data = self.session.client.query(query, valid_time) - breakpoint() objects = [ {self.parse_as(element.object_type, obj) for obj in search(element.parser, data)} for element in nibble.signature diff --git a/octopoes/tests/integration/test_https_availability_nibble.py b/octopoes/tests/integration/test_https_availability_nibble.py index 168210d197c..09e26ae07e0 100644 --- a/octopoes/tests/integration/test_https_availability_nibble.py +++ b/octopoes/tests/integration/test_https_availability_nibble.py @@ -7,8 +7,8 @@ from nibbles.runner import NibblesRunner from octopoes.core.service import OctopoesService -from octopoes.models import Reference from octopoes.models.ooi.dns.zone import Hostname +from octopoes.models.ooi.findings import Finding, KATFindingType from octopoes.models.ooi.network import IPAddressV4, IPPort, Network, Protocol from octopoes.models.ooi.service import IPService, Service from octopoes.models.ooi.web import HostnameHTTPURL, HTTPHeader, HTTPResource, WebScheme, Website @@ -48,9 +48,6 @@ def test_https_availability_query(xtdb_octopoes_service: OctopoesService, event_ port = IPPort(port=80, address=ip_address.reference, protocol=Protocol.TCP) xtdb_octopoes_service.ooi_repository.save(port, valid_time) - port443 = IPPort(port=443, address=ip_address.reference, protocol=Protocol.TCP) - xtdb_octopoes_service.ooi_repository.save(port443, valid_time) - ip_service = IPService(ip_port=port.reference, service=service.reference) xtdb_octopoes_service.ooi_repository.save(ip_service, valid_time) @@ -67,5 +64,13 @@ def test_https_availability_query(xtdb_octopoes_service: OctopoesService, event_ event_manager.complete_process_events(xtdb_octopoes_service) - args: list[Reference | None] = [ip_address.reference, None, None, None] - print(xtdb_octopoes_service.ooi_repository.nibble_query(ip_address, https_availability, valid_time, args)) + assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 1 + assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 1 + + port443 = IPPort(port=443, address=ip_address.reference, protocol=Protocol.TCP) + xtdb_octopoes_service.ooi_repository.save(port443, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 0 + assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 0 From 6c51943591f84e45ee334175411d6a9169ac0fca Mon Sep 17 00:00:00 2001 From: Benny Date: Thu, 9 Jan 2025 12:10:23 +0100 Subject: [PATCH 03/17] Make precommit happy --- .../https_availability/https_availability.py | 4 +- octopoes/nibbles/https_availability/nibble.py | 37 ++++++++++++------- octopoes/nibbles/runner.py | 3 +- .../test_https_availability_nibble.py | 9 +---- 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/octopoes/nibbles/https_availability/https_availability.py b/octopoes/nibbles/https_availability/https_availability.py index 8f917e52cd9..f88781f7cc2 100644 --- a/octopoes/nibbles/https_availability/https_availability.py +++ b/octopoes/nibbles/https_availability/https_availability.py @@ -2,11 +2,11 @@ from octopoes.models import OOI from octopoes.models.ooi.findings import Finding, KATFindingType -from octopoes.models.ooi.network import IPAddressV4, IPPort +from octopoes.models.ooi.network import IPAddress, IPPort from octopoes.models.ooi.web import Website -def nibble(ipv4: IPAddressV4, port80: IPPort, website: Website, port443s: int) -> Iterator[OOI]: +def nibble(ipv4: IPAddress, port80: IPPort, website: Website, port443s: int) -> Iterator[OOI]: if port443s < 2: ft = KATFindingType(id="KAT-HTTPS-NOT-AVAILABLE") yield Finding( diff --git a/octopoes/nibbles/https_availability/nibble.py b/octopoes/nibbles/https_availability/nibble.py index 8edd3dd8a1a..b1720698bab 100644 --- a/octopoes/nibbles/https_availability/nibble.py +++ b/octopoes/nibbles/https_availability/nibble.py @@ -1,6 +1,6 @@ from nibbles.definitions import NibbleDefinition, NibbleParameter from octopoes.models import Reference -from octopoes.models.ooi.network import IPAddressV4, IPPort +from octopoes.models.ooi.network import IPAddress, IPPort from octopoes.models.ooi.web import Website @@ -22,29 +22,38 @@ def pull(statements: list[str]) -> str: return pull( [ f""" - [?ipaddress :object_type "IPAddressV4"] - [?ipaddress :IPAddressV4/primary_key "{str(targets[0])}"] - [?ipport80 :IPPort/address ?ipaddress] - [?ipport80 :IPPort/port 80] - [?ip_service :IPService/ip_port ?ipport80] - [?website :Website/ip_service ?ip_service] - (or - (and [?ipport443 :IPPort/address ?ipaddress][?ipport443 :IPPort/port 443]) - (and [(identity 0) ?ipport443]) - ) + (or + (and [?ipaddress :object_type "IPAddressV4"] + [?ipaddress :IPAddressV4/primary_key "{str(targets[0])}"] + ) + (and [?ipaddress :object_type "IPAddressV6"] + [?ipaddress :IPAddressV6/primary_key "{str(targets[0])}"] + ) + ) + [?ipport80 :IPPort/address ?ipaddress] + [?ipport80 :IPPort/port 80] + [?ip_service :IPService/ip_port ?ipport80] + [?website :Website/ip_service ?ip_service] + (or + (and [?ipport443 :IPPort/address ?ipaddress][?ipport443 :IPPort/port 443]) + [(identity nil) ?ipport443] + ) """ ] ) - return "potato" + elif sgn == "0100" or sgn == "0010" or sgn == "1110": + return "TODO" + else: + return "TODO" NIBBLE = NibbleDefinition( id="https_availability", signature=[ - NibbleParameter(object_type=IPAddressV4, parser="[*][?object_type == 'IPAddressV4'][]"), + NibbleParameter(object_type=IPAddress, parser="[*][?object_type == 'IPAddressV4'][]"), NibbleParameter(object_type=IPPort, parser="[*][?object_type == 'IPPort'][]"), NibbleParameter(object_type=Website, parser="[*][?object_type == 'Website'][]"), - NibbleParameter(object_type=int, parser='[*][-1][]'), + NibbleParameter(object_type=int, parser="[*][-1][]"), ], query=query, ) diff --git a/octopoes/nibbles/runner.py b/octopoes/nibbles/runner.py index 1a8828a7c6b..925b4e05a13 100644 --- a/octopoes/nibbles/runner.py +++ b/octopoes/nibbles/runner.py @@ -179,7 +179,8 @@ def _run(self, ooi: OOI, valid_time: datetime) -> dict[str, dict[tuple[Any, ...] nibblet_nibbles = {self.nibbles[nibblet.method] for nibblet in nibblets if nibblet.method in self.nibbles} for nibble in filter( - lambda x: any(isinstance(ooi, param.object_type) for param in x.signature) and x not in nibblet_nibbles, + lambda nibb: any(isinstance(ooi, sgn.object_type) for sgn in nibb.signature) + and nibb not in nibblet_nibbles, self.nibbles.values(), ): if nibble.enabled: diff --git a/octopoes/tests/integration/test_https_availability_nibble.py b/octopoes/tests/integration/test_https_availability_nibble.py index 09e26ae07e0..951064a0e8c 100644 --- a/octopoes/tests/integration/test_https_availability_nibble.py +++ b/octopoes/tests/integration/test_https_availability_nibble.py @@ -4,7 +4,6 @@ import pytest from nibbles.https_availability.nibble import NIBBLE as https_availability -from nibbles.runner import NibblesRunner from octopoes.core.service import OctopoesService from octopoes.models.ooi.dns.zone import Hostname @@ -20,13 +19,7 @@ def test_https_availability_query(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): - nibbler = NibblesRunner( - xtdb_octopoes_service.ooi_repository, - xtdb_octopoes_service.origin_repository, - xtdb_octopoes_service.nibbler.nibble_repository, - ) - xtdb_octopoes_service.nibbler.disable() - nibbler.nibbles = {https_availability.id: https_availability} + xtdb_octopoes_service.nibbler.nibbles = {https_availability.id: https_availability} network = Network(name="internet") xtdb_octopoes_service.ooi_repository.save(network, valid_time) From ca29c3e9602f859e6c267d8c982fcc256ff711cb Mon Sep 17 00:00:00 2001 From: Benny Date: Mon, 13 Jan 2025 15:26:57 +0100 Subject: [PATCH 04/17] Finalize --- octopoes/.ci/docker-compose.yml | 2 +- octopoes/nibbles/definitions.py | 4 ++ .../https_availability/https_availability.py | 6 ++- octopoes/nibbles/https_availability/nibble.py | 51 +++++++++---------- octopoes/nibbles/runner.py | 16 +++--- .../test_https_availability_nibble.py | 1 - 6 files changed, 43 insertions(+), 37 deletions(-) diff --git a/octopoes/.ci/docker-compose.yml b/octopoes/.ci/docker-compose.yml index ae46e5cd8ae..d4e33b42e13 100644 --- a/octopoes/.ci/docker-compose.yml +++ b/octopoes/.ci/docker-compose.yml @@ -17,7 +17,7 @@ services: args: ENVIRONMENT: dev context: . - command: pytest -s tests/integration/test_https_availability_nibble.py --timeout=300 + command: pytest tests/integration --timeout=300 depends_on: - xtdb - ci_octopoes diff --git a/octopoes/nibbles/definitions.py b/octopoes/nibbles/definitions.py index 51d048ed325..78775c36057 100644 --- a/octopoes/nibbles/definitions.py +++ b/octopoes/nibbles/definitions.py @@ -53,6 +53,10 @@ def __hash__(self): def _ini(self) -> dict[str, Any]: return {"id": self.id, "enabled": self.enabled, "checksum": self._checksum} + @property + def triggers(self) -> set[type[OOI]]: + return {sgn.object_type for sgn in self.signature if issubclass(sgn.object_type, OOI)} + def get_nibble_definitions() -> dict[str, NibbleDefinition]: nibble_definitions = {} diff --git a/octopoes/nibbles/https_availability/https_availability.py b/octopoes/nibbles/https_availability/https_availability.py index f88781f7cc2..99909cf9cd8 100644 --- a/octopoes/nibbles/https_availability/https_availability.py +++ b/octopoes/nibbles/https_availability/https_availability.py @@ -6,9 +6,13 @@ from octopoes.models.ooi.web import Website -def nibble(ipv4: IPAddress, port80: IPPort, website: Website, port443s: int) -> Iterator[OOI]: +def nibble(ipaddress: IPAddress, ipport80: IPPort, website: Website, port443s: int) -> Iterator[OOI]: + _ = ipaddress + _ = ipport80 + # The Null in the XTDB query is counted for one, hence any port443 object starts at > 1 if port443s < 2: ft = KATFindingType(id="KAT-HTTPS-NOT-AVAILABLE") + yield ft yield Finding( ooi=website.reference, finding_type=ft.reference, diff --git a/octopoes/nibbles/https_availability/nibble.py b/octopoes/nibbles/https_availability/nibble.py index b1720698bab..53aa18e62dc 100644 --- a/octopoes/nibbles/https_availability/nibble.py +++ b/octopoes/nibbles/https_availability/nibble.py @@ -17,40 +17,39 @@ def pull(statements: list[str]) -> str: }} """ + base_query = [ + """ + [?website :Website/ip_service ?ip_service] + [?ipservice :IPService/ip_port ?ipport80] + [?ipport80 :IPPort/port 80] + [?ipport80 :IPPort/address ?ipaddress] + (or + (and [?ipport443 :IPPort/address ?ipaddress][?ipport443 :IPPort/port 443]) + [(identity nil) ?ipport443] + ) + """ + ] + sgn = "".join(str(int(isinstance(target, Reference))) for target in targets) if sgn == "1000": - return pull( - [ - f""" - (or - (and [?ipaddress :object_type "IPAddressV4"] - [?ipaddress :IPAddressV4/primary_key "{str(targets[0])}"] - ) - (and [?ipaddress :object_type "IPAddressV6"] - [?ipaddress :IPAddressV6/primary_key "{str(targets[0])}"] - ) - ) - [?ipport80 :IPPort/address ?ipaddress] - [?ipport80 :IPPort/port 80] - [?ip_service :IPService/ip_port ?ipport80] - [?website :Website/ip_service ?ip_service] - (or - (and [?ipport443 :IPPort/address ?ipaddress][?ipport443 :IPPort/port 443]) - [(identity nil) ?ipport443] - ) - """ - ] - ) - elif sgn == "0100" or sgn == "0010" or sgn == "1110": - return "TODO" + return pull([f'[?ipaddress :IPAddress/primary_key "{str(targets[0])}"]'] + base_query) + elif sgn == "0100": + if int(str(targets[1]).split("|")[-1]) == 80: + return pull([f'[?ipport80 :IPPort/primary_key "{str(targets[1])}"]'] + base_query) + else: + return pull(base_query) + elif sgn == "0010": + return pull([f'[?website :Website/primary_key "{str(targets[2])}"]'] + base_query) else: - return "TODO" + return pull(base_query) NIBBLE = NibbleDefinition( id="https_availability", signature=[ - NibbleParameter(object_type=IPAddress, parser="[*][?object_type == 'IPAddressV4'][]"), + NibbleParameter( + object_type=IPAddress, parser="[*][?object_type == 'IPAddressV6' || object_type == 'IPAddressV4'][]" + ), NibbleParameter(object_type=IPPort, parser="[*][?object_type == 'IPPort'][]"), NibbleParameter(object_type=Website, parser="[*][?object_type == 'Website'][]"), NibbleParameter(object_type=int, parser="[*][-1][]"), diff --git a/octopoes/nibbles/runner.py b/octopoes/nibbles/runner.py index 925b4e05a13..96882691161 100644 --- a/octopoes/nibbles/runner.py +++ b/octopoes/nibbles/runner.py @@ -179,16 +179,16 @@ def _run(self, ooi: OOI, valid_time: datetime) -> dict[str, dict[tuple[Any, ...] nibblet_nibbles = {self.nibbles[nibblet.method] for nibblet in nibblets if nibblet.method in self.nibbles} for nibble in filter( - lambda nibb: any(isinstance(ooi, sgn.object_type) for sgn in nibb.signature) - and nibb not in nibblet_nibbles, + lambda nibbly: nibbly.enabled + and nibbly not in nibblet_nibbles + and any(isinstance(ooi, t) for t in nibbly.triggers), self.nibbles.values(), ): - if nibble.enabled: - if len(nibble.signature) > 1: - self._write(valid_time) - args = self.ooi_repository.nibble_query(ooi, nibble, valid_time) - results = {tuple(arg): set(flatten([nibble(arg)])) for arg in args} - return_value |= {nibble.id: results} + if len(nibble.signature) > 1: + self._write(valid_time) + args = self.ooi_repository.nibble_query(ooi, nibble, valid_time) + results = {tuple(arg): set(flatten([nibble(arg)])) for arg in args} + return_value |= {nibble.id: results} self.cache = merge_results(self.cache, {ooi: return_value}) return return_value diff --git a/octopoes/tests/integration/test_https_availability_nibble.py b/octopoes/tests/integration/test_https_availability_nibble.py index 951064a0e8c..de2f01c8938 100644 --- a/octopoes/tests/integration/test_https_availability_nibble.py +++ b/octopoes/tests/integration/test_https_availability_nibble.py @@ -62,7 +62,6 @@ def test_https_availability_query(xtdb_octopoes_service: OctopoesService, event_ port443 = IPPort(port=443, address=ip_address.reference, protocol=Protocol.TCP) xtdb_octopoes_service.ooi_repository.save(port443, valid_time) - event_manager.complete_process_events(xtdb_octopoes_service) assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 0 From 2373d0c068d8306693ac4c519e15cc95c36cb90a Mon Sep 17 00:00:00 2001 From: Benny Date: Mon, 13 Jan 2025 15:53:34 +0100 Subject: [PATCH 05/17] Close the corners --- .../nibbles/disallowed_csp_hostnames/nibble.py | 2 +- octopoes/nibbles/https_availability/nibble.py | 14 +++++++++++--- octopoes/tests/test_bits.py | 4 ++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/octopoes/nibbles/disallowed_csp_hostnames/nibble.py b/octopoes/nibbles/disallowed_csp_hostnames/nibble.py index fcf230be3e3..e4430a1455f 100644 --- a/octopoes/nibbles/disallowed_csp_hostnames/nibble.py +++ b/octopoes/nibbles/disallowed_csp_hostnames/nibble.py @@ -100,7 +100,7 @@ def query(targets: list[Reference | None]) -> str: NIBBLE = NibbleDefinition( - id="disallowed-csp-hostnames", + id="disallowed_csp_hostnames", signature=[ NibbleParameter(object_type=HTTPHeaderHostname, parser="[*][?object_type == 'HTTPHeaderHostname'][]"), NibbleParameter(object_type=Config, parser="[*][?object_type == 'Config'][]", optional=True), diff --git a/octopoes/nibbles/https_availability/nibble.py b/octopoes/nibbles/https_availability/nibble.py index 53aa18e62dc..91f4617f293 100644 --- a/octopoes/nibbles/https_availability/nibble.py +++ b/octopoes/nibbles/https_availability/nibble.py @@ -30,16 +30,24 @@ def pull(statements: list[str]) -> str: """ ] + ref_queries = [ + f'[?ipaddress :IPAddress/primary_key "{str(targets[0])}"]', + f'[?ipport80 :IPPort/primary_key "{str(targets[1])}"]', + f'[?website :Website/primary_key "{str(targets[2])}"]', + ] + sgn = "".join(str(int(isinstance(target, Reference))) for target in targets) if sgn == "1000": - return pull([f'[?ipaddress :IPAddress/primary_key "{str(targets[0])}"]'] + base_query) + return pull(ref_queries[0:1] + base_query) elif sgn == "0100": if int(str(targets[1]).split("|")[-1]) == 80: - return pull([f'[?ipport80 :IPPort/primary_key "{str(targets[1])}"]'] + base_query) + return pull(ref_queries[1:2] + base_query) else: return pull(base_query) elif sgn == "0010": - return pull([f'[?website :Website/primary_key "{str(targets[2])}"]'] + base_query) + return pull(ref_queries[2:3] + base_query) + elif sgn == "1110": + return pull(ref_queries + base_query) else: return pull(base_query) diff --git a/octopoes/tests/test_bits.py b/octopoes/tests/test_bits.py index e2e4f154755..d7af1df3e5e 100644 --- a/octopoes/tests/test_bits.py +++ b/octopoes/tests/test_bits.py @@ -1,4 +1,4 @@ -from bits.https_availability.https_availability import run as run_https_availability +from nibbles.https_availability.https_availability import nibble as run_https_availability from nibbles.oois_in_headers.oois_in_headers import nibble as run_oois_in_headers from octopoes.models.ooi.config import Config @@ -42,7 +42,7 @@ def test_url_extracted_by_oois_in_headers_relative_path(http_resource_https): def test_finding_generated_when_443_not_open_and_80_is_open(): port_80 = IPPort(address="fake", protocol="tcp", port=80) website = Website(ip_service="fake", hostname="fake") - results = list(run_https_availability(None, [port_80, website], {})) + results = list(run_https_availability(None, port_80, website, 1)) finding = results[0] assert isinstance(finding, Finding) assert finding.description == "HTTP port is open, but HTTPS port is not open" From 368bf699fe262467ae7deb5bcec540d054d33a57 Mon Sep 17 00:00:00 2001 From: Benny Date: Mon, 13 Jan 2025 15:59:35 +0100 Subject: [PATCH 06/17] Fix unit test --- octopoes/tests/test_bits.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/octopoes/tests/test_bits.py b/octopoes/tests/test_bits.py index d7af1df3e5e..a998565eb8f 100644 --- a/octopoes/tests/test_bits.py +++ b/octopoes/tests/test_bits.py @@ -2,7 +2,7 @@ from nibbles.oois_in_headers.oois_in_headers import nibble as run_oois_in_headers from octopoes.models.ooi.config import Config -from octopoes.models.ooi.findings import Finding +from octopoes.models.ooi.findings import Finding, KATFindingType from octopoes.models.ooi.network import IPPort from octopoes.models.ooi.web import URL, HTTPHeader, HTTPHeaderURL, Website @@ -43,6 +43,7 @@ def test_finding_generated_when_443_not_open_and_80_is_open(): port_80 = IPPort(address="fake", protocol="tcp", port=80) website = Website(ip_service="fake", hostname="fake") results = list(run_https_availability(None, port_80, website, 1)) - finding = results[0] - assert isinstance(finding, Finding) + finding = [result for result in results if isinstance(result, Finding)][0] assert finding.description == "HTTP port is open, but HTTPS port is not open" + katfindingtype = [result for result in results if not isinstance(result, Finding)][0] + assert isinstance(katfindingtype, KATFindingType) From b8488168e751797afa6eba01bc644a8c2a0bd580 Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 15 Jan 2025 09:12:02 +0100 Subject: [PATCH 07/17] Init --- octopoes/.ci/docker-compose.yml | 2 +- octopoes/bits/internetnl/bit.py | 24 ---- octopoes/bits/internetnl/internetnl.py | 50 ------- octopoes/nibbles/definitions.py | 13 +- .../{bits => nibbles}/internetnl/__init__.py | 0 octopoes/nibbles/internetnl/internetnl.py | 18 +++ octopoes/nibbles/internetnl/nibble.py | 130 ++++++++++++++++++ .../octopoes/repositories/ooi_repository.py | 3 +- .../integration/test_internetnl_nibble.py | 69 ++++++++++ octopoes/tools/xtdb-cli.py | 4 +- octopoes/tools/xtdb_client.py | 3 +- 11 files changed, 235 insertions(+), 81 deletions(-) delete mode 100644 octopoes/bits/internetnl/bit.py delete mode 100644 octopoes/bits/internetnl/internetnl.py rename octopoes/{bits => nibbles}/internetnl/__init__.py (100%) create mode 100644 octopoes/nibbles/internetnl/internetnl.py create mode 100644 octopoes/nibbles/internetnl/nibble.py create mode 100644 octopoes/tests/integration/test_internetnl_nibble.py diff --git a/octopoes/.ci/docker-compose.yml b/octopoes/.ci/docker-compose.yml index d4e33b42e13..d03dc811b4d 100644 --- a/octopoes/.ci/docker-compose.yml +++ b/octopoes/.ci/docker-compose.yml @@ -17,7 +17,7 @@ services: args: ENVIRONMENT: dev context: . - command: pytest tests/integration --timeout=300 + command: pytest -s tests/integration/test_internetnl_nibble.py --timeout=300 depends_on: - xtdb - ci_octopoes diff --git a/octopoes/bits/internetnl/bit.py b/octopoes/bits/internetnl/bit.py deleted file mode 100644 index e4b07dc359f..00000000000 --- a/octopoes/bits/internetnl/bit.py +++ /dev/null @@ -1,24 +0,0 @@ -from bits.definitions import BitDefinition, BitParameterDefinition -from octopoes.models.ooi.dns.zone import Hostname -from octopoes.models.ooi.findings import Finding -from octopoes.models.ooi.web import Website - -BIT = BitDefinition( - id="internet-nl", - consumes=Hostname, - parameters=[ - BitParameterDefinition(ooi_type=Finding, relation_path="ooi [is Hostname]"), # findings on hostnames - BitParameterDefinition( - ooi_type=Finding, relation_path="ooi [is HTTPResource].website.hostname" - ), # findings on resources - BitParameterDefinition( - ooi_type=Finding, relation_path="ooi [is HTTPHeader].resource.website.hostname" - ), # findings on headers - BitParameterDefinition(ooi_type=Finding, relation_path="ooi [is Website].hostname"), # findings on websites - BitParameterDefinition( - ooi_type=Finding, relation_path="ooi [is HostnameHTTPURL].netloc" - ), # findings on weburls - BitParameterDefinition(ooi_type=Website, relation_path="hostname"), # only websites have to comply - ], - module="bits.internetnl.internetnl", -) diff --git a/octopoes/bits/internetnl/internetnl.py b/octopoes/bits/internetnl/internetnl.py deleted file mode 100644 index a0058b22fc8..00000000000 --- a/octopoes/bits/internetnl/internetnl.py +++ /dev/null @@ -1,50 +0,0 @@ -from collections.abc import Iterator -from typing import Any - -from octopoes.models import OOI -from octopoes.models.ooi.dns.zone import Hostname -from octopoes.models.ooi.findings import Finding, KATFindingType -from octopoes.models.ooi.web import Website - - -def run(input_ooi: Hostname, additional_oois: list[Finding | Website], config: dict[str, Any]) -> Iterator[OOI]: - # only websites have to comply with the internetnl rules - websites = [websites for websites in additional_oois if isinstance(websites, Website)] - if not websites: - return - - finding_ids = [finding.finding_type.tokenized.id for finding in additional_oois if isinstance(finding, Finding)] - - result = "" - internetnl_findings = { - "KAT-WEBSERVER-NO-IPV6": "This webserver does not have an IPv6 address", - "KAT-NAMESERVER-NO-TWO-IPV6": "This webserver does not have at least two nameservers with an IPv6 address", - "KAT-NO-DNSSEC": "This webserver is not DNSSEC signed", - "KAT-INVALID-DNSSEC": "The DNSSEC signature of this webserver is not valid", - "KAT-NO-HSTS": "This website has at least one webpage with a missing Strict-Transport-Policy header", - "KAT-NO-CSP": "This website has at least one webpage with a missing Content-Security-Policy header", - "KAT-NO-X-FRAME-OPTIONS": "This website has at least one webpage with a missing X-Frame-Options header", - "KAT-NO-X-CONTENT-TYPE-OPTIONS": ( - "This website has at least one webpage with a missing X-Content-Type-Options header" - ), - "KAT-CSP-VULNERABILITIES": "This website has at least one webpage with a mis-configured CSP header", - "KAT-HSTS-VULNERABILITIES": "This website has at least one webpage with a mis-configured HSTS header", - "KAT-NO-CERTIFICATE": "This website does not have an SSL certificate", - "KAT-HTTPS-NOT-AVAILABLE": "HTTPS is not available for this website", - "KAT-SSL-CERT-HOSTNAME-MISMATCH": "The SSL certificate of this website does not match the hostname", - "KAT-HTTPS-REDIRECT": "This website has at least one HTTP URL that does not redirect to HTTPS", - } - - for finding, description in internetnl_findings.items(): - if finding in finding_ids: - result += f"{description}\n" - - if result: - ft = KATFindingType(id="KAT-INTERNETNL") - yield ft - f = Finding( - finding_type=ft.reference, - ooi=input_ooi.reference, - description=f"This hostname has at least one website with the following finding(s): {result}", - ) - yield f diff --git a/octopoes/nibbles/definitions.py b/octopoes/nibbles/definitions.py index 78775c36057..dda5825e132 100644 --- a/octopoes/nibbles/definitions.py +++ b/octopoes/nibbles/definitions.py @@ -19,9 +19,10 @@ class NibbleParameter(BaseModel): - object_type: type[Any] + object_type: Any parser: str = "[]" optional: bool = False + additional: set[type[OOI]] = set() def __eq__(self, other): if isinstance(other, NibbleParameter): @@ -31,12 +32,20 @@ def __eq__(self, other): else: return False + @property + def triggers(self) -> set[type[OOI]]: + if isinstance(self.object_type, type) and issubclass(self.object_type, OOI): + return {self.object_type} | self.additional + else: + return self.additional + class NibbleDefinition(BaseModel): id: str signature: list[NibbleParameter] query: str | Callable[[list[Reference | None]], str] | None = None enabled: bool = True + additional: set[type[OOI]] = set() _payload: MethodType | None = None _checksum: str | None = None @@ -55,7 +64,7 @@ def _ini(self) -> dict[str, Any]: @property def triggers(self) -> set[type[OOI]]: - return {sgn.object_type for sgn in self.signature if issubclass(sgn.object_type, OOI)} + return set.union(*[sgn.triggers for sgn in self.signature]) | self.additional def get_nibble_definitions() -> dict[str, NibbleDefinition]: diff --git a/octopoes/bits/internetnl/__init__.py b/octopoes/nibbles/internetnl/__init__.py similarity index 100% rename from octopoes/bits/internetnl/__init__.py rename to octopoes/nibbles/internetnl/__init__.py diff --git a/octopoes/nibbles/internetnl/internetnl.py b/octopoes/nibbles/internetnl/internetnl.py new file mode 100644 index 00000000000..85e24a274a9 --- /dev/null +++ b/octopoes/nibbles/internetnl/internetnl.py @@ -0,0 +1,18 @@ +from collections.abc import Iterator + +from octopoes.models import OOI +from octopoes.models.ooi.dns.zone import Hostname +from octopoes.models.ooi.findings import Finding, KATFindingType + + +def nibble(hostname: Hostname, findings: list[Finding]) -> Iterator[OOI]: + result = "\n".join([str(finding.description) for finding in findings]) + + if result: + ft = KATFindingType(id="KAT-INTERNETNL") + yield ft + yield Finding( + finding_type=ft.reference, + ooi=hostname.reference, + description=f"This hostname has at least one website with the following finding(s): {result}", + ) diff --git a/octopoes/nibbles/internetnl/nibble.py b/octopoes/nibbles/internetnl/nibble.py new file mode 100644 index 00000000000..58c6cac1d7a --- /dev/null +++ b/octopoes/nibbles/internetnl/nibble.py @@ -0,0 +1,130 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models import Reference +from octopoes.models.ooi.dns.zone import Hostname, Network +from octopoes.models.ooi.findings import Finding +from octopoes.models.ooi.web import Website + +finding_types = [ + "KAT-WEBSERVER-NO-IPV6", + "KAT-NAMESERVER-NO-TWO-IPV6", + "KAT-NO-DNSSEC", + "KAT-INVALID-DNSSEC", + "KAT-NO-HSTS", + "KAT-NO-CSP", + "KAT-NO-X-FRAME-OPTIONS", + "KAT-NO-X-CONTENT-TYPE-OPTIONS", + "KAT-CSP-VULNERABILITIES", + "KAT-HSTS-VULNERABILITIES", + "KAT-NO-CERTIFICATE", + "KAT-HTTPS-NOT-AVAILABLE", + "KAT-SSL-CERT-HOSTNAME-MISMATCH", + "KAT-HTTPS-REDIRECT", +] + + +def or_finding_types() -> str: + clauses = "".join([f'[?finding :Finding/finding_type "{ft}"]' for ft in finding_types]) + return f"(or {clauses})" + + +def query(targets: list[Reference | None]) -> str: + def pull(statements: list[str]) -> str: + return f""" + {{ + :query {{ + :find [(pull ?hostname [*]) (pull ?finding [*])] + :where [ + {" ".join(statements)} + ] + }} + }} + """ + + base_query = [ + """ + [?website :Website/hostname ?hostname] + [?finding :Finding/ooi ?ooi] + (or + (or + (and + [?ooi :Hostname/primary_key ?hostname] + ) + (and + [(identity nil) ?ooi] + ) + ) + (or + (and + [?ooi :HTTPResource/website ?website] + [?website :Website/hostname ?hostname] + ) + (and + [(identity nil) ?ooi] + [(identity nil) ?website] + ) + ) + (or + (and + [?ooi :HTTPHeader/website ?resource] + [?resource :HTTPResource/website ?website] + [?website :Website/hostname ?hostname] + ) + (and + [(identity nil) ?ooi] + [(identity nil) ?resource] + [(identity nil) ?website] + ) + ) + (or + (and + [?ooi :Website/hostname ?hostname] + ) + (and + [(identity nil) ?ooi] + ) + ) + (or + (and + [?ooi :HostnameHTTPURL/hostname ?hostname] + ) + (and + [(identity nil) ?ooi] + ) + ) + ) + """ + ] + + sgn = "".join(str(int(isinstance(target, Reference))) for target in targets) + ref_query = ["[?hostname :Hostname/primary_key]"] + if sgn == "100": + ref_query = [f'[?hostname :Hostname/primary_key "{str(targets[0])}"]'] + elif sgn == "010": + ref_query = [f'[?hostname :Hostname/primary_key "{str(targets[1]).split("|")[-1]}"]'] + elif sgn == "001": + tokens = str(targets[2]).split("|")[1:-1] + target_reference = Reference.from_str("|".join(tokens)) + if tokens[0] == "Hostname": + hostname = target_reference.tokenized + elif tokens[0] == "HTTPResource": + hostname = target_reference.tokenized.website.hostname + elif tokens[0] == "HTTPHeader": + hostname = target_reference.tokenized.resource.website.hostname + elif tokens[0] in {"Website", "HostnameHTTPURL"}: + hostname = target_reference.tokenized.hostname + else: + raise ValueError() + hostname_pk = Hostname(name=hostname.name, network=Network(name=hostname.network.name).reference).reference + ref_query = [f'[?hostname :Hostname/primary_key "{str(hostname_pk)}"]'] + return pull(ref_query + base_query) + + +NIBBLE = NibbleDefinition( + id="internet_nl", + signature=[ + NibbleParameter(object_type=Hostname, parser="[*][?object_type == 'IPPort'][]"), + NibbleParameter(object_type=list[Website], parser="[[*][?object_type == 'Website'][]]", additional={Website}), + NibbleParameter(object_type=list[Finding], parser="[[*][?object_type == 'Findings'][]]", additional={Finding}), + ], + query=query, +) diff --git a/octopoes/octopoes/repositories/ooi_repository.py b/octopoes/octopoes/repositories/ooi_repository.py index c171c42c19a..d2739acd8e7 100644 --- a/octopoes/octopoes/repositories/ooi_repository.py +++ b/octopoes/octopoes/repositories/ooi_repository.py @@ -919,13 +919,14 @@ def nibble_query( first = True arguments = [ ooi.reference - if sgn.object_type == type_by_name(ooi.get_ooi_type()) and (first and not (first := False)) + if type_by_name(ooi.get_ooi_type()) in sgn.triggers and (first and not (first := False)) else None for sgn in nibble.signature ] else: arguments = [None for _ in nibble.signature] query = nibble.query if isinstance(nibble.query, str) else nibble.query(arguments) + breakpoint() data = self.session.client.query(query, valid_time) objects = [ {self.parse_as(element.object_type, obj) for obj in search(element.parser, data)} diff --git a/octopoes/tests/integration/test_internetnl_nibble.py b/octopoes/tests/integration/test_internetnl_nibble.py new file mode 100644 index 00000000000..ab437f563df --- /dev/null +++ b/octopoes/tests/integration/test_internetnl_nibble.py @@ -0,0 +1,69 @@ +import os +from datetime import datetime +from unittest.mock import Mock + +import pytest +from nibbles.internetnl.nibble import NIBBLE as internetnl + +from octopoes.core.service import OctopoesService +from octopoes.models.ooi.dns.zone import Hostname +from octopoes.models.ooi.findings import Finding, KATFindingType +from octopoes.models.ooi.network import IPAddressV4, IPPort, Network, Protocol +from octopoes.models.ooi.service import IPService, Service +from octopoes.models.ooi.web import HostnameHTTPURL, HTTPHeader, HTTPResource, WebScheme, Website + +if os.environ.get("CI") != "1": + pytest.skip("Needs XTDB multinode container.", allow_module_level=True) + +STATIC_IP = ".".join((4 * "1 ").split()) + + +def test_internetnl_query(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + xtdb_octopoes_service.nibbler.nibbles = {internetnl.id: internetnl} + + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + + hostname = Hostname(name="example.com", network=network.reference) + xtdb_octopoes_service.ooi_repository.save(hostname, valid_time) + + web_url = HostnameHTTPURL( + network=network.reference, netloc=hostname.reference, port=443, path="/", scheme=WebScheme.HTTP + ) + xtdb_octopoes_service.ooi_repository.save(web_url, valid_time) + + service = Service(name="https") + xtdb_octopoes_service.ooi_repository.save(service, valid_time) + + ip_address = IPAddressV4(address=STATIC_IP, network=network.reference) + xtdb_octopoes_service.ooi_repository.save(ip_address, valid_time) + + port = IPPort(port=80, address=ip_address.reference, protocol=Protocol.TCP) + xtdb_octopoes_service.ooi_repository.save(port, valid_time) + + ip_service = IPService(ip_port=port.reference, service=service.reference) + xtdb_octopoes_service.ooi_repository.save(ip_service, valid_time) + + website = Website(ip_service=ip_service.reference, hostname=hostname.reference) + xtdb_octopoes_service.ooi_repository.save(website, valid_time) + + resource = HTTPResource(website=website.reference, web_url=web_url.reference) + xtdb_octopoes_service.ooi_repository.save(resource, valid_time) + + header = HTTPHeader( + resource=resource.reference, key="strict-transport-security", value="max-age=21536000; includeSubDomains" + ) + xtdb_octopoes_service.ooi_repository.save(header, valid_time) + + ft = KATFindingType(id="KAT-HTTPS-NOT-AVAILABLE") + xtdb_octopoes_service.ooi_repository.save(ft, valid_time) + + finding = Finding( + ooi=website.reference, finding_type=ft.reference, description="HTTP port is open, but HTTPS port is not open" + ) + xtdb_octopoes_service.ooi_repository.save(finding, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 2 + assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 2 diff --git a/octopoes/tools/xtdb-cli.py b/octopoes/tools/xtdb-cli.py index 31a63730733..0aa0575d97c 100755 --- a/octopoes/tools/xtdb-cli.py +++ b/octopoes/tools/xtdb-cli.py @@ -59,9 +59,9 @@ def query( client: XTDBClient = ctx.obj["client"] if edn: - click.echo(json.dumps(client.query(edn, valid_time, tx_time, tx_id))) + click.echo(client.query(edn, valid_time, tx_time, tx_id)) else: - click.echo(json.dumps(client.query(valid_time=valid_time, tx_time=tx_time, tx_id=tx_id))) + click.echo(client.query(valid_time=valid_time, tx_time=tx_time, tx_id=tx_id)) @cli.command(help="List all keys in node") diff --git a/octopoes/tools/xtdb_client.py b/octopoes/tools/xtdb_client.py index ac599c75937..f4e2ac90573 100644 --- a/octopoes/tools/xtdb_client.py +++ b/octopoes/tools/xtdb_client.py @@ -41,7 +41,7 @@ def query( valid_time: datetime.datetime | None = None, tx_time: datetime.datetime | None = None, tx_id: int | None = None, - ) -> JsonValue: + ) -> str: params = {} if valid_time is not None: params["valid-time"] = valid_time.isoformat() @@ -52,6 +52,7 @@ def query( res = self._client.post("/query", params=params, content=query, headers={"Content-Type": "application/edn"}) + return res.text return res.json() def entity( From 681d050358117dc60dbcbe7a2ac9f42c385b3fa7 Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 15 Jan 2025 10:54:27 +0100 Subject: [PATCH 08/17] Integrate feedback --- octopoes/nibbles/runner.py | 4 +--- .../tests/integration/test_https_availability_nibble.py | 8 ++++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/octopoes/nibbles/runner.py b/octopoes/nibbles/runner.py index 96882691161..73192cb12c3 100644 --- a/octopoes/nibbles/runner.py +++ b/octopoes/nibbles/runner.py @@ -179,9 +179,7 @@ def _run(self, ooi: OOI, valid_time: datetime) -> dict[str, dict[tuple[Any, ...] nibblet_nibbles = {self.nibbles[nibblet.method] for nibblet in nibblets if nibblet.method in self.nibbles} for nibble in filter( - lambda nibbly: nibbly.enabled - and nibbly not in nibblet_nibbles - and any(isinstance(ooi, t) for t in nibbly.triggers), + lambda x: x.enabled and x not in nibblet_nibbles and any(isinstance(ooi, t) for t in x.triggers), self.nibbles.values(), ): if len(nibble.signature) > 1: diff --git a/octopoes/tests/integration/test_https_availability_nibble.py b/octopoes/tests/integration/test_https_availability_nibble.py index de2f01c8938..314078d806a 100644 --- a/octopoes/tests/integration/test_https_availability_nibble.py +++ b/octopoes/tests/integration/test_https_availability_nibble.py @@ -58,7 +58,15 @@ def test_https_availability_query(xtdb_octopoes_service: OctopoesService, event_ event_manager.complete_process_events(xtdb_octopoes_service) assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 1 + assert ( + xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).items[0].description + == "HTTP port is open, but HTTPS port is not open" + ) assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 1 + assert ( + xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).items[0].id + == "KAT-HTTPS-NOT-AVAILABLE" + ) port443 = IPPort(port=443, address=ip_address.reference, protocol=Protocol.TCP) xtdb_octopoes_service.ooi_repository.save(port443, valid_time) From 3e6c87e764ecb3ddf5daa4f9eab9427c1182e715 Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 15 Jan 2025 11:11:51 +0100 Subject: [PATCH 09/17] Fix nibble --- octopoes/nibbles/https_availability/https_availability.py | 3 +-- octopoes/nibbles/https_availability/nibble.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/octopoes/nibbles/https_availability/https_availability.py b/octopoes/nibbles/https_availability/https_availability.py index 99909cf9cd8..e1c5b9f4f31 100644 --- a/octopoes/nibbles/https_availability/https_availability.py +++ b/octopoes/nibbles/https_availability/https_availability.py @@ -9,8 +9,7 @@ def nibble(ipaddress: IPAddress, ipport80: IPPort, website: Website, port443s: int) -> Iterator[OOI]: _ = ipaddress _ = ipport80 - # The Null in the XTDB query is counted for one, hence any port443 object starts at > 1 - if port443s < 2: + if port443s < 1: ft = KATFindingType(id="KAT-HTTPS-NOT-AVAILABLE") yield ft yield Finding( diff --git a/octopoes/nibbles/https_availability/nibble.py b/octopoes/nibbles/https_availability/nibble.py index 91f4617f293..09f2f99413c 100644 --- a/octopoes/nibbles/https_availability/nibble.py +++ b/octopoes/nibbles/https_availability/nibble.py @@ -9,7 +9,7 @@ def pull(statements: list[str]) -> str: return f""" {{ :query {{ - :find [(pull ?ipaddress [*]) (pull ?ipport80 [*]) (pull ?website [*]) (count ?ipport443)] + :find [(pull ?ipaddress [*]) (pull ?ipport80 [*]) (pull ?website [*]) (- (count ?ipport443) 1)] :where [ {" ".join(statements)} ] From 99cc00c5fc91c0a37b3b291831f4902588ade4dd Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 15 Jan 2025 11:17:09 +0100 Subject: [PATCH 10/17] Fix nibble test --- octopoes/tests/test_bits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/octopoes/tests/test_bits.py b/octopoes/tests/test_bits.py index a998565eb8f..c0f4605657f 100644 --- a/octopoes/tests/test_bits.py +++ b/octopoes/tests/test_bits.py @@ -42,7 +42,7 @@ def test_url_extracted_by_oois_in_headers_relative_path(http_resource_https): def test_finding_generated_when_443_not_open_and_80_is_open(): port_80 = IPPort(address="fake", protocol="tcp", port=80) website = Website(ip_service="fake", hostname="fake") - results = list(run_https_availability(None, port_80, website, 1)) + results = list(run_https_availability(None, port_80, website, 0)) finding = [result for result in results if isinstance(result, Finding)][0] assert finding.description == "HTTP port is open, but HTTPS port is not open" katfindingtype = [result for result in results if not isinstance(result, Finding)][0] From 2ca65dd35920b49fa3444be546cc19d76a42bc26 Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 15 Jan 2025 14:08:13 +0100 Subject: [PATCH 11/17] or-join ftw --- octopoes/nibbles/internetnl/nibble.py | 69 ++++++++------------------- 1 file changed, 21 insertions(+), 48 deletions(-) diff --git a/octopoes/nibbles/internetnl/nibble.py b/octopoes/nibbles/internetnl/nibble.py index 58c6cac1d7a..b35fe6f6630 100644 --- a/octopoes/nibbles/internetnl/nibble.py +++ b/octopoes/nibbles/internetnl/nibble.py @@ -32,7 +32,11 @@ def pull(statements: list[str]) -> str: return f""" {{ :query {{ - :find [(pull ?hostname [*]) (pull ?finding [*])] + :find [ + (pull ?hostname [*]) + (pull ?website [*]) + (pull ?finding [*]) + ] :where [ {" ".join(statements)} ] @@ -42,54 +46,23 @@ def pull(statements: list[str]) -> str: base_query = [ """ + [?hostname :object_type "Hostname"] [?website :Website/hostname ?hostname] - [?finding :Finding/ooi ?ooi] - (or - (or - (and - [?ooi :Hostname/primary_key ?hostname] - ) - (and - [(identity nil) ?ooi] - ) + (or-join [?finding] + [?finding :Finding/ooi ?hostname] + (and + [?hostnamehttpurl :HostnameHTTPURL/netloc ?hostname] + [?finding :Finding/ooi ?hostnamehttpurl] ) - (or - (and - [?ooi :HTTPResource/website ?website] - [?website :Website/hostname ?hostname] - ) - (and - [(identity nil) ?ooi] - [(identity nil) ?website] - ) + [?finding :Finding/ooi ?website] + (and + [?resource :HTTPResource/website ?website] + [?finding :Finding/ooi ?resource] ) - (or - (and - [?ooi :HTTPHeader/website ?resource] - [?resource :HTTPResource/website ?website] - [?website :Website/hostname ?hostname] - ) - (and - [(identity nil) ?ooi] - [(identity nil) ?resource] - [(identity nil) ?website] - ) - ) - (or - (and - [?ooi :Website/hostname ?hostname] - ) - (and - [(identity nil) ?ooi] - ) - ) - (or - (and - [?ooi :HostnameHTTPURL/hostname ?hostname] - ) - (and - [(identity nil) ?ooi] - ) + (and + [?header :HTTPHeader/resource ?resource] + [?resource :HTTPResource/website ?website] + [?finding :Finding/ooi ?header] ) ) """ @@ -106,12 +79,12 @@ def pull(statements: list[str]) -> str: target_reference = Reference.from_str("|".join(tokens)) if tokens[0] == "Hostname": hostname = target_reference.tokenized + elif tokens[0] in {"HostnameHTTPURL", "Website"}: + hostname = target_reference.tokenized.hostname elif tokens[0] == "HTTPResource": hostname = target_reference.tokenized.website.hostname elif tokens[0] == "HTTPHeader": hostname = target_reference.tokenized.resource.website.hostname - elif tokens[0] in {"Website", "HostnameHTTPURL"}: - hostname = target_reference.tokenized.hostname else: raise ValueError() hostname_pk = Hostname(name=hostname.name, network=Network(name=hostname.network.name).reference).reference From ff021ea2900981290cf4a7514db40a7293d03b44 Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 15 Jan 2025 15:03:23 +0100 Subject: [PATCH 12/17] Nail internetnl (almost) --- octopoes/nibbles/internetnl/nibble.py | 6 ++---- octopoes/octopoes/repositories/ooi_repository.py | 4 +++- octopoes/tests/test_ooi_repository.py | 4 ++++ octopoes/tools/xtdb-cli.py | 4 ++-- octopoes/tools/xtdb_client.py | 3 +-- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/octopoes/nibbles/internetnl/nibble.py b/octopoes/nibbles/internetnl/nibble.py index b35fe6f6630..cfc14d1c979 100644 --- a/octopoes/nibbles/internetnl/nibble.py +++ b/octopoes/nibbles/internetnl/nibble.py @@ -2,7 +2,6 @@ from octopoes.models import Reference from octopoes.models.ooi.dns.zone import Hostname, Network from octopoes.models.ooi.findings import Finding -from octopoes.models.ooi.web import Website finding_types = [ "KAT-WEBSERVER-NO-IPV6", @@ -95,9 +94,8 @@ def pull(statements: list[str]) -> str: NIBBLE = NibbleDefinition( id="internet_nl", signature=[ - NibbleParameter(object_type=Hostname, parser="[*][?object_type == 'IPPort'][]"), - NibbleParameter(object_type=list[Website], parser="[[*][?object_type == 'Website'][]]", additional={Website}), - NibbleParameter(object_type=list[Finding], parser="[[*][?object_type == 'Findings'][]]", additional={Finding}), + NibbleParameter(object_type=Hostname, parser="[*][?object_type == 'Hostname'][]"), + NibbleParameter(object_type=list[Finding], parser="[[*][?object_type == 'Finding'][]]", additional={Finding}), ], query=query, ) diff --git a/octopoes/octopoes/repositories/ooi_repository.py b/octopoes/octopoes/repositories/ooi_repository.py index d2739acd8e7..ed71b696665 100644 --- a/octopoes/octopoes/repositories/ooi_repository.py +++ b/octopoes/octopoes/repositories/ooi_repository.py @@ -284,6 +284,9 @@ def parse_as(cls, type_: type | list[type], obj: dict | list | set | Any) -> tup # list --> tuple if isinstance(type_, list): return tuple(cls.parse_as(type_t, o) for o, type_t in zip(obj, type_)) + elif hasattr(type_, "__origin__") and hasattr(type_, "__args__") and type_.__origin__ is list: + t = [type_.__args__[0]] * len(obj) + return tuple(cls.parse_as(t, o) for o, t in zip(obj, t)) else: return tuple(cls.parse_as(type_, o) for o in obj) else: @@ -926,7 +929,6 @@ def nibble_query( else: arguments = [None for _ in nibble.signature] query = nibble.query if isinstance(nibble.query, str) else nibble.query(arguments) - breakpoint() data = self.session.client.query(query, valid_time) objects = [ {self.parse_as(element.object_type, obj) for obj in search(element.parser, data)} diff --git a/octopoes/tests/test_ooi_repository.py b/octopoes/tests/test_ooi_repository.py index 1c4c6e131de..0497973d682 100644 --- a/octopoes/tests/test_ooi_repository.py +++ b/octopoes/tests/test_ooi_repository.py @@ -164,3 +164,7 @@ def pull(ooi: OOI): [network, url, network] ) assert self.repository.parse_as(Network, [pull(network), pull(url), pull(url)]) == tuple([network, url, url]) + + assert self.repository.parse_as(list[Network], [pull(network), pull(network), pull(network)]) == tuple( + [network, network, network] + ) diff --git a/octopoes/tools/xtdb-cli.py b/octopoes/tools/xtdb-cli.py index 0aa0575d97c..31a63730733 100755 --- a/octopoes/tools/xtdb-cli.py +++ b/octopoes/tools/xtdb-cli.py @@ -59,9 +59,9 @@ def query( client: XTDBClient = ctx.obj["client"] if edn: - click.echo(client.query(edn, valid_time, tx_time, tx_id)) + click.echo(json.dumps(client.query(edn, valid_time, tx_time, tx_id))) else: - click.echo(client.query(valid_time=valid_time, tx_time=tx_time, tx_id=tx_id)) + click.echo(json.dumps(client.query(valid_time=valid_time, tx_time=tx_time, tx_id=tx_id))) @cli.command(help="List all keys in node") diff --git a/octopoes/tools/xtdb_client.py b/octopoes/tools/xtdb_client.py index f4e2ac90573..ac599c75937 100644 --- a/octopoes/tools/xtdb_client.py +++ b/octopoes/tools/xtdb_client.py @@ -41,7 +41,7 @@ def query( valid_time: datetime.datetime | None = None, tx_time: datetime.datetime | None = None, tx_id: int | None = None, - ) -> str: + ) -> JsonValue: params = {} if valid_time is not None: params["valid-time"] = valid_time.isoformat() @@ -52,7 +52,6 @@ def query( res = self._client.post("/query", params=params, content=query, headers={"Content-Type": "application/edn"}) - return res.text return res.json() def entity( From 89fa2ac8dea0d159a8f37fb51e2545de4e8d8241 Mon Sep 17 00:00:00 2001 From: Benny Date: Tue, 28 Jan 2025 13:18:45 +0100 Subject: [PATCH 13/17] Add jcs --- octopoes/poetry.lock | 13 ++++++++++++- octopoes/pyproject.toml | 1 + octopoes/requirements-dev.txt | 3 +++ octopoes/requirements.txt | 3 +++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/octopoes/poetry.lock b/octopoes/poetry.lock index 41925cfb6d9..81c7327c92e 100644 --- a/octopoes/poetry.lock +++ b/octopoes/poetry.lock @@ -796,6 +796,17 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "jcs" +version = "0.2.1" +description = "JCS - JSON Canonicalization" +optional = false +python-versions = ">=3.6.2" +files = [ + {file = "jcs-0.2.1-py3-none-any.whl", hash = "sha256:e23a3e1de60f832d33cd811bb9c3b3be79219cdf95f63b88f0972732c3fa8476"}, + {file = "jcs-0.2.1.tar.gz", hash = "sha256:9f20360b2f3b0a410d65493b448f96306d80e37fb46283f3f4aa5db2c7c1472b"}, +] + [[package]] name = "jmespath-community" version = "1.1.3" @@ -2347,4 +2358,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "e41ab6fe70fe531c50ec680a5620b21e7b5fac4eb45b9841eff945234a445ff2" +content-hash = "f79d45fed4bf27cd7d15665c1ca25ba5ac85911d4de10abaa89afea63fec62c6" diff --git a/octopoes/pyproject.toml b/octopoes/pyproject.toml index b682f7c4501..822ee4e0bf0 100644 --- a/octopoes/pyproject.toml +++ b/octopoes/pyproject.toml @@ -46,6 +46,7 @@ structlog = "^24.2.0" asgiref = "^3.8.1" jmespath-community = "^1.1.3" xxhash = "^3.5.0" +jcs = "^0.2.1" [tool.poetry.group.dev.dependencies] robotframework = "^7.0" diff --git a/octopoes/requirements-dev.txt b/octopoes/requirements-dev.txt index 4162d46aa68..a551741a832 100644 --- a/octopoes/requirements-dev.txt +++ b/octopoes/requirements-dev.txt @@ -362,6 +362,9 @@ importlib-metadata==8.5.0 ; python_version >= "3.10" and python_version < "4.0" iniconfig==2.0.0 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 +jcs==0.2.1 ; python_version >= "3.10" and python_version < "4.0" \ + --hash=sha256:9f20360b2f3b0a410d65493b448f96306d80e37fb46283f3f4aa5db2c7c1472b \ + --hash=sha256:e23a3e1de60f832d33cd811bb9c3b3be79219cdf95f63b88f0972732c3fa8476 jmespath-community==1.1.3 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:4b80a7e533d33952a5ecb258f5b8c5851244705139726c187c2795978c2e98fb jsonpatch==1.33 ; python_version >= "3.10" and python_version < "4.0" \ diff --git a/octopoes/requirements.txt b/octopoes/requirements.txt index 4538f3b45de..5284e773504 100644 --- a/octopoes/requirements.txt +++ b/octopoes/requirements.txt @@ -293,6 +293,9 @@ idna==3.10 ; python_version >= "3.10" and python_version < "4.0" \ importlib-metadata==8.5.0 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b \ --hash=sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7 +jcs==0.2.1 ; python_version >= "3.10" and python_version < "4.0" \ + --hash=sha256:9f20360b2f3b0a410d65493b448f96306d80e37fb46283f3f4aa5db2c7c1472b \ + --hash=sha256:e23a3e1de60f832d33cd811bb9c3b3be79219cdf95f63b88f0972732c3fa8476 jmespath-community==1.1.3 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:4b80a7e533d33952a5ecb258f5b8c5851244705139726c187c2795978c2e98fb jsonschema-specifications==2024.10.1 ; python_version >= "3.10" and python_version < "4.0" \ From 5cca668446461c2f8bfaa5da97d2a42bf8deabf1 Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 29 Jan 2025 10:20:17 +0100 Subject: [PATCH 14/17] Implement serialization --- octopoes/.ci/docker-compose.yml | 2 +- octopoes/nibbles/internetnl/nibble.py | 37 +++++++------- octopoes/nibbles/runner.py | 48 +++++++++++++------ .../integration/test_internetnl_nibble.py | 2 +- 4 files changed, 55 insertions(+), 34 deletions(-) diff --git a/octopoes/.ci/docker-compose.yml b/octopoes/.ci/docker-compose.yml index d03dc811b4d..d4e33b42e13 100644 --- a/octopoes/.ci/docker-compose.yml +++ b/octopoes/.ci/docker-compose.yml @@ -17,7 +17,7 @@ services: args: ENVIRONMENT: dev context: . - command: pytest -s tests/integration/test_internetnl_nibble.py --timeout=300 + command: pytest tests/integration --timeout=300 depends_on: - xtdb - ci_octopoes diff --git a/octopoes/nibbles/internetnl/nibble.py b/octopoes/nibbles/internetnl/nibble.py index cfc14d1c979..672e18e3d8e 100644 --- a/octopoes/nibbles/internetnl/nibble.py +++ b/octopoes/nibbles/internetnl/nibble.py @@ -67,27 +67,30 @@ def pull(statements: list[str]) -> str: """ ] + null_query = '{:query {:find [(pull ?var [])] :where [[?var :object_type ""]]}}' sgn = "".join(str(int(isinstance(target, Reference))) for target in targets) ref_query = ["[?hostname :Hostname/primary_key]"] - if sgn == "100": + if sgn.startswith("1"): ref_query = [f'[?hostname :Hostname/primary_key "{str(targets[0])}"]'] - elif sgn == "010": - ref_query = [f'[?hostname :Hostname/primary_key "{str(targets[1]).split("|")[-1]}"]'] - elif sgn == "001": - tokens = str(targets[2]).split("|")[1:-1] - target_reference = Reference.from_str("|".join(tokens)) - if tokens[0] == "Hostname": - hostname = target_reference.tokenized - elif tokens[0] in {"HostnameHTTPURL", "Website"}: - hostname = target_reference.tokenized.hostname - elif tokens[0] == "HTTPResource": - hostname = target_reference.tokenized.website.hostname - elif tokens[0] == "HTTPHeader": - hostname = target_reference.tokenized.resource.website.hostname + elif sgn.endswith("1"): + ref = str(targets[1]).split("|") + if ref[-1] == "KAT-INTERNETNL": + return null_query else: - raise ValueError() - hostname_pk = Hostname(name=hostname.name, network=Network(name=hostname.network.name).reference).reference - ref_query = [f'[?hostname :Hostname/primary_key "{str(hostname_pk)}"]'] + tokens = ref[1:-1] + target_reference = Reference.from_str("|".join(tokens)) + if tokens[0] == "Hostname": + hostname = target_reference.tokenized + elif tokens[0] in {"HostnameHTTPURL", "Website"}: + hostname = target_reference.tokenized.hostname + elif tokens[0] == "HTTPResource": + hostname = target_reference.tokenized.website.hostname + elif tokens[0] == "HTTPHeader": + hostname = target_reference.tokenized.resource.website.hostname + else: + return null_query + hostname_pk = Hostname(name=hostname.name, network=Network(name=hostname.network.name).reference).reference + ref_query = [f'[?hostname :Hostname/primary_key "{str(hostname_pk)}"]'] return pull(ref_query + base_query) diff --git a/octopoes/nibbles/runner.py b/octopoes/nibbles/runner.py index 73192cb12c3..5e3eb88dd55 100644 --- a/octopoes/nibbles/runner.py +++ b/octopoes/nibbles/runner.py @@ -1,8 +1,11 @@ -import json from collections.abc import Iterable from datetime import datetime +from enum import Enum +from ipaddress import IPv4Address, IPv6Address from typing import Any +import jcs +from pydantic import AnyUrl, BaseModel from xxhash import xxh3_128_hexdigest as xxh3 from nibbles.definitions import NibbleDefinition, get_nibble_definitions @@ -51,21 +54,39 @@ def flatten(items: Iterable[Any | Iterable[Any | None] | None]) -> Iterable[OOI] continue +def serialize(obj: Any) -> str: + """ + Serialize arbitrary objects + """ + + def breakdown(obj: Any) -> Iterable[str]: + """ + Breakdown Iterable objects so they can be `model_dump`'ed + """ + if isinstance(obj, Iterable) and not isinstance(obj, str | bytes): + if isinstance(obj, dict): + yield jcs.canonicalize(obj).decode() + else: + for item in obj: + yield from breakdown(item) + else: + if isinstance(obj, BaseModel): + yield serialize(obj.model_dump()) + elif isinstance(obj, Enum): + yield jcs.canonicalize(obj.value).decode() + elif isinstance(obj, AnyUrl | IPv6Address | IPv4Address | Reference): + yield jcs.canonicalize(str(obj)).decode() + else: + yield jcs.canonicalize(obj).decode() + + return "|".join(breakdown(obj)) + + def nibble_hasher(data: Iterable, additional: str | None = None) -> str: """ Hash the nibble generated data with its content together with the nibble checksum """ - return xxh3( - "".join( - [ - json.dumps(json.loads(ooi.model_dump_json()), sort_keys=True) - if isinstance(ooi, OOI) - else json.dumps(ooi, sort_keys=True) - for ooi in data - ] - ) - + (additional or "") - ) + return xxh3("|".join([serialize(ooi) for ooi in data]) + (additional or "")) class NibblesRunner: @@ -79,9 +100,6 @@ def __init__( self.nibbles: dict[str, NibbleDefinition] = get_nibble_definitions() self.federated: bool = False - def __del__(self): - self._write(datetime.now()) - def update_nibbles(self, valid_time: datetime, new_nibbles: dict[str, NibbleDefinition] = get_nibble_definitions()): old_checksums = {nibble.id: nibble._checksum for nibble in self.nibbles.values()} self.nibbles = new_nibbles diff --git a/octopoes/tests/integration/test_internetnl_nibble.py b/octopoes/tests/integration/test_internetnl_nibble.py index ab437f563df..da36a49fd8e 100644 --- a/octopoes/tests/integration/test_internetnl_nibble.py +++ b/octopoes/tests/integration/test_internetnl_nibble.py @@ -18,7 +18,7 @@ STATIC_IP = ".".join((4 * "1 ").split()) -def test_internetnl_query(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): +def test_internetnl_nibble(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): xtdb_octopoes_service.nibbler.nibbles = {internetnl.id: internetnl} network = Network(name="internet") From 65200784ba084a489628a07d814fa44679ce39ab Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 29 Jan 2025 12:33:17 +0100 Subject: [PATCH 15/17] Fix internetnl [FIXED] --- octopoes/nibbles/internetnl/nibble.py | 2 +- .../integration/test_internetnl_nibble.py | 66 ++++++++++++++++++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/octopoes/nibbles/internetnl/nibble.py b/octopoes/nibbles/internetnl/nibble.py index 672e18e3d8e..9e48b98ca61 100644 --- a/octopoes/nibbles/internetnl/nibble.py +++ b/octopoes/nibbles/internetnl/nibble.py @@ -47,7 +47,7 @@ def pull(statements: list[str]) -> str: """ [?hostname :object_type "Hostname"] [?website :Website/hostname ?hostname] - (or-join [?finding] + (or-join [?finding ?hostname ?website] [?finding :Finding/ooi ?hostname] (and [?hostnamehttpurl :HostnameHTTPURL/netloc ?hostname] diff --git a/octopoes/tests/integration/test_internetnl_nibble.py b/octopoes/tests/integration/test_internetnl_nibble.py index da36a49fd8e..83b10ddedeb 100644 --- a/octopoes/tests/integration/test_internetnl_nibble.py +++ b/octopoes/tests/integration/test_internetnl_nibble.py @@ -1,16 +1,20 @@ import os from datetime import datetime +from itertools import permutations from unittest.mock import Mock import pytest from nibbles.internetnl.nibble import NIBBLE as internetnl +from nibbles.internetnl.nibble import query from octopoes.core.service import OctopoesService +from octopoes.models import Reference from octopoes.models.ooi.dns.zone import Hostname from octopoes.models.ooi.findings import Finding, KATFindingType from octopoes.models.ooi.network import IPAddressV4, IPPort, Network, Protocol from octopoes.models.ooi.service import IPService, Service from octopoes.models.ooi.web import HostnameHTTPURL, HTTPHeader, HTTPResource, WebScheme, Website +from octopoes.repositories.ooi_repository import OOIRepository if os.environ.get("CI") != "1": pytest.skip("Needs XTDB multinode container.", allow_module_level=True) @@ -28,11 +32,11 @@ def test_internetnl_nibble(xtdb_octopoes_service: OctopoesService, event_manager xtdb_octopoes_service.ooi_repository.save(hostname, valid_time) web_url = HostnameHTTPURL( - network=network.reference, netloc=hostname.reference, port=443, path="/", scheme=WebScheme.HTTP + network=network.reference, netloc=hostname.reference, port=80, path="/", scheme=WebScheme.HTTP ) xtdb_octopoes_service.ooi_repository.save(web_url, valid_time) - service = Service(name="https") + service = Service(name="http") xtdb_octopoes_service.ooi_repository.save(service, valid_time) ip_address = IPAddressV4(address=STATIC_IP, network=network.reference) @@ -67,3 +71,61 @@ def test_internetnl_nibble(xtdb_octopoes_service: OctopoesService, event_manager assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 2 assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 2 + + +def create_port( + xtdb_ooi_repository: OOIRepository, refs: tuple, ip: str, port: int, valid_time: datetime +) -> IPAddressV4: + network, hostname, service = refs + ip_obj = IPAddressV4(address=ip, network=network) + ipport = IPPort(port=port, address=ip_obj.reference, protocol=Protocol.TCP) + ip_service = IPService(ip_port=ipport.reference, service=service) + website = Website(ip_service=ip_service.reference, hostname=hostname) + ft = KATFindingType(id="KAT-HTTPS-NOT-AVAILABLE") + finding = Finding( + ooi=website.reference, finding_type=ft.reference, description="HTTP port is open, but HTTPS port is not open" + ) + + xtdb_ooi_repository.save(ipport, valid_time) + xtdb_ooi_repository.save(ip_obj, valid_time) + xtdb_ooi_repository.save(ip_service, valid_time) + xtdb_ooi_repository.save(website, valid_time) + xtdb_ooi_repository.save(ft, valid_time) + xtdb_ooi_repository.save(finding, valid_time) + + xtdb_ooi_repository.commit() + + return ip + + +def ip_generator(): + for ip in permutations(range(2, 256), 4): + yield ".".join([str(x) for x in ip]) + + +def test_internetnl_nibble_query(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + xtdb_octopoes_service.nibbler.nibbles = {internetnl.id: internetnl} + + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + + ip = ip_generator() + http_service = Service(name="http") + + for i in range(3): + h = Hostname(name=f"www.x{i}.xyz", network=network.reference) + xtdb_octopoes_service.ooi_repository.save(h, valid_time) + http_refs = (network.reference, h.reference, http_service.reference) + create_port(xtdb_octopoes_service.ooi_repository, http_refs, next(ip), 80, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + + edn = query([None, None]) + print(edn) + result = xtdb_octopoes_service.ooi_repository.session.client.query(edn) + print(result) + + edn = query([Reference.from_str("Hostname|internet|www.x1.xyz"), None]) + print(edn) + result = xtdb_octopoes_service.ooi_repository.session.client.query(edn) + print(result) From cd25ff75a950feb1c20edb9770c508d7802ee9a9 Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 29 Jan 2025 14:02:50 +0100 Subject: [PATCH 16/17] 3.10 happy again? --- octopoes/nibbles/definitions.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/octopoes/nibbles/definitions.py b/octopoes/nibbles/definitions.py index dda5825e132..1f797e03823 100644 --- a/octopoes/nibbles/definitions.py +++ b/octopoes/nibbles/definitions.py @@ -4,7 +4,7 @@ from collections.abc import Callable, Iterable from pathlib import Path from types import MethodType, ModuleType -from typing import Any +from typing import Any, get_origin import structlog from pydantic import BaseModel @@ -34,7 +34,11 @@ def __eq__(self, other): @property def triggers(self) -> set[type[OOI]]: - if isinstance(self.object_type, type) and issubclass(self.object_type, OOI): + if ( + isinstance(self.object_type, type) + and get_origin(self.object_type) is None + and issubclass(self.object_type, OOI) + ): return {self.object_type} | self.additional else: return self.additional From 5a40851aa9e770f0b7ffcbe77926b88466c1a240 Mon Sep 17 00:00:00 2001 From: Benny Date: Wed, 29 Jan 2025 14:46:09 +0100 Subject: [PATCH 17/17] Actual internetnl test --- .../integration/test_internetnl_nibble.py | 41 +++++++++++++++++-- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/octopoes/tests/integration/test_internetnl_nibble.py b/octopoes/tests/integration/test_internetnl_nibble.py index 83b10ddedeb..fd49aeaf41e 100644 --- a/octopoes/tests/integration/test_internetnl_nibble.py +++ b/octopoes/tests/integration/test_internetnl_nibble.py @@ -4,6 +4,7 @@ from unittest.mock import Mock import pytest +from jmespath import search from nibbles.internetnl.nibble import NIBBLE as internetnl from nibbles.internetnl.nibble import query @@ -121,11 +122,43 @@ def test_internetnl_nibble_query(xtdb_octopoes_service: OctopoesService, event_m event_manager.complete_process_events(xtdb_octopoes_service) edn = query([None, None]) - print(edn) result = xtdb_octopoes_service.ooi_repository.session.client.query(edn) - print(result) + assert ( + len( + { + xtdb_octopoes_service.ooi_repository.parse_as(Hostname, obj) + for obj in search(internetnl.signature[0].parser, result) + } + ) + == 3 + ) + assert ( + len( + { + xtdb_octopoes_service.ooi_repository.parse_as(list[Finding], obj) + for obj in search(internetnl.signature[1].parser, result) + }.pop() + ) + == 6 + ) edn = query([Reference.from_str("Hostname|internet|www.x1.xyz"), None]) - print(edn) result = xtdb_octopoes_service.ooi_repository.session.client.query(edn) - print(result) + assert ( + len( + { + xtdb_octopoes_service.ooi_repository.parse_as(Hostname, obj) + for obj in search(internetnl.signature[0].parser, result) + } + ) + == 1 + ) + assert ( + len( + { + xtdb_octopoes_service.ooi_repository.parse_as(list[Finding], obj) + for obj in search(internetnl.signature[1].parser, result) + }.pop() + ) + == 2 + )