diff --git a/octopoes/bits/https_availability/bit.py b/octopoes/bits/https_availability/bit.py deleted file mode 100644 index 762dde5e3dc..00000000000 --- a/octopoes/bits/https_availability/bit.py +++ /dev/null @@ -1,15 +0,0 @@ -from bits.definitions import BitDefinition, BitParameterDefinition -from octopoes.models.ooi.network import IPAddress, IPPort -from octopoes.models.ooi.web import Website - -BIT = BitDefinition( - id="https-availability", - consumes=IPAddress, - parameters=[ - BitParameterDefinition(ooi_type=IPPort, relation_path="address"), - BitParameterDefinition( - ooi_type=Website, relation_path="ip_service.ip_port.address" - ), # we place the findings on the http websites - ], - module="bits.https_availability.https_availability", -) diff --git a/octopoes/bits/https_availability/https_availability.py b/octopoes/bits/https_availability/https_availability.py deleted file mode 100644 index 2abec18b340..00000000000 --- a/octopoes/bits/https_availability/https_availability.py +++ /dev/null @@ -1,21 +0,0 @@ -from collections.abc import Iterator -from typing import Any - -from octopoes.models import OOI -from octopoes.models.ooi.findings import Finding, KATFindingType -from octopoes.models.ooi.network import IPAddress, IPPort -from octopoes.models.ooi.web import Website - - -def run(input_ooi: IPAddress, additional_oois: list[IPPort | Website], config: dict[str, Any]) -> Iterator[OOI]: - websites = [website for website in additional_oois if isinstance(website, Website)] - - open_ports = [port.port for port in additional_oois if isinstance(port, IPPort)] - if 80 in open_ports and 443 not in open_ports: - ft = KATFindingType(id="KAT-HTTPS-NOT-AVAILABLE") - for website in websites: - yield Finding( - ooi=website.reference, - finding_type=ft.reference, - description="HTTP port is open, but HTTPS port is not open", - ) diff --git a/octopoes/bits/internetnl/bit.py b/octopoes/bits/internetnl/bit.py deleted file mode 100644 index e4b07dc359f..00000000000 --- a/octopoes/bits/internetnl/bit.py +++ /dev/null @@ -1,24 +0,0 @@ -from bits.definitions import BitDefinition, BitParameterDefinition -from octopoes.models.ooi.dns.zone import Hostname -from octopoes.models.ooi.findings import Finding -from octopoes.models.ooi.web import Website - -BIT = BitDefinition( - id="internet-nl", - consumes=Hostname, - parameters=[ - BitParameterDefinition(ooi_type=Finding, relation_path="ooi [is Hostname]"), # findings on hostnames - BitParameterDefinition( - ooi_type=Finding, relation_path="ooi [is HTTPResource].website.hostname" - ), # findings on resources - BitParameterDefinition( - ooi_type=Finding, relation_path="ooi [is HTTPHeader].resource.website.hostname" - ), # findings on headers - BitParameterDefinition(ooi_type=Finding, relation_path="ooi [is Website].hostname"), # findings on websites - BitParameterDefinition( - ooi_type=Finding, relation_path="ooi [is HostnameHTTPURL].netloc" - ), # findings on weburls - BitParameterDefinition(ooi_type=Website, relation_path="hostname"), # only websites have to comply - ], - module="bits.internetnl.internetnl", -) diff --git a/octopoes/bits/internetnl/internetnl.py b/octopoes/bits/internetnl/internetnl.py deleted file mode 100644 index a0058b22fc8..00000000000 --- a/octopoes/bits/internetnl/internetnl.py +++ /dev/null @@ -1,50 +0,0 @@ -from collections.abc import Iterator -from typing import Any - -from octopoes.models import OOI -from octopoes.models.ooi.dns.zone import Hostname -from octopoes.models.ooi.findings import Finding, KATFindingType -from octopoes.models.ooi.web import Website - - -def run(input_ooi: Hostname, additional_oois: list[Finding | Website], config: dict[str, Any]) -> Iterator[OOI]: - # only websites have to comply with the internetnl rules - websites = [websites for websites in additional_oois if isinstance(websites, Website)] - if not websites: - return - - finding_ids = [finding.finding_type.tokenized.id for finding in additional_oois if isinstance(finding, Finding)] - - result = "" - internetnl_findings = { - "KAT-WEBSERVER-NO-IPV6": "This webserver does not have an IPv6 address", - "KAT-NAMESERVER-NO-TWO-IPV6": "This webserver does not have at least two nameservers with an IPv6 address", - "KAT-NO-DNSSEC": "This webserver is not DNSSEC signed", - "KAT-INVALID-DNSSEC": "The DNSSEC signature of this webserver is not valid", - "KAT-NO-HSTS": "This website has at least one webpage with a missing Strict-Transport-Policy header", - "KAT-NO-CSP": "This website has at least one webpage with a missing Content-Security-Policy header", - "KAT-NO-X-FRAME-OPTIONS": "This website has at least one webpage with a missing X-Frame-Options header", - "KAT-NO-X-CONTENT-TYPE-OPTIONS": ( - "This website has at least one webpage with a missing X-Content-Type-Options header" - ), - "KAT-CSP-VULNERABILITIES": "This website has at least one webpage with a mis-configured CSP header", - "KAT-HSTS-VULNERABILITIES": "This website has at least one webpage with a mis-configured HSTS header", - "KAT-NO-CERTIFICATE": "This website does not have an SSL certificate", - "KAT-HTTPS-NOT-AVAILABLE": "HTTPS is not available for this website", - "KAT-SSL-CERT-HOSTNAME-MISMATCH": "The SSL certificate of this website does not match the hostname", - "KAT-HTTPS-REDIRECT": "This website has at least one HTTP URL that does not redirect to HTTPS", - } - - for finding, description in internetnl_findings.items(): - if finding in finding_ids: - result += f"{description}\n" - - if result: - ft = KATFindingType(id="KAT-INTERNETNL") - yield ft - f = Finding( - finding_type=ft.reference, - ooi=input_ooi.reference, - description=f"This hostname has at least one website with the following finding(s): {result}", - ) - yield f diff --git a/octopoes/nibbles/definitions.py b/octopoes/nibbles/definitions.py index 51d048ed325..1f797e03823 100644 --- a/octopoes/nibbles/definitions.py +++ b/octopoes/nibbles/definitions.py @@ -4,7 +4,7 @@ from collections.abc import Callable, Iterable from pathlib import Path from types import MethodType, ModuleType -from typing import Any +from typing import Any, get_origin import structlog from pydantic import BaseModel @@ -19,9 +19,10 @@ class NibbleParameter(BaseModel): - object_type: type[Any] + object_type: Any parser: str = "[]" optional: bool = False + additional: set[type[OOI]] = set() def __eq__(self, other): if isinstance(other, NibbleParameter): @@ -31,12 +32,24 @@ def __eq__(self, other): else: return False + @property + def triggers(self) -> set[type[OOI]]: + if ( + isinstance(self.object_type, type) + and get_origin(self.object_type) is None + and issubclass(self.object_type, OOI) + ): + return {self.object_type} | self.additional + else: + return self.additional + class NibbleDefinition(BaseModel): id: str signature: list[NibbleParameter] query: str | Callable[[list[Reference | None]], str] | None = None enabled: bool = True + additional: set[type[OOI]] = set() _payload: MethodType | None = None _checksum: str | None = None @@ -53,6 +66,10 @@ def __hash__(self): def _ini(self) -> dict[str, Any]: return {"id": self.id, "enabled": self.enabled, "checksum": self._checksum} + @property + def triggers(self) -> set[type[OOI]]: + return set.union(*[sgn.triggers for sgn in self.signature]) | self.additional + def get_nibble_definitions() -> dict[str, NibbleDefinition]: nibble_definitions = {} diff --git a/octopoes/nibbles/disallowed_csp_hostnames/nibble.py b/octopoes/nibbles/disallowed_csp_hostnames/nibble.py index fcf230be3e3..e4430a1455f 100644 --- a/octopoes/nibbles/disallowed_csp_hostnames/nibble.py +++ b/octopoes/nibbles/disallowed_csp_hostnames/nibble.py @@ -100,7 +100,7 @@ def query(targets: list[Reference | None]) -> str: NIBBLE = NibbleDefinition( - id="disallowed-csp-hostnames", + id="disallowed_csp_hostnames", signature=[ NibbleParameter(object_type=HTTPHeaderHostname, parser="[*][?object_type == 'HTTPHeaderHostname'][]"), NibbleParameter(object_type=Config, parser="[*][?object_type == 'Config'][]", optional=True), diff --git a/octopoes/bits/https_availability/__init__.py b/octopoes/nibbles/https_availability/__init__.py similarity index 100% rename from octopoes/bits/https_availability/__init__.py rename to octopoes/nibbles/https_availability/__init__.py diff --git a/octopoes/nibbles/https_availability/https_availability.py b/octopoes/nibbles/https_availability/https_availability.py new file mode 100644 index 00000000000..e1c5b9f4f31 --- /dev/null +++ b/octopoes/nibbles/https_availability/https_availability.py @@ -0,0 +1,19 @@ +from collections.abc import Iterator + +from octopoes.models import OOI +from octopoes.models.ooi.findings import Finding, KATFindingType +from octopoes.models.ooi.network import IPAddress, IPPort +from octopoes.models.ooi.web import Website + + +def nibble(ipaddress: IPAddress, ipport80: IPPort, website: Website, port443s: int) -> Iterator[OOI]: + _ = ipaddress + _ = ipport80 + if port443s < 1: + ft = KATFindingType(id="KAT-HTTPS-NOT-AVAILABLE") + yield ft + yield Finding( + ooi=website.reference, + finding_type=ft.reference, + description="HTTP port is open, but HTTPS port is not open", + ) diff --git a/octopoes/nibbles/https_availability/nibble.py b/octopoes/nibbles/https_availability/nibble.py new file mode 100644 index 00000000000..09f2f99413c --- /dev/null +++ b/octopoes/nibbles/https_availability/nibble.py @@ -0,0 +1,66 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models import Reference +from octopoes.models.ooi.network import IPAddress, IPPort +from octopoes.models.ooi.web import Website + + +def query(targets: list[Reference | None]) -> str: + def pull(statements: list[str]) -> str: + return f""" + {{ + :query {{ + :find [(pull ?ipaddress [*]) (pull ?ipport80 [*]) (pull ?website [*]) (- (count ?ipport443) 1)] + :where [ + {" ".join(statements)} + ] + }} + }} + """ + + base_query = [ + """ + [?website :Website/ip_service ?ip_service] + [?ipservice :IPService/ip_port ?ipport80] + [?ipport80 :IPPort/port 80] + [?ipport80 :IPPort/address ?ipaddress] + (or + (and [?ipport443 :IPPort/address ?ipaddress][?ipport443 :IPPort/port 443]) + [(identity nil) ?ipport443] + ) + """ + ] + + ref_queries = [ + f'[?ipaddress :IPAddress/primary_key "{str(targets[0])}"]', + f'[?ipport80 :IPPort/primary_key "{str(targets[1])}"]', + f'[?website :Website/primary_key "{str(targets[2])}"]', + ] + + sgn = "".join(str(int(isinstance(target, Reference))) for target in targets) + if sgn == "1000": + return pull(ref_queries[0:1] + base_query) + elif sgn == "0100": + if int(str(targets[1]).split("|")[-1]) == 80: + return pull(ref_queries[1:2] + base_query) + else: + return pull(base_query) + elif sgn == "0010": + return pull(ref_queries[2:3] + base_query) + elif sgn == "1110": + return pull(ref_queries + base_query) + else: + return pull(base_query) + + +NIBBLE = NibbleDefinition( + id="https_availability", + signature=[ + NibbleParameter( + object_type=IPAddress, parser="[*][?object_type == 'IPAddressV6' || object_type == 'IPAddressV4'][]" + ), + NibbleParameter(object_type=IPPort, parser="[*][?object_type == 'IPPort'][]"), + NibbleParameter(object_type=Website, parser="[*][?object_type == 'Website'][]"), + NibbleParameter(object_type=int, parser="[*][-1][]"), + ], + query=query, +) diff --git a/octopoes/bits/internetnl/__init__.py b/octopoes/nibbles/internetnl/__init__.py similarity index 100% rename from octopoes/bits/internetnl/__init__.py rename to octopoes/nibbles/internetnl/__init__.py diff --git a/octopoes/nibbles/internetnl/internetnl.py b/octopoes/nibbles/internetnl/internetnl.py new file mode 100644 index 00000000000..85e24a274a9 --- /dev/null +++ b/octopoes/nibbles/internetnl/internetnl.py @@ -0,0 +1,18 @@ +from collections.abc import Iterator + +from octopoes.models import OOI +from octopoes.models.ooi.dns.zone import Hostname +from octopoes.models.ooi.findings import Finding, KATFindingType + + +def nibble(hostname: Hostname, findings: list[Finding]) -> Iterator[OOI]: + result = "\n".join([str(finding.description) for finding in findings]) + + if result: + ft = KATFindingType(id="KAT-INTERNETNL") + yield ft + yield Finding( + finding_type=ft.reference, + ooi=hostname.reference, + description=f"This hostname has at least one website with the following finding(s): {result}", + ) diff --git a/octopoes/nibbles/internetnl/nibble.py b/octopoes/nibbles/internetnl/nibble.py new file mode 100644 index 00000000000..9e48b98ca61 --- /dev/null +++ b/octopoes/nibbles/internetnl/nibble.py @@ -0,0 +1,104 @@ +from nibbles.definitions import NibbleDefinition, NibbleParameter +from octopoes.models import Reference +from octopoes.models.ooi.dns.zone import Hostname, Network +from octopoes.models.ooi.findings import Finding + +finding_types = [ + "KAT-WEBSERVER-NO-IPV6", + "KAT-NAMESERVER-NO-TWO-IPV6", + "KAT-NO-DNSSEC", + "KAT-INVALID-DNSSEC", + "KAT-NO-HSTS", + "KAT-NO-CSP", + "KAT-NO-X-FRAME-OPTIONS", + "KAT-NO-X-CONTENT-TYPE-OPTIONS", + "KAT-CSP-VULNERABILITIES", + "KAT-HSTS-VULNERABILITIES", + "KAT-NO-CERTIFICATE", + "KAT-HTTPS-NOT-AVAILABLE", + "KAT-SSL-CERT-HOSTNAME-MISMATCH", + "KAT-HTTPS-REDIRECT", +] + + +def or_finding_types() -> str: + clauses = "".join([f'[?finding :Finding/finding_type "{ft}"]' for ft in finding_types]) + return f"(or {clauses})" + + +def query(targets: list[Reference | None]) -> str: + def pull(statements: list[str]) -> str: + return f""" + {{ + :query {{ + :find [ + (pull ?hostname [*]) + (pull ?website [*]) + (pull ?finding [*]) + ] + :where [ + {" ".join(statements)} + ] + }} + }} + """ + + base_query = [ + """ + [?hostname :object_type "Hostname"] + [?website :Website/hostname ?hostname] + (or-join [?finding ?hostname ?website] + [?finding :Finding/ooi ?hostname] + (and + [?hostnamehttpurl :HostnameHTTPURL/netloc ?hostname] + [?finding :Finding/ooi ?hostnamehttpurl] + ) + [?finding :Finding/ooi ?website] + (and + [?resource :HTTPResource/website ?website] + [?finding :Finding/ooi ?resource] + ) + (and + [?header :HTTPHeader/resource ?resource] + [?resource :HTTPResource/website ?website] + [?finding :Finding/ooi ?header] + ) + ) + """ + ] + + null_query = '{:query {:find [(pull ?var [])] :where [[?var :object_type ""]]}}' + sgn = "".join(str(int(isinstance(target, Reference))) for target in targets) + ref_query = ["[?hostname :Hostname/primary_key]"] + if sgn.startswith("1"): + ref_query = [f'[?hostname :Hostname/primary_key "{str(targets[0])}"]'] + elif sgn.endswith("1"): + ref = str(targets[1]).split("|") + if ref[-1] == "KAT-INTERNETNL": + return null_query + else: + tokens = ref[1:-1] + target_reference = Reference.from_str("|".join(tokens)) + if tokens[0] == "Hostname": + hostname = target_reference.tokenized + elif tokens[0] in {"HostnameHTTPURL", "Website"}: + hostname = target_reference.tokenized.hostname + elif tokens[0] == "HTTPResource": + hostname = target_reference.tokenized.website.hostname + elif tokens[0] == "HTTPHeader": + hostname = target_reference.tokenized.resource.website.hostname + else: + return null_query + hostname_pk = Hostname(name=hostname.name, network=Network(name=hostname.network.name).reference).reference + ref_query = [f'[?hostname :Hostname/primary_key "{str(hostname_pk)}"]'] + return pull(ref_query + base_query) + + +NIBBLE = NibbleDefinition( + id="internet_nl", + signature=[ + NibbleParameter(object_type=Hostname, parser="[*][?object_type == 'Hostname'][]"), + NibbleParameter(object_type=list[Finding], parser="[[*][?object_type == 'Finding'][]]", additional={Finding}), + ], + query=query, +) diff --git a/octopoes/nibbles/runner.py b/octopoes/nibbles/runner.py index 1a8828a7c6b..5e3eb88dd55 100644 --- a/octopoes/nibbles/runner.py +++ b/octopoes/nibbles/runner.py @@ -1,8 +1,11 @@ -import json from collections.abc import Iterable from datetime import datetime +from enum import Enum +from ipaddress import IPv4Address, IPv6Address from typing import Any +import jcs +from pydantic import AnyUrl, BaseModel from xxhash import xxh3_128_hexdigest as xxh3 from nibbles.definitions import NibbleDefinition, get_nibble_definitions @@ -51,21 +54,39 @@ def flatten(items: Iterable[Any | Iterable[Any | None] | None]) -> Iterable[OOI] continue +def serialize(obj: Any) -> str: + """ + Serialize arbitrary objects + """ + + def breakdown(obj: Any) -> Iterable[str]: + """ + Breakdown Iterable objects so they can be `model_dump`'ed + """ + if isinstance(obj, Iterable) and not isinstance(obj, str | bytes): + if isinstance(obj, dict): + yield jcs.canonicalize(obj).decode() + else: + for item in obj: + yield from breakdown(item) + else: + if isinstance(obj, BaseModel): + yield serialize(obj.model_dump()) + elif isinstance(obj, Enum): + yield jcs.canonicalize(obj.value).decode() + elif isinstance(obj, AnyUrl | IPv6Address | IPv4Address | Reference): + yield jcs.canonicalize(str(obj)).decode() + else: + yield jcs.canonicalize(obj).decode() + + return "|".join(breakdown(obj)) + + def nibble_hasher(data: Iterable, additional: str | None = None) -> str: """ Hash the nibble generated data with its content together with the nibble checksum """ - return xxh3( - "".join( - [ - json.dumps(json.loads(ooi.model_dump_json()), sort_keys=True) - if isinstance(ooi, OOI) - else json.dumps(ooi, sort_keys=True) - for ooi in data - ] - ) - + (additional or "") - ) + return xxh3("|".join([serialize(ooi) for ooi in data]) + (additional or "")) class NibblesRunner: @@ -79,9 +100,6 @@ def __init__( self.nibbles: dict[str, NibbleDefinition] = get_nibble_definitions() self.federated: bool = False - def __del__(self): - self._write(datetime.now()) - def update_nibbles(self, valid_time: datetime, new_nibbles: dict[str, NibbleDefinition] = get_nibble_definitions()): old_checksums = {nibble.id: nibble._checksum for nibble in self.nibbles.values()} self.nibbles = new_nibbles @@ -179,15 +197,14 @@ def _run(self, ooi: OOI, valid_time: datetime) -> dict[str, dict[tuple[Any, ...] nibblet_nibbles = {self.nibbles[nibblet.method] for nibblet in nibblets if nibblet.method in self.nibbles} for nibble in filter( - lambda x: any(isinstance(ooi, param.object_type) for param in x.signature) and x not in nibblet_nibbles, + lambda x: x.enabled and x not in nibblet_nibbles and any(isinstance(ooi, t) for t in x.triggers), self.nibbles.values(), ): - if nibble.enabled: - if len(nibble.signature) > 1: - self._write(valid_time) - args = self.ooi_repository.nibble_query(ooi, nibble, valid_time) - results = {tuple(arg): set(flatten([nibble(arg)])) for arg in args} - return_value |= {nibble.id: results} + if len(nibble.signature) > 1: + self._write(valid_time) + args = self.ooi_repository.nibble_query(ooi, nibble, valid_time) + results = {tuple(arg): set(flatten([nibble(arg)])) for arg in args} + return_value |= {nibble.id: results} self.cache = merge_results(self.cache, {ooi: return_value}) return return_value diff --git a/octopoes/octopoes/repositories/ooi_repository.py b/octopoes/octopoes/repositories/ooi_repository.py index c171c42c19a..ed71b696665 100644 --- a/octopoes/octopoes/repositories/ooi_repository.py +++ b/octopoes/octopoes/repositories/ooi_repository.py @@ -284,6 +284,9 @@ def parse_as(cls, type_: type | list[type], obj: dict | list | set | Any) -> tup # list --> tuple if isinstance(type_, list): return tuple(cls.parse_as(type_t, o) for o, type_t in zip(obj, type_)) + elif hasattr(type_, "__origin__") and hasattr(type_, "__args__") and type_.__origin__ is list: + t = [type_.__args__[0]] * len(obj) + return tuple(cls.parse_as(t, o) for o, t in zip(obj, t)) else: return tuple(cls.parse_as(type_, o) for o in obj) else: @@ -919,7 +922,7 @@ def nibble_query( first = True arguments = [ ooi.reference - if sgn.object_type == type_by_name(ooi.get_ooi_type()) and (first and not (first := False)) + if type_by_name(ooi.get_ooi_type()) in sgn.triggers and (first and not (first := False)) else None for sgn in nibble.signature ] diff --git a/octopoes/poetry.lock b/octopoes/poetry.lock index 41925cfb6d9..81c7327c92e 100644 --- a/octopoes/poetry.lock +++ b/octopoes/poetry.lock @@ -796,6 +796,17 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "jcs" +version = "0.2.1" +description = "JCS - JSON Canonicalization" +optional = false +python-versions = ">=3.6.2" +files = [ + {file = "jcs-0.2.1-py3-none-any.whl", hash = "sha256:e23a3e1de60f832d33cd811bb9c3b3be79219cdf95f63b88f0972732c3fa8476"}, + {file = "jcs-0.2.1.tar.gz", hash = "sha256:9f20360b2f3b0a410d65493b448f96306d80e37fb46283f3f4aa5db2c7c1472b"}, +] + [[package]] name = "jmespath-community" version = "1.1.3" @@ -2347,4 +2358,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "e41ab6fe70fe531c50ec680a5620b21e7b5fac4eb45b9841eff945234a445ff2" +content-hash = "f79d45fed4bf27cd7d15665c1ca25ba5ac85911d4de10abaa89afea63fec62c6" diff --git a/octopoes/pyproject.toml b/octopoes/pyproject.toml index b682f7c4501..822ee4e0bf0 100644 --- a/octopoes/pyproject.toml +++ b/octopoes/pyproject.toml @@ -46,6 +46,7 @@ structlog = "^24.2.0" asgiref = "^3.8.1" jmespath-community = "^1.1.3" xxhash = "^3.5.0" +jcs = "^0.2.1" [tool.poetry.group.dev.dependencies] robotframework = "^7.0" diff --git a/octopoes/requirements-dev.txt b/octopoes/requirements-dev.txt index 4162d46aa68..a551741a832 100644 --- a/octopoes/requirements-dev.txt +++ b/octopoes/requirements-dev.txt @@ -362,6 +362,9 @@ importlib-metadata==8.5.0 ; python_version >= "3.10" and python_version < "4.0" iniconfig==2.0.0 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 +jcs==0.2.1 ; python_version >= "3.10" and python_version < "4.0" \ + --hash=sha256:9f20360b2f3b0a410d65493b448f96306d80e37fb46283f3f4aa5db2c7c1472b \ + --hash=sha256:e23a3e1de60f832d33cd811bb9c3b3be79219cdf95f63b88f0972732c3fa8476 jmespath-community==1.1.3 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:4b80a7e533d33952a5ecb258f5b8c5851244705139726c187c2795978c2e98fb jsonpatch==1.33 ; python_version >= "3.10" and python_version < "4.0" \ diff --git a/octopoes/requirements.txt b/octopoes/requirements.txt index 4538f3b45de..5284e773504 100644 --- a/octopoes/requirements.txt +++ b/octopoes/requirements.txt @@ -293,6 +293,9 @@ idna==3.10 ; python_version >= "3.10" and python_version < "4.0" \ importlib-metadata==8.5.0 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b \ --hash=sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7 +jcs==0.2.1 ; python_version >= "3.10" and python_version < "4.0" \ + --hash=sha256:9f20360b2f3b0a410d65493b448f96306d80e37fb46283f3f4aa5db2c7c1472b \ + --hash=sha256:e23a3e1de60f832d33cd811bb9c3b3be79219cdf95f63b88f0972732c3fa8476 jmespath-community==1.1.3 ; python_version >= "3.10" and python_version < "4.0" \ --hash=sha256:4b80a7e533d33952a5ecb258f5b8c5851244705139726c187c2795978c2e98fb jsonschema-specifications==2024.10.1 ; python_version >= "3.10" and python_version < "4.0" \ diff --git a/octopoes/tests/integration/test_https_availability_nibble.py b/octopoes/tests/integration/test_https_availability_nibble.py new file mode 100644 index 00000000000..314078d806a --- /dev/null +++ b/octopoes/tests/integration/test_https_availability_nibble.py @@ -0,0 +1,76 @@ +import os +from datetime import datetime +from unittest.mock import Mock + +import pytest +from nibbles.https_availability.nibble import NIBBLE as https_availability + +from octopoes.core.service import OctopoesService +from octopoes.models.ooi.dns.zone import Hostname +from octopoes.models.ooi.findings import Finding, KATFindingType +from octopoes.models.ooi.network import IPAddressV4, IPPort, Network, Protocol +from octopoes.models.ooi.service import IPService, Service +from octopoes.models.ooi.web import HostnameHTTPURL, HTTPHeader, HTTPResource, WebScheme, Website + +if os.environ.get("CI") != "1": + pytest.skip("Needs XTDB multinode container.", allow_module_level=True) + +STATIC_IP = ".".join((4 * "1 ").split()) + + +def test_https_availability_query(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + xtdb_octopoes_service.nibbler.nibbles = {https_availability.id: https_availability} + + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + + hostname = Hostname(name="example.com", network=network.reference) + xtdb_octopoes_service.ooi_repository.save(hostname, valid_time) + + web_url = HostnameHTTPURL( + network=network.reference, netloc=hostname.reference, port=443, path="/", scheme=WebScheme.HTTP + ) + xtdb_octopoes_service.ooi_repository.save(web_url, valid_time) + + service = Service(name="https") + xtdb_octopoes_service.ooi_repository.save(service, valid_time) + + ip_address = IPAddressV4(address=STATIC_IP, network=network.reference) + xtdb_octopoes_service.ooi_repository.save(ip_address, valid_time) + + port = IPPort(port=80, address=ip_address.reference, protocol=Protocol.TCP) + xtdb_octopoes_service.ooi_repository.save(port, valid_time) + + ip_service = IPService(ip_port=port.reference, service=service.reference) + xtdb_octopoes_service.ooi_repository.save(ip_service, valid_time) + + website = Website(ip_service=ip_service.reference, hostname=hostname.reference) + xtdb_octopoes_service.ooi_repository.save(website, valid_time) + + resource = HTTPResource(website=website.reference, web_url=web_url.reference) + xtdb_octopoes_service.ooi_repository.save(resource, valid_time) + + header = HTTPHeader( + resource=resource.reference, key="strict-transport-security", value="max-age=21536000; includeSubDomains" + ) + xtdb_octopoes_service.ooi_repository.save(header, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 1 + assert ( + xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).items[0].description + == "HTTP port is open, but HTTPS port is not open" + ) + assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 1 + assert ( + xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).items[0].id + == "KAT-HTTPS-NOT-AVAILABLE" + ) + + port443 = IPPort(port=443, address=ip_address.reference, protocol=Protocol.TCP) + xtdb_octopoes_service.ooi_repository.save(port443, valid_time) + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 0 + assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 0 diff --git a/octopoes/tests/integration/test_internetnl_nibble.py b/octopoes/tests/integration/test_internetnl_nibble.py new file mode 100644 index 00000000000..fd49aeaf41e --- /dev/null +++ b/octopoes/tests/integration/test_internetnl_nibble.py @@ -0,0 +1,164 @@ +import os +from datetime import datetime +from itertools import permutations +from unittest.mock import Mock + +import pytest +from jmespath import search +from nibbles.internetnl.nibble import NIBBLE as internetnl +from nibbles.internetnl.nibble import query + +from octopoes.core.service import OctopoesService +from octopoes.models import Reference +from octopoes.models.ooi.dns.zone import Hostname +from octopoes.models.ooi.findings import Finding, KATFindingType +from octopoes.models.ooi.network import IPAddressV4, IPPort, Network, Protocol +from octopoes.models.ooi.service import IPService, Service +from octopoes.models.ooi.web import HostnameHTTPURL, HTTPHeader, HTTPResource, WebScheme, Website +from octopoes.repositories.ooi_repository import OOIRepository + +if os.environ.get("CI") != "1": + pytest.skip("Needs XTDB multinode container.", allow_module_level=True) + +STATIC_IP = ".".join((4 * "1 ").split()) + + +def test_internetnl_nibble(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + xtdb_octopoes_service.nibbler.nibbles = {internetnl.id: internetnl} + + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + + hostname = Hostname(name="example.com", network=network.reference) + xtdb_octopoes_service.ooi_repository.save(hostname, valid_time) + + web_url = HostnameHTTPURL( + network=network.reference, netloc=hostname.reference, port=80, path="/", scheme=WebScheme.HTTP + ) + xtdb_octopoes_service.ooi_repository.save(web_url, valid_time) + + service = Service(name="http") + xtdb_octopoes_service.ooi_repository.save(service, valid_time) + + ip_address = IPAddressV4(address=STATIC_IP, network=network.reference) + xtdb_octopoes_service.ooi_repository.save(ip_address, valid_time) + + port = IPPort(port=80, address=ip_address.reference, protocol=Protocol.TCP) + xtdb_octopoes_service.ooi_repository.save(port, valid_time) + + ip_service = IPService(ip_port=port.reference, service=service.reference) + xtdb_octopoes_service.ooi_repository.save(ip_service, valid_time) + + website = Website(ip_service=ip_service.reference, hostname=hostname.reference) + xtdb_octopoes_service.ooi_repository.save(website, valid_time) + + resource = HTTPResource(website=website.reference, web_url=web_url.reference) + xtdb_octopoes_service.ooi_repository.save(resource, valid_time) + + header = HTTPHeader( + resource=resource.reference, key="strict-transport-security", value="max-age=21536000; includeSubDomains" + ) + xtdb_octopoes_service.ooi_repository.save(header, valid_time) + + ft = KATFindingType(id="KAT-HTTPS-NOT-AVAILABLE") + xtdb_octopoes_service.ooi_repository.save(ft, valid_time) + + finding = Finding( + ooi=website.reference, finding_type=ft.reference, description="HTTP port is open, but HTTPS port is not open" + ) + xtdb_octopoes_service.ooi_repository.save(finding, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + + assert xtdb_octopoes_service.ooi_repository.list_oois({Finding}, valid_time).count == 2 + assert xtdb_octopoes_service.ooi_repository.list_oois({KATFindingType}, valid_time).count == 2 + + +def create_port( + xtdb_ooi_repository: OOIRepository, refs: tuple, ip: str, port: int, valid_time: datetime +) -> IPAddressV4: + network, hostname, service = refs + ip_obj = IPAddressV4(address=ip, network=network) + ipport = IPPort(port=port, address=ip_obj.reference, protocol=Protocol.TCP) + ip_service = IPService(ip_port=ipport.reference, service=service) + website = Website(ip_service=ip_service.reference, hostname=hostname) + ft = KATFindingType(id="KAT-HTTPS-NOT-AVAILABLE") + finding = Finding( + ooi=website.reference, finding_type=ft.reference, description="HTTP port is open, but HTTPS port is not open" + ) + + xtdb_ooi_repository.save(ipport, valid_time) + xtdb_ooi_repository.save(ip_obj, valid_time) + xtdb_ooi_repository.save(ip_service, valid_time) + xtdb_ooi_repository.save(website, valid_time) + xtdb_ooi_repository.save(ft, valid_time) + xtdb_ooi_repository.save(finding, valid_time) + + xtdb_ooi_repository.commit() + + return ip + + +def ip_generator(): + for ip in permutations(range(2, 256), 4): + yield ".".join([str(x) for x in ip]) + + +def test_internetnl_nibble_query(xtdb_octopoes_service: OctopoesService, event_manager: Mock, valid_time: datetime): + xtdb_octopoes_service.nibbler.nibbles = {internetnl.id: internetnl} + + network = Network(name="internet") + xtdb_octopoes_service.ooi_repository.save(network, valid_time) + + ip = ip_generator() + http_service = Service(name="http") + + for i in range(3): + h = Hostname(name=f"www.x{i}.xyz", network=network.reference) + xtdb_octopoes_service.ooi_repository.save(h, valid_time) + http_refs = (network.reference, h.reference, http_service.reference) + create_port(xtdb_octopoes_service.ooi_repository, http_refs, next(ip), 80, valid_time) + + event_manager.complete_process_events(xtdb_octopoes_service) + + edn = query([None, None]) + result = xtdb_octopoes_service.ooi_repository.session.client.query(edn) + assert ( + len( + { + xtdb_octopoes_service.ooi_repository.parse_as(Hostname, obj) + for obj in search(internetnl.signature[0].parser, result) + } + ) + == 3 + ) + assert ( + len( + { + xtdb_octopoes_service.ooi_repository.parse_as(list[Finding], obj) + for obj in search(internetnl.signature[1].parser, result) + }.pop() + ) + == 6 + ) + + edn = query([Reference.from_str("Hostname|internet|www.x1.xyz"), None]) + result = xtdb_octopoes_service.ooi_repository.session.client.query(edn) + assert ( + len( + { + xtdb_octopoes_service.ooi_repository.parse_as(Hostname, obj) + for obj in search(internetnl.signature[0].parser, result) + } + ) + == 1 + ) + assert ( + len( + { + xtdb_octopoes_service.ooi_repository.parse_as(list[Finding], obj) + for obj in search(internetnl.signature[1].parser, result) + }.pop() + ) + == 2 + ) diff --git a/octopoes/tests/test_bits.py b/octopoes/tests/test_bits.py index e2e4f154755..c0f4605657f 100644 --- a/octopoes/tests/test_bits.py +++ b/octopoes/tests/test_bits.py @@ -1,8 +1,8 @@ -from bits.https_availability.https_availability import run as run_https_availability +from nibbles.https_availability.https_availability import nibble as run_https_availability from nibbles.oois_in_headers.oois_in_headers import nibble as run_oois_in_headers from octopoes.models.ooi.config import Config -from octopoes.models.ooi.findings import Finding +from octopoes.models.ooi.findings import Finding, KATFindingType from octopoes.models.ooi.network import IPPort from octopoes.models.ooi.web import URL, HTTPHeader, HTTPHeaderURL, Website @@ -42,7 +42,8 @@ def test_url_extracted_by_oois_in_headers_relative_path(http_resource_https): def test_finding_generated_when_443_not_open_and_80_is_open(): port_80 = IPPort(address="fake", protocol="tcp", port=80) website = Website(ip_service="fake", hostname="fake") - results = list(run_https_availability(None, [port_80, website], {})) - finding = results[0] - assert isinstance(finding, Finding) + results = list(run_https_availability(None, port_80, website, 0)) + finding = [result for result in results if isinstance(result, Finding)][0] assert finding.description == "HTTP port is open, but HTTPS port is not open" + katfindingtype = [result for result in results if not isinstance(result, Finding)][0] + assert isinstance(katfindingtype, KATFindingType) diff --git a/octopoes/tests/test_ooi_repository.py b/octopoes/tests/test_ooi_repository.py index 1c4c6e131de..0497973d682 100644 --- a/octopoes/tests/test_ooi_repository.py +++ b/octopoes/tests/test_ooi_repository.py @@ -164,3 +164,7 @@ def pull(ooi: OOI): [network, url, network] ) assert self.repository.parse_as(Network, [pull(network), pull(url), pull(url)]) == tuple([network, url, url]) + + assert self.repository.parse_as(list[Network], [pull(network), pull(network), pull(network)]) == tuple( + [network, network, network] + )