From b2fa9469b83b38c19e0df1fcf9b2c1465cfb9797 Mon Sep 17 00:00:00 2001 From: "Philipp Heil (zkdev)" Date: Mon, 18 Nov 2024 14:53:23 +0100 Subject: [PATCH 1/2] Migrate ClamAV to node filter --- config.py | 25 ++++++++++++++----------- malware/__main__.py | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/config.py b/config.py index 56973c5f..9e686375 100644 --- a/config.py +++ b/config.py @@ -83,14 +83,14 @@ class ClamAVConfig: time after which an artefact must be re-scanned at latest :param str aws_cfg_name cfg-element used to create s3 client to retrieve artefacts - :param tuple[str] artefact_types: - list of artefact types which should be scanned, other artefact types are skipped + :param Callable[Node, bool] node_filter: + filter of artefact nodes to explicitly in- or exclude artefacts from the clamav scan ''' delivery_service_url: str lookup_new_backlog_item_interval: int rescan_interval: int aws_cfg_name: str - artefact_types: tuple[str] + node_filter: collections.abc.Callable[[cnudie.iter.Node], bool] @dataclasses.dataclass(frozen=True) @@ -407,22 +407,25 @@ def deserialise_clamav_config( on_absent_message='artefacts of access type s3 will not be scanned' ) - artefact_types = tuple(deserialise_config_property( + matching_configs_raw = deserialise_config_property( config=clamav_config, - property_key='artefact_types', + property_key='matching_configs', default_config=default_config, - default_value=( - ocm.ArtefactType.OCI_IMAGE, - 'application/tar+vm-image-rootfs', - ), - )) + default_value=[], + ) + matching_configs = config_filter.matching_configs_from_dicts( + dicts=matching_configs_raw, + ) + node_filter = config_filter.filter_for_matching_configs( + configs=matching_configs, + ) return ClamAVConfig( delivery_service_url=delivery_service_url, lookup_new_backlog_item_interval=lookup_new_backlog_item_interval, rescan_interval=rescan_interval, aws_cfg_name=aws_cfg_name, - artefact_types=artefact_types, + node_filter=node_filter, ) diff --git a/malware/__main__.py b/malware/__main__.py index 30dd4758..f3b94435 100644 --- a/malware/__main__.py +++ b/malware/__main__.py @@ -248,7 +248,7 @@ def scan_and_upload( artefact=backlog_item.artefact, ) - if not resource_node.resource.type in clamav_config.artefact_types: + if not clamav_config.node_filter(resource_node): return comp = resource_node.component From 170c548c09e2bdc68f464487f1bfeb3b25509955 Mon Sep 17 00:00:00 2001 From: "Philipp Heil (zkdev)" Date: Mon, 18 Nov 2024 14:54:24 +0100 Subject: [PATCH 2/2] Rm artefact-type filter Use generic node-filter instead --- artefact_enumerator.py | 9 +--- bdba/__main__.py | 3 -- .../extensions/crds/scan-configuration.yaml | 16 ------- config.py | 42 ------------------- .../kind/cluster/values-extensions.yaml | 9 ---- 5 files changed, 2 insertions(+), 77 deletions(-) diff --git a/artefact_enumerator.py b/artefact_enumerator.py index cf41ea0f..6f051405 100644 --- a/artefact_enumerator.py +++ b/artefact_enumerator.py @@ -145,7 +145,6 @@ def create_compliance_snapshot( def _iter_ocm_artefact_nodes( components: tuple[config.Component], - artefact_types: tuple[str], node_filter: collections.abc.Callable[[cnudie.iter.Node], bool], delivery_client: delivery.client.DeliveryServiceClient, component_descriptor_lookup: cnudie.retrieve.ComponentDescriptorLookupById, @@ -178,23 +177,20 @@ def _iter_ocm_artefact_nodes( component=component, lookup=component_descriptor_lookup, node_filter=lambda node: ( - cnudie.iter.Filter.resources(node) and - node.artefact.type in artefact_types and - node_filter(node) + cnudie.iter.Filter.resources(node) + and node_filter(node) ), ) def _iter_ocm_artefacts( components: tuple[config.Component], - artefact_types: tuple[str], node_filter: collections.abc.Callable[[cnudie.iter.Node], bool], delivery_client: delivery.client.DeliveryServiceClient, component_descriptor_lookup: cnudie.retrieve.ComponentDescriptorLookupById, ) -> collections.abc.Generator[dso.model.ComponentArtefactId, None, None]: for artefact_node in _iter_ocm_artefact_nodes( components=components, - artefact_types=artefact_types, node_filter=node_filter, delivery_client=delivery_client, component_descriptor_lookup=component_descriptor_lookup, @@ -545,7 +541,6 @@ def enumerate_artefacts( ocm_artefacts = set(_iter_ocm_artefacts( components=scan_config.artefact_enumerator_config.components, - artefact_types=scan_config.artefact_enumerator_config.artefact_types, node_filter=scan_config.artefact_enumerator_config.node_filter, delivery_client=delivery_client, component_descriptor_lookup=component_descriptor_lookup, diff --git a/bdba/__main__.py b/bdba/__main__.py index 03e3c68b..d72ec929 100644 --- a/bdba/__main__.py +++ b/bdba/__main__.py @@ -127,9 +127,6 @@ def scan( artefact=backlog_item.artefact, ) - if not resource_node.resource.type in bdba_config.artefact_types: - return - if not bdba_config.node_filter(resource_node): return diff --git a/charts/extensions/crds/scan-configuration.yaml b/charts/extensions/crds/scan-configuration.yaml index da0f7864..21fec25f 100644 --- a/charts/extensions/crds/scan-configuration.yaml +++ b/charts/extensions/crds/scan-configuration.yaml @@ -37,10 +37,6 @@ spec: type: string lookup_new_backlog_item_interval: type: integer - artefact_types: - type: array - items: - type: string matching_configs: type: array items: @@ -106,10 +102,6 @@ spec: type: string compliance_snapshot_grace_period: type: integer - artefact_types: - type: array - items: - type: string matching_configs: type: array items: @@ -178,10 +170,6 @@ spec: type: string processing_mode: type: string - artefact_types: - type: array - items: - type: string matching_configs: type: array items: @@ -258,10 +246,6 @@ spec: type: integer enable_issue_assignees: type: boolean - artefact_types: - type: array - items: - type: string matching_configs: type: array items: diff --git a/config.py b/config.py index 9e686375..e39eceee 100644 --- a/config.py +++ b/config.py @@ -56,8 +56,6 @@ class ArtefactEnumeratorConfig: :param str delivery_service_url :param int compliance_snapshot_grace_period: time after which inactive compliance snapshots are deleted from the delivery-db - :param tuple[str] artefact_types: - list of artefact types for which compliance snapshots should be created :param Callable[Node, bool] node_filter: filter of artefact nodes to explicitly in- or exclude artefacts compliance snapshot creation :param tuple[Component] components: @@ -67,7 +65,6 @@ class ArtefactEnumeratorConfig: ''' delivery_service_url: str compliance_snapshot_grace_period: int - artefact_types: tuple[str] node_filter: collections.abc.Callable[[cnudie.iter.Node], bool] components: tuple[Component] sprints_time_range: TimeRange @@ -112,8 +109,6 @@ class BDBAConfig: name of config element to use for creating a s3 client :param ProcessingMode processing_mode: defines the scanning behaviour in case there is already an existing scan - :param tuple[str] artefact_types: - list of artefact types which should be scanned, other artefact types are skipped :param Callable[Node, bool] node_filter: filter of artefact nodes to explicitly in- or exclude artefacts from the bdba scan :param CveRescoringRuleSet cve_rescoring_ruleset: @@ -136,7 +131,6 @@ class BDBAConfig: cvss_version: bdba.model.CVSSVersion aws_cfg_set_name: str processing_mode: bdba.model.ProcessingMode - artefact_types: tuple[str] node_filter: collections.abc.Callable[[cnudie.iter.Node], bool] cve_rescoring_ruleset: rescore.model.CveRescoringRuleSet | None auto_assess_max_severity: dso.cvss.CVESeverity @@ -192,8 +186,6 @@ class IssueReplicatorConfig: labels matching one of these regexes won't be removed upon an issue update :param int number_included_closed_issues: number of closed issues to consider when evaluating creating vs re-opening an issue - :param tuple[str] artefact_types: - list of artefact types for which issues should be created, other artefact types are skipped :param Callable[Node, bool] node_filter: filter of artefact nodes to explicitly in- or exclude artefacts from the issue replication :param CveRescoringRuleSet cve_rescoring_ruleset: @@ -212,7 +204,6 @@ class IssueReplicatorConfig: github_issue_template_cfgs: tuple[image_scan.GithubIssueTemplateCfg] github_issue_labels_to_preserve: set[str] number_included_closed_issues: int - artefact_types: tuple[str] node_filter: collections.abc.Callable[[cnudie.iter.Node], bool] cve_rescoring_ruleset: rescore.model.CveRescoringRuleSet | None finding_type_issue_replication_cfgs: tuple[FindingTypeIssueReplicationCfgBase] @@ -311,16 +302,6 @@ def deserialise_artefact_enumerator_config( default_value=60 * 60 * 24, # 24h ) - artefact_types = tuple(deserialise_config_property( - config=artefact_enumerator_config, - property_key='artefact_types', - default_config=default_config, - default_value=( - ocm.ArtefactType.OCI_IMAGE, - 'application/tar+vm-image-rootfs', - ), - )) - matching_configs_raw = deserialise_config_property( config=artefact_enumerator_config, property_key='matching_configs', @@ -365,7 +346,6 @@ def deserialise_artefact_enumerator_config( return ArtefactEnumeratorConfig( delivery_service_url=delivery_service_url, compliance_snapshot_grace_period=compliance_snapshot_grace_period, - artefact_types=artefact_types, node_filter=node_filter, components=components, sprints_time_range=sprints_time_range, @@ -492,16 +472,6 @@ def deserialise_bdba_config( ) processing_mode = bdba.model.ProcessingMode(processing_mode_raw) - artefact_types = tuple(deserialise_config_property( - config=bdba_config, - property_key='artefact_types', - default_config=default_config, - default_value=( - ocm.ArtefactType.OCI_IMAGE, - 'application/tar+vm-image-rootfs', - ), - )) - matching_configs_raw = deserialise_config_property( config=bdba_config, property_key='matching_configs', @@ -588,7 +558,6 @@ def deserialise_bdba_config( cvss_version=cvss_version, aws_cfg_set_name=aws_cfg_set_name, processing_mode=processing_mode, - artefact_types=artefact_types, node_filter=node_filter, cve_rescoring_ruleset=cve_rescoring_ruleset, auto_assess_max_severity=auto_assess_max_severity, @@ -788,16 +757,6 @@ def deserialise_issue_replicator_config( default_value=0, ) - artefact_types = tuple(deserialise_config_property( - config=issue_replicator_config, - property_key='artefact_types', - default_config=default_config, - default_value=( - ocm.ArtefactType.OCI_IMAGE, - 'application/tar+vm-image-rootfs', - ), - )) - matching_configs_raw = deserialise_config_property( config=issue_replicator_config, property_key='matching_configs', @@ -869,7 +828,6 @@ def deserialise_issue_replicator_config( github_issue_template_cfgs=github_issue_template_cfgs, github_issue_labels_to_preserve=github_issue_labels_to_preserve, number_included_closed_issues=number_included_closed_issues, - artefact_types=artefact_types, node_filter=node_filter, cve_rescoring_ruleset=cve_rescoring_ruleset, finding_type_issue_replication_cfgs=finding_type_issue_replication_cfgs, diff --git a/local-setup/kind/cluster/values-extensions.yaml b/local-setup/kind/cluster/values-extensions.yaml index 521c5d30..71dca753 100644 --- a/local-setup/kind/cluster/values-extensions.yaml +++ b/local-setup/kind/cluster/values-extensions.yaml @@ -36,9 +36,6 @@ configuration: spec: {} # artefactEnumerator: # delivery_service_url: http://delivery-service.delivery.svc.cluster.local:8080 - # artefact_types: - # - ociImage - # - application/tar+vm-image-rootfs # components: # - component_name: ocm.software/ocm-gear # - component_name: ocm.software/ocmcli @@ -51,9 +48,6 @@ configuration: # rescan_interval: 3600 # 1h # cfg_name: local # group_id: ... - # artefact_types: - # - ociImage - # - application/tar+vm-image-rootfs # auto_assess_max_severity: MEDIUM # cve_rescoring_rules: # - category_value: network_exposure:public @@ -260,9 +254,6 @@ configuration: # - OSL # - RPL-1.5 # - sleepycat - # artefact_types: - # - ociImage - # - application/tar+vm-image-rootfs # finding_type_issue_replication_configs: # - finding_type: finding/vulnerability # enable_issue_assignees: False