From 4a5548ae3c6badecfac2c434b6741f0a02212296 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Hinse Date: Fri, 6 Dec 2024 15:21:39 -0500 Subject: [PATCH] Change the option from 'Ingest metadata only' to 'Ingest full event data' and make it off by default --- packages/flare/bin/constants.py | 2 +- packages/flare/bin/cron_job_ingest_events.py | 14 +++++----- packages/flare/bin/flare.py | 4 +-- packages/flare/tests/bin/conftest.py | 2 +- .../flare/tests/bin/test_flare_wrapper.py | 6 ++--- .../flare/tests/bin/test_ingest_events.py | 11 ++++++-- .../ConfigurationUserPreferencesStep.tsx | 26 +++++++++---------- .../react-components/src/models/constants.ts | 2 +- .../src/utils/setupConfiguration.ts | 14 +++++----- 9 files changed, 45 insertions(+), 36 deletions(-) diff --git a/packages/flare/bin/constants.py b/packages/flare/bin/constants.py index 53d8e3b..d5062ba 100644 --- a/packages/flare/bin/constants.py +++ b/packages/flare/bin/constants.py @@ -13,7 +13,7 @@ class PasswordKeys(Enum): API_KEY = "api_key" TENANT_ID = "tenant_id" - INGEST_METADATA_ONLY = "ingest_metadata_only" + INGEST_FULL_EVENT_DATA = "ingest_full_event_data" SEVERITIES_FILTER = "severities_filter" SOURCE_TYPES_FILTER = "source_types_filter" diff --git a/packages/flare/bin/cron_job_ingest_events.py b/packages/flare/bin/cron_job_ingest_events.py index 0fc9bd3..cb1a80c 100644 --- a/packages/flare/bin/cron_job_ingest_events.py +++ b/packages/flare/bin/cron_job_ingest_events.py @@ -95,7 +95,9 @@ def main( api_key = get_api_key(storage_passwords=storage_passwords) tenant_id = get_tenant_id(storage_passwords=storage_passwords) - ingest_metadata_only = get_ingest_metadata_only(storage_passwords=storage_passwords) + ingest_full_event_data = get_ingest_full_event_data( + storage_passwords=storage_passwords + ) severities_filter = get_severities_filter(storage_passwords=storage_passwords) source_types_filter = get_source_types_filter(storage_passwords=storage_passwords) @@ -107,7 +109,7 @@ def main( kvstore=kvstore, api_key=api_key, tenant_id=tenant_id, - ingest_metadata_only=ingest_metadata_only, + ingest_full_event_data=ingest_full_event_data, severities=severities_filter, source_types=source_types_filter, flare_api_cls=flare_api_cls, @@ -156,11 +158,11 @@ def get_tenant_id(storage_passwords: StoragePasswords) -> int: return tenant_id -def get_ingest_metadata_only(storage_passwords: StoragePasswords) -> bool: +def get_ingest_full_event_data(storage_passwords: StoragePasswords) -> bool: return ( get_storage_password_value( storage_passwords=storage_passwords, - password_key=PasswordKeys.INGEST_METADATA_ONLY.value, + password_key=PasswordKeys.INGEST_FULL_EVENT_DATA.value, ) == "true" ) @@ -312,7 +314,7 @@ def fetch_feed( kvstore: KVStoreCollections, api_key: str, tenant_id: int, - ingest_metadata_only: bool, + ingest_full_event_data: bool, severities: list[str], source_types: list[str], flare_api_cls: FlareAPI = FlareAPI, @@ -326,7 +328,7 @@ def fetch_feed( for event_next in flare_api.fetch_feed_events( next=next, start_date=start_date, - ingest_metadata_only=ingest_metadata_only, + ingest_full_event_data=ingest_full_event_data, severities=severities, source_types=source_types, ): diff --git a/packages/flare/bin/flare.py b/packages/flare/bin/flare.py index b6ffc55..f0e486d 100644 --- a/packages/flare/bin/flare.py +++ b/packages/flare/bin/flare.py @@ -49,7 +49,7 @@ def fetch_feed_events( *, next: Optional[str] = None, start_date: Optional[date] = None, - ingest_metadata_only: bool, + ingest_full_event_data: bool, severities: list[str], source_types: list[str], ) -> Iterator[tuple[dict, str]]: @@ -63,7 +63,7 @@ def fetch_feed_events( self.logger.debug(event_feed) next_token = event_feed["next"] for event in event_feed["items"]: - if not ingest_metadata_only: + if ingest_full_event_data: event = self._fetch_full_event_from_uid( uid=event["metadata"]["uid"] ) diff --git a/packages/flare/tests/bin/conftest.py b/packages/flare/tests/bin/conftest.py index 80ac3e4..d820c07 100644 --- a/packages/flare/tests/bin/conftest.py +++ b/packages/flare/tests/bin/conftest.py @@ -104,7 +104,7 @@ def fetch_feed_events( self, next: Optional[str], start_date: Optional[datetime], - ingest_metadata_only: bool, + ingest_full_event_data: bool, severities: list[str], source_types: list[str], ) -> List[tuple[dict, str]]: diff --git a/packages/flare/tests/bin/test_flare_wrapper.py b/packages/flare/tests/bin/test_flare_wrapper.py index a40540b..2fca185 100644 --- a/packages/flare/tests/bin/test_flare_wrapper.py +++ b/packages/flare/tests/bin/test_flare_wrapper.py @@ -43,7 +43,7 @@ def test_flare_full_data_without_metadata( for event, next_token in flare_api.fetch_feed_events( next=None, start_date=None, - ingest_metadata_only=True, + ingest_full_event_data=False, severities=[], source_types=[], ): @@ -106,7 +106,7 @@ def test_flare_full_data_with_metadata( for event, next_token in flare_api.fetch_feed_events( next=None, start_date=None, - ingest_metadata_only=False, + ingest_full_event_data=True, severities=[], source_types=[], ): @@ -150,7 +150,7 @@ def test_flare_full_data_with_metadata_and_exception( flare_api.fetch_feed_events( next=None, start_date=None, - ingest_metadata_only=False, + ingest_full_event_data=True, severities=[], source_types=[], ) diff --git a/packages/flare/tests/bin/test_ingest_events.py b/packages/flare/tests/bin/test_ingest_events.py index b4419ae..ff23fcc 100644 --- a/packages/flare/tests/bin/test_ingest_events.py +++ b/packages/flare/tests/bin/test_ingest_events.py @@ -19,6 +19,7 @@ from cron_job_ingest_events import fetch_feed from cron_job_ingest_events import get_api_key from cron_job_ingest_events import get_collection_value +from cron_job_ingest_events import get_ingest_full_event_data from cron_job_ingest_events import get_last_fetched from cron_job_ingest_events import get_last_ingested_tenant_id from cron_job_ingest_events import get_start_date @@ -188,6 +189,12 @@ def test_get_last_fetched_expect_none(kvstore: FakeKVStoreCollections) -> None: assert get_last_fetched(kvstore=kvstore) is None +def test_get_default_ingest_full_event_data_value( + storage_passwords: FakeStoragePasswords, +) -> None: + assert get_ingest_full_event_data(storage_passwords=storage_passwords) is False + + @pytest.mark.parametrize( "kvstore", [ @@ -212,7 +219,7 @@ def test_fetch_feed_expect_exception( kvstore=kvstore, api_key="some_key", tenant_id=11111, - ingest_metadata_only=False, + ingest_full_event_data=True, severities=[], source_types=[], ): @@ -238,7 +245,7 @@ def test_fetch_feed_expect_feed_response( kvstore=kvstore, api_key="some_key", tenant_id=11111, - ingest_metadata_only=False, + ingest_full_event_data=True, severities=[], source_types=[], flare_api_cls=FakeFlareAPI, diff --git a/packages/react-components/src/components/ConfigurationUserPreferencesStep.tsx b/packages/react-components/src/components/ConfigurationUserPreferencesStep.tsx index 3857eb3..40c7c13 100644 --- a/packages/react-components/src/components/ConfigurationUserPreferencesStep.tsx +++ b/packages/react-components/src/components/ConfigurationUserPreferencesStep.tsx @@ -17,7 +17,7 @@ import { fetchCurrentIndexName, fetchSeverityFilters, fetchSourceTypeFilters, - fetchIngestMetadataOnly, + fetchIngestFullEventData, fetchSeveritiesFilter, fetchTenantId, fetchUserTenants, @@ -51,13 +51,13 @@ const ConfigurationUserPreferencesStep: FC<{ const [selectedSourceTypes, setSelectedSourceTypes] = useState([]); const [indexName, setIndexName] = useState(''); const [indexNames, setIndexNames] = useState([]); - const [isIngestingMetadataOnly, setIsIngestingMetadataOnly] = useState(false); + const [isIngestingFullEventData, setIsIngestingFullEventData] = useState(false); const [isLoading, setIsLoading] = useState(false); const handleTenantIdChange = (e): void => setTenantId(parseInt(e.target.value, 10)); const handleIndexNameChange = (e): void => setIndexName(e.target.value); - const handleIsIngestingMetadataChange = (e): void => - setIsIngestingMetadataOnly(e.target.checked); + const handleIsIngestingFullEventDataChange = (e): void => + setIsIngestingFullEventData(e.target.checked); const handleSubmitUserPreferences = (): void => { setIsLoading(true); @@ -66,7 +66,7 @@ const ConfigurationUserPreferencesStep: FC<{ apiKey, Number(tenantId), indexName, - isIngestingMetadataOnly, + isIngestingFullEventData, getSeverityFilterValue(selectedSeverities, severities), getSourceTypesFilterValue(selectedSourceTypes, sourceTypeCategories) ) @@ -93,7 +93,7 @@ const ConfigurationUserPreferencesStep: FC<{ if (configurationStep === ConfigurationStep.UserPreferences) { Promise.all([ fetchTenantId(), - fetchIngestMetadataOnly(), + fetchIngestFullEventData(), fetchCurrentIndexName(), fetchUserTenants(apiKey), fetchAvailableIndexNames(), @@ -105,7 +105,7 @@ const ConfigurationUserPreferencesStep: FC<{ .then( ([ id, - ingestMetadataOnly, + ingestFullEventData, index, userTenants, availableIndexNames, @@ -115,7 +115,7 @@ const ConfigurationUserPreferencesStep: FC<{ sourceTypeFilter, ]) => { setTenantId(id); - setIsIngestingMetadataOnly(ingestMetadataOnly); + setIsIngestingFullEventData(ingestFullEventData); setIndexName(index); if (id === undefined && userTenants.length > 0) { setTenantId(userTenants[0].id); @@ -250,18 +250,18 @@ const ConfigurationUserPreferencesStep: FC<{
- +
- Select this option if you want to ingest only the metadata of the - events instead of the full data to it. + Select this option if you want to ingest the full data of the events + instead of the metadata of them.
diff --git a/packages/react-components/src/models/constants.ts b/packages/react-components/src/models/constants.ts index be1ad13..8ac7701 100644 --- a/packages/react-components/src/models/constants.ts +++ b/packages/react-components/src/models/constants.ts @@ -15,7 +15,7 @@ export const KV_COLLECTION_VALUE = 'value'; export enum PasswordKeys { API_KEY = 'api_key', TENANT_ID = 'tenant_id', - INGEST_METADATA_ONLY = 'ingest_metadata_only', + INGEST_FULL_EVENT_DATA = 'ingest_full_event_data', SEVERITIES_FILTER = 'severities_filter', SOURCE_TYPES_FILTER = 'source_types_filter', } diff --git a/packages/react-components/src/utils/setupConfiguration.ts b/packages/react-components/src/utils/setupConfiguration.ts index 764b654..694c9c3 100644 --- a/packages/react-components/src/utils/setupConfiguration.ts +++ b/packages/react-components/src/utils/setupConfiguration.ts @@ -131,7 +131,7 @@ async function saveConfiguration( apiKey: string, tenantId: number, indexName: string, - isIngestingMetadataOnly: boolean, + isIngestingFullEventData: boolean, severitiesFilter: string, sourceTypesFilter: string ): Promise { @@ -141,8 +141,8 @@ async function saveConfiguration( await savePassword(storagePasswords, PasswordKeys.TENANT_ID, `${tenantId}`); await savePassword( storagePasswords, - PasswordKeys.INGEST_METADATA_ONLY, - `${isIngestingMetadataOnly}` + PasswordKeys.INGEST_FULL_EVENT_DATA, + `${isIngestingFullEventData}` ); await savePassword(storagePasswords, PasswordKeys.SEVERITIES_FILTER, `${severitiesFilter}`); await savePassword(storagePasswords, PasswordKeys.SOURCE_TYPES_FILTER, `${sourceTypesFilter}`); @@ -243,9 +243,9 @@ async function fetchTenantId(): Promise { }); } -async function fetchIngestMetadataOnly(): Promise { - return fetchPassword(PasswordKeys.INGEST_METADATA_ONLY).then((isIngestingMetadataOnly) => { - return isIngestingMetadataOnly === 'true'; +async function fetchIngestFullEventData(): Promise { + return fetchPassword(PasswordKeys.INGEST_FULL_EVENT_DATA).then((isIngestingFullEventData) => { + return isIngestingFullEventData === 'true'; }); } @@ -434,7 +434,7 @@ export { fetchSeverityFilters, fetchCollectionItems, fetchCurrentIndexName, - fetchIngestMetadataOnly, + fetchIngestFullEventData, fetchSeveritiesFilter, fetchSourceTypeFilters, fetchSourceTypesFilter,