Change the option from 'Ingest metadata only' to 'Ingest full event d…

…ata' and make it off by default

packages/flare/bin/constants.py

@@ -13,7 +13,7 @@
 class PasswordKeys(Enum):
     API_KEY = "api_key"
     TENANT_ID = "tenant_id"
-    INGEST_METADATA_ONLY = "ingest_metadata_only"
+    INGEST_FULL_EVENT_DATA = "ingest_full_event_data"
     SEVERITIES_FILTER = "severities_filter"
     SOURCE_TYPES_FILTER = "source_types_filter"
 

packages/flare/bin/cron_job_ingest_events.py

@@ -95,7 +95,9 @@ def main(
 
     api_key = get_api_key(storage_passwords=storage_passwords)
     tenant_id = get_tenant_id(storage_passwords=storage_passwords)
-    ingest_metadata_only = get_ingest_metadata_only(storage_passwords=storage_passwords)
+    ingest_full_event_data = get_ingest_full_event_data(
+        storage_passwords=storage_passwords
+    )
     severities_filter = get_severities_filter(storage_passwords=storage_passwords)
     source_types_filter = get_source_types_filter(storage_passwords=storage_passwords)
 
@@ -107,7 +109,7 @@ def main(
         kvstore=kvstore,
         api_key=api_key,
         tenant_id=tenant_id,
-        ingest_metadata_only=ingest_metadata_only,
+        ingest_full_event_data=ingest_full_event_data,
         severities=severities_filter,
         source_types=source_types_filter,
         flare_api_cls=flare_api_cls,
@@ -156,11 +158,11 @@ def get_tenant_id(storage_passwords: StoragePasswords) -> int:
     return tenant_id
 
 
-def get_ingest_metadata_only(storage_passwords: StoragePasswords) -> bool:
+def get_ingest_full_event_data(storage_passwords: StoragePasswords) -> bool:
     return (
         get_storage_password_value(
             storage_passwords=storage_passwords,
-            password_key=PasswordKeys.INGEST_METADATA_ONLY.value,
+            password_key=PasswordKeys.INGEST_FULL_EVENT_DATA.value,
         )
         == "true"
     )
@@ -312,7 +314,7 @@ def fetch_feed(
     kvstore: KVStoreCollections,
     api_key: str,
     tenant_id: int,
-    ingest_metadata_only: bool,
+    ingest_full_event_data: bool,
     severities: list[str],
     source_types: list[str],
     flare_api_cls: FlareAPI = FlareAPI,
@@ -326,7 +328,7 @@ def fetch_feed(
         for event_next in flare_api.fetch_feed_events(
             next=next,
             start_date=start_date,
-            ingest_metadata_only=ingest_metadata_only,
+            ingest_full_event_data=ingest_full_event_data,
             severities=severities,
             source_types=source_types,
         ):

packages/flare/bin/flare.py

@@ -49,7 +49,7 @@ def fetch_feed_events(
         *,
         next: Optional[str] = None,
         start_date: Optional[date] = None,
-        ingest_metadata_only: bool,
+        ingest_full_event_data: bool,
         severities: list[str],
         source_types: list[str],
     ) -> Iterator[tuple[dict, str]]:
@@ -63,7 +63,7 @@ def fetch_feed_events(
             self.logger.debug(event_feed)
             next_token = event_feed["next"]
             for event in event_feed["items"]:
-                if not ingest_metadata_only:
+                if ingest_full_event_data:
                     event = self._fetch_full_event_from_uid(
                         uid=event["metadata"]["uid"]
                     )

packages/flare/tests/bin/conftest.py

@@ -104,7 +104,7 @@ def fetch_feed_events(
         self,
         next: Optional[str],
         start_date: Optional[datetime],
-        ingest_metadata_only: bool,
+        ingest_full_event_data: bool,
         severities: list[str],
         source_types: list[str],
     ) -> List[tuple[dict, str]]:

packages/flare/tests/bin/test_flare_wrapper.py

@@ -43,7 +43,7 @@ def test_flare_full_data_without_metadata(
     for event, next_token in flare_api.fetch_feed_events(
         next=None,
         start_date=None,
-        ingest_metadata_only=True,
+        ingest_full_event_data=False,
         severities=[],
         source_types=[],
     ):
@@ -106,7 +106,7 @@ def test_flare_full_data_with_metadata(
     for event, next_token in flare_api.fetch_feed_events(
         next=None,
         start_date=None,
-        ingest_metadata_only=False,
+        ingest_full_event_data=True,
         severities=[],
         source_types=[],
     ):
@@ -150,7 +150,7 @@ def test_flare_full_data_with_metadata_and_exception(
             flare_api.fetch_feed_events(
                 next=None,
                 start_date=None,
-                ingest_metadata_only=False,
+                ingest_full_event_data=True,
                 severities=[],
                 source_types=[],
             )

packages/flare/tests/bin/test_ingest_events.py

@@ -19,6 +19,7 @@
 from cron_job_ingest_events import fetch_feed
 from cron_job_ingest_events import get_api_key
 from cron_job_ingest_events import get_collection_value
+from cron_job_ingest_events import get_ingest_full_event_data
 from cron_job_ingest_events import get_last_fetched
 from cron_job_ingest_events import get_last_ingested_tenant_id
 from cron_job_ingest_events import get_start_date
@@ -188,6 +189,12 @@ def test_get_last_fetched_expect_none(kvstore: FakeKVStoreCollections) -> None:
     assert get_last_fetched(kvstore=kvstore) is None
 
 
+def test_get_default_ingest_full_event_data_value(
+    storage_passwords: FakeStoragePasswords,
+) -> None:
+    assert get_ingest_full_event_data(storage_passwords=storage_passwords) is False
+
+
 @pytest.mark.parametrize(
     "kvstore",
     [
@@ -212,7 +219,7 @@ def test_fetch_feed_expect_exception(
         kvstore=kvstore,
         api_key="some_key",
         tenant_id=11111,
-        ingest_metadata_only=False,
+        ingest_full_event_data=True,
         severities=[],
         source_types=[],
     ):
@@ -238,7 +245,7 @@ def test_fetch_feed_expect_feed_response(
         kvstore=kvstore,
         api_key="some_key",
         tenant_id=11111,
-        ingest_metadata_only=False,
+        ingest_full_event_data=True,
         severities=[],
         source_types=[],
         flare_api_cls=FakeFlareAPI,

packages/react-components/src/components/ConfigurationUserPreferencesStep.tsx

@@ -17,7 +17,7 @@ import {
     fetchCurrentIndexName,
     fetchSeverityFilters,
     fetchSourceTypeFilters,
-    fetchIngestMetadataOnly,
+    fetchIngestFullEventData,
     fetchSeveritiesFilter,
     fetchTenantId,
     fetchUserTenants,
@@ -51,13 +51,13 @@ const ConfigurationUserPreferencesStep: FC<{
     const [selectedSourceTypes, setSelectedSourceTypes] = useState<SourceType[]>([]);
     const [indexName, setIndexName] = useState('');
     const [indexNames, setIndexNames] = useState<string[]>([]);
-    const [isIngestingMetadataOnly, setIsIngestingMetadataOnly] = useState(false);
+    const [isIngestingFullEventData, setIsIngestingFullEventData] = useState(false);
     const [isLoading, setIsLoading] = useState(false);
 
     const handleTenantIdChange = (e): void => setTenantId(parseInt(e.target.value, 10));
     const handleIndexNameChange = (e): void => setIndexName(e.target.value);
     const handleIsIngestingMetadataChange = (e): void =>
-        setIsIngestingMetadataOnly(e.target.checked);
+        setIsIngestingFullEventData(e.target.checked);
 
     const handleSubmitUserPreferences = (): void => {
         setIsLoading(true);
@@ -66,7 +66,7 @@ const ConfigurationUserPreferencesStep: FC<{
             apiKey,
             Number(tenantId),
             indexName,
-            isIngestingMetadataOnly,
+            isIngestingFullEventData,
             getSeverityFilterValue(selectedSeverities, severities),
             getSourceTypesFilterValue(selectedSourceTypes, sourceTypeCategories)
         )
@@ -93,7 +93,7 @@ const ConfigurationUserPreferencesStep: FC<{
         if (configurationStep === ConfigurationStep.UserPreferences) {
             Promise.all([
                 fetchTenantId(),
-                fetchIngestMetadataOnly(),
+                fetchIngestFullEventData(),
                 fetchCurrentIndexName(),
                 fetchUserTenants(apiKey),
                 fetchAvailableIndexNames(),
@@ -105,7 +105,7 @@ const ConfigurationUserPreferencesStep: FC<{
                 .then(
                     ([
                         id,
-                        ingestMetadataOnly,
+                        ingestFullEventData,
                         index,
                         userTenants,
                         availableIndexNames,
@@ -115,7 +115,7 @@ const ConfigurationUserPreferencesStep: FC<{
                         sourceTypeFilter,
                     ]) => {
                         setTenantId(id);
-                        setIsIngestingMetadataOnly(ingestMetadataOnly);
+                        setIsIngestingFullEventData(ingestFullEventData);
                         setIndexName(index);
                         if (id === undefined && userTenants.length > 0) {
                             setTenantId(userTenants[0].id);
@@ -250,17 +250,17 @@ const ConfigurationUserPreferencesStep: FC<{
                 </div>
                 <div className="form-item">
                     <div className="label-tooltip">
-                        <Label>Basic event ingestion</Label>
+                        <Label>Full event data ingestion</Label>
                         <Tooltip>
                             <div>
-                                Select this option if you want to ingest only the metadata of the
-                                events instead of the full data to it.
+                                Select this option if you want to ingest the full data of the events
+                                instead of the metadata of them.
                             </div>
                         </Tooltip>
                     </div>
                     <span className="switch-container">
                         <Switch
-                            value={isIngestingMetadataOnly}
+                            value={isIngestingFullEventData}
                             onChange={handleIsIngestingMetadataChange}
                         />
                     </span>

packages/react-components/src/models/constants.ts

@@ -15,7 +15,7 @@ export const KV_COLLECTION_VALUE = 'value';
 export enum PasswordKeys {
     API_KEY = 'api_key',
     TENANT_ID = 'tenant_id',
-    INGEST_METADATA_ONLY = 'ingest_metadata_only',
+    INGEST_FULL_EVENT_DATA = 'ingest_full_event_data',
     SEVERITIES_FILTER = 'severities_filter',
     SOURCE_TYPES_FILTER = 'source_types_filter',
 }

packages/react-components/src/utils/setupConfiguration.ts

@@ -131,7 +131,7 @@ async function saveConfiguration(
     apiKey: string,
     tenantId: number,
     indexName: string,
-    isIngestingMetadataOnly: boolean,
+    isIngestingFullEventData: boolean,
     severitiesFilter: string,
     sourceTypesFilter: string
 ): Promise<void> {
@@ -141,8 +141,8 @@ async function saveConfiguration(
     await savePassword(storagePasswords, PasswordKeys.TENANT_ID, `${tenantId}`);
     await savePassword(
         storagePasswords,
-        PasswordKeys.INGEST_METADATA_ONLY,
-        `${isIngestingMetadataOnly}`
+        PasswordKeys.INGEST_FULL_EVENT_DATA,
+        `${isIngestingFullEventData}`
     );
     await savePassword(storagePasswords, PasswordKeys.SEVERITIES_FILTER, `${severitiesFilter}`);
     await savePassword(storagePasswords, PasswordKeys.SOURCE_TYPES_FILTER, `${sourceTypesFilter}`);
@@ -243,9 +243,9 @@ async function fetchTenantId(): Promise<number | undefined> {
     });
 }
 
-async function fetchIngestMetadataOnly(): Promise<boolean> {
-    return fetchPassword(PasswordKeys.INGEST_METADATA_ONLY).then((isIngestingMetadataOnly) => {
-        return isIngestingMetadataOnly === 'true';
+async function fetchIngestFullEventData(): Promise<boolean> {
+    return fetchPassword(PasswordKeys.INGEST_FULL_EVENT_DATA).then((isIngestingFullEventData) => {
+        return isIngestingFullEventData === 'true';
     });
 }
 
@@ -434,7 +434,7 @@ export {
     fetchSeverityFilters,
     fetchCollectionItems,
     fetchCurrentIndexName,
-    fetchIngestMetadataOnly,
+    fetchIngestFullEventData,
     fetchSeveritiesFilter,
     fetchSourceTypeFilters,
     fetchSourceTypesFilter,