diff --git a/Config/forced_loose.yaml b/Config/forced_loose.yaml index bea59d6..c90dba2 100644 --- a/Config/forced_loose.yaml +++ b/Config/forced_loose.yaml @@ -1,5 +1,5 @@ --- -Customs: +bigmap: - 5938188786f77474f723e87f # Case 0031 - 5c12301c86f77419522ba7e4 # Flash drive with fake info - 593965cf86f774087a77e1b6 # Case 0048 @@ -13,9 +13,10 @@ Customs: - 5939a00786f7742fe8132936 # Golden Zibbo lighter - 5939e5a786f77461f11c0098 # Secure Folder 0013 - 64e74a3d4d49d23b2c39d319 # item_quest_clock_07 (Out of Time) +- 6614230055afee107f05e998 # The Unheard's phone #- 64bd1abff3a668f08805ce4f # Secure Flash drive V4 REMOVED BY BSG -Woods: +woods: - 5938878586f7741b797c562f # Case 0052 - 5d3ec50586f774183a607442 # Jaeger's message Underneath the wooden lookout post. - 5af04e0a86f7743a532b79e2 # Single-axis Fiber Optic Gyroscope: item_barter_electr_gyroscope @@ -24,7 +25,7 @@ Woods: #- 64bde2248f3a947a990aa4a5 # Sliderkey Secure Flash drive #1 REMOVED BY BSG #- 64bde265807321a9b905f076 # Sliderkey Secure Flash drive #2 REMOVED BY BSG -Shoreline: +shoreline: - 5a294d7c86f7740651337cf9 # Drone 1 SAS disk - 5a294d8486f774068638cd93 # Drone 2 SAS disk: ambiguous with itemTpl 5a294f1686f774340c7b7e4a - 5efdafc1e70b5e33f86de058 # Sanitar's Surgery kit marked with a blue symbol @@ -41,8 +42,12 @@ Shoreline: - 5b43237186f7742f3a4ab252 # Chemical container: item_quest_chem_container - 5a29284f86f77463ef3db363 # Toughbook reinforced laptop - 64e74a534d49d23b2c39d31b # item_quest_clock_10 (Out of Time) +- 6614238e0d240a5f5d0f679d # Skier and Peacekeeper correspondence +- 661421c7c1f2f548c50ee649 # The Unheard's laptop +- 6614217b6d9d5abcad0ff098 # The Unheard's phone +- 661423200d240a5f5d0f679b # The Unheard's laptop -Interchange: +interchange: - 5ae9a18586f7746e381e16a3 # OLI cargo manifests - 5ae9a0dd86f7742e5f454a05 # Goshan cargo manifests - 5ae9a1b886f77404c8537c62 # Idea cargo manifests @@ -53,11 +58,15 @@ Interchange: - 5b4c81bd86f77418a75ae159 # Chemical container item_quest_chem_container3 - 64e74a5ac2b4f829615ec336 # item_quest_clock_11 (Out of Time) -Factory: +factory4_day: - 591093bb86f7747caa7bb2ee # On the neck of the dead scav in the bunker (Postman Pat Part 2) - 593a87af86f774122f54a951 # Syringe with a chemical -Lighthouse: +factory4_night: +- 591093bb86f7747caa7bb2ee # On the neck of the dead scav in the bunker (Postman Pat Part 2) +- 593a87af86f774122f54a951 # Syringe with a chemical + +lighthouse: - 61904c9df62c89219a56e034 # The message is tucked under the bottom of the door to the cabin. - 619268ad78f4fa33f173dbe5 # Water pump operation data On the desk between other documents in the upper office. - 619268de2be33f2604340159 # Pumping Station Operation Data In the upper floor office on the shelf. @@ -69,9 +78,10 @@ Lighthouse: - 6399f54b0a36db13c823ad21 # Radio transmitter body (Key to the Tower) - 64e74a64aac4cd0a7264ecdf # item_quest_clock_12 (Out of Time) - 578f87a3245977356274f2cb # +- 661666458c2aa9cb1602503b # Hard drive # - 64b91627dd13d43b9d01d6d1 # Toughbook reinforced laptop (Event quest) REMOVED BY BSG -ReserveBase: +rezervbase: - 60915994c49cf53e4772cc38 # Military documents 1 on the table inside bunker control room (Documents) - 60a3b6359c427533db36cf84 # Military documents 2 On the bottom shelf of the cupboard near the corner. - 60a3b65c27adf161da7b6e14 # Military documents 3 Inside the cupboard next to the 4x4 Weapon Box. @@ -82,15 +92,16 @@ ReserveBase: - 6398a072e301557ae24cec92 # Original Lightkeeper Intelligence (Snatch) - 64e74a4baac4cd0a7264ecdd # item_quest_clock_09 (Out of Time) -Laboratory: +laboratory: - 5eff135be0d3331e9d282b7b # Flash drive marked with blue tape (TerraGroup employee) - 6398a4cfb5992f573c6562b3 # Secured tape +- 64e74a44c2b4f829615ec334 # Picture 8 #- 64e74a44c2b4f829615ec334 # item_quest_clock_08 (Out of Time) #1 REMOVED BY BSG #- 64bdcfed8f3a947a990aa49a # Hermetic container for storing various chemicals #1 REMOVED BY BSG #- 64bdd008b0bf3baa6702f35f # Hermetic container for storing various chemicals #2 REMOVED BY BSG #- 64bdd014f3a668f08805ce64 # Hermetic container for storing various chemicals #3 REMOVED BY BSG -Streets of Tarkov: +tarkovstreets: - 63a943cead5cc12f22161ff7 # Accountant's notes (Audit) - 638cbc68a63f1b49be6a3010 # Registered letter (Youve Got Mail) - 638df4cc7b560b03794a18d2 # AG guitar pick (Audiophile) @@ -114,6 +125,10 @@ Streets of Tarkov: - 64f5b4f71a5f313cb144c06c # Secret component (Beyond the Red Meat - Part 2) - 657acb2ac900be5902191ac9 # Cadastral registry records -Sandbox: +sandbox: +- 6582bd252b50c61c565828e2 # Bottle of Le Jean wine +- 6575a6ca8778e96ded05a802 # TerraGroup scientist's hard drive + +sandbox_high: - 6582bd252b50c61c565828e2 # Bottle of Le Jean wine - 6575a6ca8778e96ded05a802 # TerraGroup scientist's hard drive \ No newline at end of file diff --git a/Config/forced_static.yaml b/Config/forced_static.yaml index 7244659..23d4d96 100644 --- a/Config/forced_static.yaml +++ b/Config/forced_static.yaml @@ -5,17 +5,17 @@ static_weapon_ids: - 5cdeb229d7f00c000e7ce174 forced_items: - Customs: + bigmap: # unknown key - containerId: custom_multiScene_00058 itemTpl: 593962ca86f774068014d9af - Streets of Tarkov: + tarkovstreets: # Backup hideout key - - containerId: container_City_SE_02_DesignStuff_00025 + - containerId: container_City_SE_02_Primorskiy_51_indoor_00001 itemTpl: 6398fd8ad3de3849057f5128 - Sandbox: + sandbox: - containerId: container_Test_for_export_00002 # Lab technician body itemTpl: 658199aa38c79576a2569e13 # TerraGroup science office key - containerId: container_custom_DesignStuff_00029 # dead body of scav near exit diff --git a/Config/map_directory_mapping.yaml b/Config/map_directory_mapping.yaml index 72154b2..6ad716f 100644 --- a/Config/map_directory_mapping.yaml +++ b/Config/map_directory_mapping.yaml @@ -29,4 +29,7 @@ Streets of Tarkov: - tarkovstreets Sandbox: name: - - Sandbox \ No newline at end of file + - Sandbox +SandboxHigh: + name: + - Sandbox_high \ No newline at end of file diff --git a/Model/Processing/DumpProcessData.cs b/Model/Processing/DumpProcessData.cs index b091d18..b627f05 100644 --- a/Model/Processing/DumpProcessData.cs +++ b/Model/Processing/DumpProcessData.cs @@ -5,6 +5,6 @@ namespace LootDumpProcessor.Model.Processing; public class DumpProcessData { public Dictionary LooseLootCounts { get; set; } = new(); - public List ContainerCounts { get; set; } = new(); + public Dictionary> ContainerCounts { get; set; } = new(); public Dictionary MapCounts { get; set; } = new(); } \ No newline at end of file diff --git a/Model/Upd.cs b/Model/Upd.cs index 9c02828..367e364 100644 --- a/Model/Upd.cs +++ b/Model/Upd.cs @@ -1,5 +1,4 @@ using System.Text.Json.Serialization; -using LootDumpProcessor.Process.Processor; using LootDumpProcessor.Utils; using Newtonsoft.Json; @@ -9,7 +8,7 @@ public class Upd : ICloneable { [JsonProperty("StackObjectsCount", NullValueHandling = NullValueHandling.Ignore)] [JsonPropertyName("StackObjectsCount")] - public int? StackObjectsCount { get; set; } + public object? StackObjectsCount { get; set; } [JsonProperty("FireMode", NullValueHandling = NullValueHandling.Ignore)] [JsonPropertyName("FireMode")] diff --git a/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs b/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs index 1dc2265..329f4de 100644 --- a/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs +++ b/Process/Processor/DumpProcessor/MultithreadSteppedDumpProcessor.cs @@ -52,22 +52,23 @@ public Dictionary ProcessDumps(List dumps) Runners.Add( Task.Factory.StartNew(() => { - if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Debug)) LoggerFactory.GetInstance().Log($"Processing static data for file {dumped.BasicInfo.FileName}", LogLevel.Debug); - var data = _jsonSerializer.Deserialize(File.ReadAllText(dumped.BasicInfo.FileName)); - var mapName = data.Data.Name; + var dataDump = _jsonSerializer.Deserialize(File.ReadAllText(dumped.BasicInfo.FileName)); + //var mapName = dataDump.Data.Name; + var mapId = dataDump.Data.Id.ToLower(); // the if statement below takes care of processing "forced" or real static data for each map, only need // to do this once per map, we dont care about doing it again lock (staticContainersLock) { - if (!staticContainers.ContainsKey(mapName)) + if (!staticContainers.ContainsKey(mapId)) { if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) - LoggerFactory.GetInstance().Log($"Doing first time process for map {mapName} of real static data", LogLevel.Info); - var mapStaticContainers = StaticLootProcessor.CreateStaticWeaponsAndStaticForcedContainers(data); + LoggerFactory.GetInstance().Log($"Doing first time process for map {mapId} of real static data", LogLevel.Info); + var mapStaticContainers = StaticLootProcessor.CreateStaticWeaponsAndStaticForcedContainers(dataDump); // .Item1 = map name + // .Item2 = force/weapon static arrays staticContainers[mapStaticContainers.Item1] = mapStaticContainers.Item2; } } @@ -77,36 +78,40 @@ public Dictionary ProcessDumps(List dumps) lock (mapStaticContainersAggregatedLock) { // Init dict if map key doesnt exist - if (!mapStaticContainersAggregated.TryGetValue(mapName, out mapAggregatedDataDict)) + if (!mapStaticContainersAggregated.TryGetValue(mapId, out mapAggregatedDataDict)) { mapAggregatedDataDict = new Dictionary(); - mapStaticContainersAggregated.Add(mapName, mapAggregatedDataDict); + mapStaticContainersAggregated.Add(mapId, mapAggregatedDataDict); } } // Only process the dump file if the date is higher (after) the configuration date - if (DumpWasMadeAfterConfigThresholdDate(dumped)) + if (!DumpWasMadeAfterConfigThresholdDate(dumped)) { - // Keep track of how many dumps we have for each map - lock (mapDumpCounterLock) - { - IncrementMapCounterDictionaryValue(mapDumpCounter, mapName); - } + return; + } - var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList.TryGetValue(data.Data.Id.ToLower(), out string[]? ignoreListForMap); - foreach (var dynamicStaticContainer in StaticLootProcessor.CreateDynamicStaticContainers(data)) + // Keep track of how many dumps we have for each map + lock (mapDumpCounterLock) + { + IncrementMapCounterDictionaryValue(mapDumpCounter, mapId); + } + + var containerIgnoreListExists = LootDumpProcessorContext.GetConfig().ContainerIgnoreList.TryGetValue(mapId, out string[]? ignoreListForMap); + foreach (var dynamicStaticContainer in StaticLootProcessor.CreateDynamicStaticContainers(dataDump)) + { + lock (mapStaticContainersAggregatedLock) { - lock (mapStaticContainersAggregatedLock) + if (containerIgnoreListExists && ignoreListForMap.Contains(dynamicStaticContainer.Id)) { - // Skip adding containers to aggredated data if container id is in ignore list - if (containerIgnoreListExists && ignoreListForMap.Contains(dynamicStaticContainer.Id)) - { - continue; - } + // Skip adding containers to aggregated data if container id is in ignore list + continue; + } - // Increment count by 1 - if (!mapAggregatedDataDict.TryAdd(dynamicStaticContainer, 1)) - mapAggregatedDataDict[dynamicStaticContainer] += 1; + // Increment times container seen in dump by 1 + if (!mapAggregatedDataDict.TryAdd(dynamicStaticContainer, 1)) + { + mapAggregatedDataDict[dynamicStaticContainer] += 1; } } } @@ -126,10 +131,10 @@ public Dictionary ProcessDumps(List dumps) td => new StaticDataPoint { Template = td.Key, - Probability = GetStaticContainerProbability(kv.Key, td, mapDumpCounter) + Probability = GetStaticContainerProbability(kv.Key, td, mapDumpCounter) // kv.Key = map name } ).ToList() - ).ToList().ForEach(kv => staticContainers[kv.Key].StaticContainers = kv.Value); + ).ToList().ForEach(kv => staticContainers[kv.Key].StaticContainers = kv.Value); // Hydrate staticContainers.StaticContainers // Static containers output.Add(OutputFileType.StaticContainer, staticContainers); @@ -146,7 +151,7 @@ public Dictionary ProcessDumps(List dumps) // Static loot distribution output.Add( OutputFileType.StaticLoot, - StaticLootProcessor.CreateStaticLootDistribution(dumpProcessData.ContainerCounts) + StaticLootProcessor.CreateStaticLootDistribution(dumpProcessData.ContainerCounts, staticContainers) ); if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) @@ -199,23 +204,23 @@ private static DumpProcessData GetDumpProcessData(List dumps) .ToList() .ForEach(tuple => { - var mapi = tuple.Key; - var g = tuple.ToList(); + var mapName = tuple.Key; + var partialFileMetaData = tuple.ToList(); if (LoggerFactory.GetInstance().CanBeLogged(LogLevel.Info)) LoggerFactory.GetInstance().Log( - $"Processing map {mapi}, total dump data to process: {g.Count}", + $"Processing map {mapName}, total dump data to process: {partialFileMetaData.Count}", LogLevel.Info ); - dumpProcessData.MapCounts[mapi] = g.Count; + dumpProcessData.MapCounts[mapName] = partialFileMetaData.Count; var lockObjectContainerCounts = new object(); var lockObjectCounts = new object(); - var counts = new LooseLootCounts(); + var looseLootCounts = new LooseLootCounts(); var lockObjectDictionaryCounts = new object(); var dictionaryCounts = new FlatKeyableDictionary(); - counts.Counts = dictionaryCounts.GetKey(); + looseLootCounts.Counts = dictionaryCounts.GetKey(); /* var dictionaryItemCounts = new FlatKeyableDictionary>(); @@ -226,22 +231,21 @@ private static DumpProcessData GetDumpProcessData(List dumps) var dictionaryItemProperties = new FlatKeyableDictionary>(); var actualDictionaryItemProperties = new FlatKeyableDictionary(); - counts.ItemProperties = actualDictionaryItemProperties.GetKey(); + looseLootCounts.ItemProperties = actualDictionaryItemProperties.GetKey(); - dumpProcessData.LooseLootCounts.Add(mapi, counts.GetKey()); + dumpProcessData.LooseLootCounts.Add(mapName, looseLootCounts.GetKey()); // add the items to the queue - foreach (var gi in g) + foreach (var partialData in partialFileMetaData) { - _partialDataToProcess.Add(gi); + _partialDataToProcess.Add(partialData); } // Call GC before running threads - g = null; + partialFileMetaData = null; tuple = null; GCHandler.Collect(); // The data storage factory has a lock, we dont want the locks to occur when multithreading - for (int i = 0; i < LootDumpProcessorContext.GetConfig().Threads; i++) { Runners.Add( @@ -254,26 +258,35 @@ private static DumpProcessData GetDumpProcessData(List dumps) try { var dumpData = _dataStorage.GetItem(partialData.ParsedDumpKey); + + // Static containers lock (lockObjectContainerCounts) { - dumpProcessData.ContainerCounts.AddRange(dumpData.Containers); + if (!dumpProcessData.ContainerCounts.ContainsKey(mapName)) + { + dumpProcessData.ContainerCounts.Add(mapName, dumpData.Containers); + } + else + { + dumpProcessData.ContainerCounts[mapName].AddRange(dumpData.Containers); + } } - // loose loot into ids on files + // Loose loot into ids on files var loadedDictionary = _dataStorage .GetItem>>( dumpData.LooseLoot.ItemProperties ); - foreach (var (k, v) in loadedDictionary) + foreach (var (uniqueKey, containerTemplate) in loadedDictionary) { - var count = dumpData.LooseLoot.Counts[k]; + var count = dumpData.LooseLoot.Counts[uniqueKey]; lock (lockObjectDictionaryCounts) { - if (dictionaryCounts.ContainsKey(k)) - dictionaryCounts[k] += count; + if (dictionaryCounts.ContainsKey(uniqueKey)) + dictionaryCounts[uniqueKey] += count; else - dictionaryCounts[k] = count; + dictionaryCounts[uniqueKey] = count; } /* @@ -288,20 +301,20 @@ private static DumpProcessData GetDumpProcessData(List dumps) lock (lockObjectDictionaryItemProperties) { - if (!dictionaryItemProperties.TryGetValue(k, out var values)) + if (!dictionaryItemProperties.TryGetValue(uniqueKey, out var values)) { values = new FlatKeyableList