context

apache · Jan 7, 2025 · 3739ff4 · 3739ff4
1 parent ed91af8
commit 3739ff4
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 0 deletions.
diff --git a/...mmon/src/main/java/org/apache/hudi/common/table/read/HoodieBaseFileGroupRecordBuffer.java b/...mmon/src/main/java/org/apache/hudi/common/table/read/HoodieBaseFileGroupRecordBuffer.java
@@ -130,6 +130,7 @@ public HoodieBaseFileGroupRecordBuffer(HoodieReaderContext<T> readerContext,
     this.readStats = readStats;
     try {
       // Store merged records for all versions for this log file, set the in-memory footprint to maxInMemoryMapSize
+      // base path over multiple instants. Driver memory small. cdc query to test out should have oom without changing to spillable map.
       this.records = new ExternalSpillableMap<>(maxMemorySizeInBytes, spillableMapBasePath, new DefaultSizeEstimator<>(),
           new HoodieRecordSizeEstimator<>(readerSchema), diskMapType, isBitCaskDiskMapCompressionEnabled);
     } catch (IOException e) {

diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieFileGroupReader.java
@@ -76,6 +76,7 @@ public final class HoodieFileGroupReader<T> implements Closeable {
   // Length of bytes to read from the base file
   private final long length;
   // Core structure to store and process records.
+  // todo: refer on spillable map
   private final HoodieFileGroupRecordBuffer<T> recordBuffer;
   private ClosableIterator<T> baseFileIterator;
   private final Option<UnaryOperator<T>> outputConverter;

diff --git a/...atasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCFileGroupIterator.scala b/...atasource/hudi-spark-common/src/main/scala/org/apache/hudi/cdc/CDCFileGroupIterator.scala
@@ -190,12 +190,14 @@ class CDCFileGroupIterator(split: HoodieCDCFileGroupSplit,
    * 1) the cdc infer case is [[LOG_FILE]];
    * 2) the cdc infer case is [[AS_IS]] and [[cdcSupplementalLoggingMode]] is 'op_key'.
    */
+//    todo spillable map.
   private var beforeImageRecords: mutable.Map[String, GenericRecord] = mutable.Map.empty
 
   /**
    * Keep the after-image data. Only one case will use this:
    * the cdc infer case is [[AS_IS]] and [[cdcSupplementalLoggingMode]] is [[OP_KEY_ONLY]] or [[DATA_BEFORE]].
    */
+  //    todo spillable map.
   private var afterImageRecords: mutable.Map[String, InternalRow] = mutable.Map.empty
 
   private var internalRowToJsonStringConverter = new InternalRowToJsonStringConverter(originTableSchema)