Avoid check metadata for archived commits in incremental queries

apache · Jan 10, 2025 · 39ca7fa · 39ca7fa
1 parent e67d0aa
commit 39ca7fa
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 11 deletions.
diff --git a/...spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala b/...spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala
@@ -201,7 +201,7 @@ object DataSourceReadOptions {
 
   val INCREMENTAL_FALLBACK_TO_FULL_TABLE_SCAN: ConfigProperty[String] = ConfigProperty
     .key("hoodie.datasource.read.incr.fallback.fulltablescan.enable")
-    .defaultValue("false")
+    .defaultValue("true")
     .markAdvanced()
     .withDocumentation("When doing an incremental query whether we should fall back to full table scans if file does not exist.")
 

diff --git a/...e/hudi-spark-common/src/main/scala/org/apache/hudi/MergeOnReadIncrementalRelationV2.scala b/...e/hudi-spark-common/src/main/scala/org/apache/hudi/MergeOnReadIncrementalRelationV2.scala
@@ -205,15 +205,7 @@ trait HoodieIncrementalRelationV2Trait extends HoodieBaseRelation {
       .analyze()
 
   protected lazy val includedCommits: immutable.Seq[HoodieInstant] = queryContext.getInstants.asScala.toList
-
-  protected lazy val commitsMetadata = includedCommits.map(
-    i => {
-      if (queryContext.getArchivedInstants.contains(i)) {
-        getCommitMetadata(i, queryContext.getArchivedTimeline)
-      } else {
-        getCommitMetadata(i, queryContext.getActiveTimeline)
-      }
-    }).asJava
+  protected lazy val commitsMetadata = includedCommits.map(getCommitMetadata(_, super.timeline)).asJava
 
   protected lazy val affectedFilesInCommits: java.util.List[StoragePathInfo] = {
     listAffectedFilesForCommits(conf, metaClient.getBasePath, commitsMetadata)

diff --git a/...ource/hudi-spark/src/test/scala/org/apache/TestIncrementalQueryWithArchivedInstants.scala b/...ource/hudi-spark/src/test/scala/org/apache/TestIncrementalQueryWithArchivedInstants.scala
@@ -61,7 +61,7 @@ class TestIncrementalQueryWithArchivedInstants extends SparkClientFunctionalTest
       "hoodie.datasource.write.recordkey.field" -> "id",
       "hoodie.datasource.write.precombined.field" -> "ts",
       "hoodie.datasource.write.partitionpath.field" -> "name",
-      "hoodie.populate.meta.fields=true" -> "false",
+      "hoodie.populate.meta.fields" -> "true",
       "hoodie.compaction.payload.class" -> "org.apache.hudi.common.model.OverwriteWithLatestAvroPayload",
       "hoodie.datasource.write.payload.class" -> "org.apache.hudi.common.model.OverwriteWithLatestAvroPayload",
       RECORD_MERGE_MODE.key -> RecordMergeMode.COMMIT_TIME_ORDERING.name)