[GLUTEN-1632][CH]Daily Update Clickhouse Version (20241228) (#8368)

* [GLUTEN-1632][CH]Daily Update Clickhouse Version (20241228) * Fix UT due to ClickHouse/ClickHouse#73422 --------- Co-authored-by: kyligence-git <[email protected]> Co-authored-by: Chang Chen <[email protected]>
apache · Dec 28, 2024 · 49f6657 · 49f6657
1 parent 74cc9d7
commit 49f6657
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 7 deletions.
diff --git a/...ds-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala b/...ds-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala
@@ -91,15 +91,14 @@ class CHListenerApi extends ListenerApi with Logging {
       "local_engine.settings.log_processors_profiles" -> "true")
     conf.setCHSettings("spark_version", SPARK_VERSION)
     // add memory limit for external sort
-    val externalSortKey = CHConf.runtimeSettings("max_bytes_before_external_sort")
-    if (conf.getLong(externalSortKey, -1) < 0) {
+    if (conf.getLong(RuntimeSettings.MAX_BYTES_BEFORE_EXTERNAL_SORT.key, -1) < 0) {
       if (conf.getBoolean("spark.memory.offHeap.enabled", defaultValue = false)) {
         val memSize = JavaUtils.byteStringAsBytes(conf.get("spark.memory.offHeap.size"))
         if (memSize > 0L) {
           val cores = conf.getInt("spark.executor.cores", 1).toLong
           val sortMemLimit = ((memSize / cores) * 0.8).toLong
           logDebug(s"max memory for sorting: $sortMemLimit")
-          conf.set(externalSortKey, sortMemLimit.toString)
+          conf.set(RuntimeSettings.MAX_BYTES_BEFORE_EXTERNAL_SORT.key, sortMemLimit.toString)
         }
       }
     }

diff --git a/...-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/RuntimeSettings.scala b/...-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/RuntimeSettings.scala
@@ -30,6 +30,12 @@ object RuntimeSettings {
       .doc("https://clickhouse.com/docs/en/operations/settings/settings#min_insert_block_size_rows")
       .longConf
       .createWithDefault(1048449)
+
+  val MAX_BYTES_BEFORE_EXTERNAL_SORT =
+    buildConf(runtimeSettings("max_bytes_before_external_sort"))
+      .doc("https://clickhouse.com/docs/en/operations/settings/query-complexity#settings-max_bytes_before_external_sort")
+      .longConf
+      .createWithDefault(0)
   // scalastyle:on line.size.limit
 
   /** Gluten Configuration */

diff --git a/...e/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteTaskNotSerializableSuite.scala b/...e/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteTaskNotSerializableSuite.scala
@@ -17,7 +17,7 @@
 package org.apache.gluten.execution.mergetree
 
 import org.apache.gluten.GlutenConfig
-import org.apache.gluten.backendsapi.clickhouse.CHConf
+import org.apache.gluten.backendsapi.clickhouse.{CHConf, RuntimeSettings}
 import org.apache.gluten.execution.GlutenClickHouseTPCHAbstractSuite
 
 import org.apache.spark.SparkConf
@@ -53,7 +53,7 @@ class GlutenClickHouseMergeTreeWriteTaskNotSerializableSuite
 
   test("GLUTEN-6470: Fix Task not serializable error when inserting mergetree data") {
 
-    val externalSortKey = CHConf.runtimeSettings("max_bytes_before_external_sort")
+    val externalSortKey = RuntimeSettings.MAX_BYTES_BEFORE_EXTERNAL_SORT.key
     assertResult(3435973836L)(spark.conf.get(externalSortKey).toLong)
 
     spark.sql(s"""

diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version
@@ -1,3 +1,3 @@
 CH_ORG=Kyligence
-CH_BRANCH=rebase_ch/20241224
-CH_COMMIT=b38537577c5
+CH_BRANCH=rebase_ch/20241228
+CH_COMMIT=bf8e58b57e9
diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp b/cpp-ch/local-engine/Common/CHUtil.cpp
@@ -82,6 +82,7 @@ namespace Setting
 {
 extern const SettingsUInt64 prefer_external_sort_block_bytes;
 extern const SettingsUInt64 max_bytes_before_external_sort;
+extern const SettingsDouble max_bytes_ratio_before_external_sort;
 extern const SettingsBool query_plan_merge_filters;
 extern const SettingsBool compile_expressions;
 extern const SettingsShortCircuitFunctionEvaluation short_circuit_function_evaluation;
@@ -644,6 +645,10 @@ void BackendInitializerUtil::initSettings(const SparkConfigs::ConfigMap & spark_
     settings[Setting::short_circuit_function_evaluation] = ShortCircuitFunctionEvaluation::DISABLE;
     ///
 
+    // After https://github.com/ClickHouse/ClickHouse/pull/73422
+    // Since we already set max_bytes_before_external_sort, set max_bytes_ratio_before_external_sort to 0
+    settings[Setting::max_bytes_ratio_before_external_sort] = 0.;
+
     for (const auto & [key, value] : spark_conf_map)
     {
         // Firstly apply spark.gluten.sql.columnar.backend.ch.runtime_config.local_engine.settings.* to settings