use metadata cache

housepower · Aug 22, 2024 · a0725f8 · a0725f8
1 parent 0b51a2c
commit a0725f8
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 2 deletions.
diff --git a/ckconfig/custom.go b/ckconfig/custom.go
@@ -173,6 +173,15 @@ func expert(exp map[string]string) map[string]interface{} {
 	return common.ConvertMapping(output)
 }
 
+func merge_tree_metadata_cache() map[string]interface{} {
+	output := make(map[string]interface{})
+	output["merge_tree_metadata_cache"] = map[string]interface{}{
+		"lru_cache_size":        1073741824,
+		"continue_if_corrupted": true,
+	}
+	return output
+}
+
 func GenerateCustomXML(filename string, conf *model.CKManClickHouseConfig, ipv6Enable bool) (string, error) {
 	rootTag := "yandex"
 	if common.CompareClickHouseVersion(conf.Version, "22.x") >= 0 {
@@ -185,6 +194,9 @@ func GenerateCustomXML(filename string, conf *model.CKManClickHouseConfig, ipv6E
 	mergo.Merge(&custom, system_log())
 	mergo.Merge(&custom, distributed_ddl(conf.Cluster))
 	mergo.Merge(&custom, prometheus())
+	if common.CompareClickHouseVersion(conf.Version, "22.4.x") >= 0 {
+		mergo.Merge(&custom, merge_tree_metadata_cache())
+	}
 	storage_configuration, backups := storage(conf.Storage)
 	mergo.Merge(&custom, storage_configuration)
 	mergo.Merge(&custom, backups)

diff --git a/ckconfig/custom_fake.xml b/ckconfig/custom_fake.xml
@@ -50,6 +50,10 @@
         <replicated_fetches_http_receive_timeout>0</replicated_fetches_http_receive_timeout>
         <replicated_fetches_http_send_timeout>0</replicated_fetches_http_send_timeout>
     </merge_tree>
+    <merge_tree_metadata_cache>
+        <continue_if_corrupted>true</continue_if_corrupted>
+        <lru_cache_size>1073741824</lru_cache_size>
+    </merge_tree_metadata_cache>
     <metric_log>
         <flush_interval_milliseconds>30000</flush_interval_milliseconds>
         <partition_by>toYYYYMMDD(event_date)</partition_by>

diff --git a/ckconfig/custom_test.go b/ckconfig/custom_test.go
@@ -111,7 +111,7 @@ func TestGenerateCustomXML(t *testing.T) {
 		Cwd:       "/home/eoi/clickhouse",
 		NeedSudo:  false,
 		Path:      "/data01/",
-		Version:   "22.3.3.44",
+		Version:   "23.3.3.44",
 	}
 	_, err := GenerateCustomXML("custom_fake.xml", conf, true)
 	assert.Nil(t, err)

diff --git a/service/clickhouse/clickhouse_service.go b/service/clickhouse/clickhouse_service.go
@@ -132,15 +132,33 @@ func (ck *CkService) CreateTable(params *model.CreateCkTableParams, dryrun bool)
 		projections += fmt.Sprintf(", PROJECTION %s (%s)", p.Name, p.Sql)
 	}
 
+	settings := make(map[string]interface{})
+	if common.CompareClickHouseVersion(ck.Config.Version, "22.4.x") > 0 {
+		settings["use_metadata_cache"] = true
+	}
+
 	create := fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s`.`%s` ON CLUSTER `%s` (%s%s%s) ENGINE = %s() PARTITION BY %s ORDER BY (%s)",
 		params.DB, params.Name, params.Cluster, strings.Join(columns, ", "), params.IndexExpr, projections, params.Engine,
 		partition, strings.Join(params.Order, ", "))
 	if params.TTLExpr != "" {
 		create += fmt.Sprintf(" TTL %s", params.TTLExpr)
 	}
 	if params.StoragePolicy != "" {
-		create += fmt.Sprintf(" SETTINGS storage_policy = '%s'", params.StoragePolicy)
+		settings["storage_policy"] = params.StoragePolicy
+	}
+	if len(settings) > 0 {
+		create += " SETTINGS "
+		idx := 0
+		for k, v := range settings {
+			if idx == len(settings) {
+				create += fmt.Sprintf("%s = '%v'", k, v)
+			} else {
+				create += fmt.Sprintf("%s = '%v',", k, v)
+			}
+			idx++
+		}
 	}
+
 	log.Logger.Debugf(create)
 	statements = append(statements, create)
 	if !dryrun {