From a87be93d13c949faec80f9f8b97cebc46d91fb4c Mon Sep 17 00:00:00 2001 From: hillium Date: Fri, 20 Dec 2024 15:44:16 +0800 Subject: [PATCH 1/4] allow pitr to create oversized indices Signed-off-by: hillium --- br/pkg/task/stream.go | 3 +++ br/tests/br_pitr/config/tidb-max-index-length.toml | 11 +++++++++++ br/tests/br_pitr/incremental_data/ingest_repair.sql | 3 +++ br/tests/br_pitr/prepare_data/ingest_repair.sql | 4 +++- br/tests/br_pitr/run.sh | 3 ++- 5 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 br/tests/br_pitr/config/tidb-max-index-length.toml diff --git a/br/pkg/task/stream.go b/br/pkg/task/stream.go index c8d297232274d..e87f7243a0fb2 100644 --- a/br/pkg/task/stream.go +++ b/br/pkg/task/stream.go @@ -1262,6 +1262,9 @@ func restoreStream( ctx, cancelFn := context.WithCancel(c) defer cancelFn() + restoreCfg := enableTiDBConfig() + defer restoreCfg() + if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil { span1 := span.Tracer().StartSpan( "restoreStream", diff --git a/br/tests/br_pitr/config/tidb-max-index-length.toml b/br/tests/br_pitr/config/tidb-max-index-length.toml new file mode 100644 index 0000000000000..9ed75ad154693 --- /dev/null +++ b/br/tests/br_pitr/config/tidb-max-index-length.toml @@ -0,0 +1,11 @@ +# config of tidb + +max-index-length = 12288 + +[security] +ssl-ca = "/tmp/backup_restore_test/certs/ca.pem" +ssl-cert = "/tmp/backup_restore_test/certs/tidb.pem" +ssl-key = "/tmp/backup_restore_test/certs/tidb.key" +cluster-ssl-ca = "/tmp/backup_restore_test/certs/ca.pem" +cluster-ssl-cert = "/tmp/backup_restore_test/certs/tidb.pem" +cluster-ssl-key = "/tmp/backup_restore_test/certs/tidb.key" diff --git a/br/tests/br_pitr/incremental_data/ingest_repair.sql b/br/tests/br_pitr/incremental_data/ingest_repair.sql index f82a78e6df3cd..0f0b9b825ed55 100644 --- a/br/tests/br_pitr/incremental_data/ingest_repair.sql +++ b/br/tests/br_pitr/incremental_data/ingest_repair.sql @@ -46,3 +46,6 @@ ALTER TABLE test.pairs9 CHANGE y y2 varchar(20); -- test partition ALTER TABLE test.pairs10 ADD INDEX i1(y); + + +CREATE INDEX huge ON test.huge_idx(blob1, blob2); diff --git a/br/tests/br_pitr/prepare_data/ingest_repair.sql b/br/tests/br_pitr/prepare_data/ingest_repair.sql index fc81fa1ced06e..763e6894b3701 100644 --- a/br/tests/br_pitr/prepare_data/ingest_repair.sql +++ b/br/tests/br_pitr/prepare_data/ingest_repair.sql @@ -43,4 +43,6 @@ INSERT INTO test.pairs10 VALUES (1,1,"1"),(2,2,"2"),(3,3,"3"),(4,4,"4"),(5,5,"5" -- test no need to repair CREATE TABLE test.pairs11(x int auto_increment primary key, y int DEFAULT RAND(), z int DEFAULT RAND()); INSERT INTO test.pairs11 VALUES (),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(),(); -ALTER TABLE test.pairs11 ADD UNIQUE KEY u1(x, y); \ No newline at end of file +ALTER TABLE test.pairs11 ADD UNIQUE KEY u1(x, y); + +CREATE TABLE test.huge_idx(id int AUTO_INCREMENT, blob1 varchar(1000), blob2 varchar(1000)); \ No newline at end of file diff --git a/br/tests/br_pitr/run.sh b/br/tests/br_pitr/run.sh index ba57723aa0465..5d0132884901e 100644 --- a/br/tests/br_pitr/run.sh +++ b/br/tests/br_pitr/run.sh @@ -24,7 +24,8 @@ res_file="$TEST_DIR/sql_res.$TEST_NAME.txt" # start a new cluster echo "restart a services" -restart_services +stop_services +start_services --tidb-cfg "$CUR/config/tidb-max-index-length.toml" # prepare the data echo "prepare the data" From 0a2ec185067dc6d45bc77d625cb4ece6dd526151 Mon Sep 17 00:00:00 2001 From: hillium Date: Fri, 20 Dec 2024 20:55:04 +0800 Subject: [PATCH 2/4] print error when failed Signed-off-by: hillium --- br/tests/br_pitr/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/br/tests/br_pitr/run.sh b/br/tests/br_pitr/run.sh index 1efa08d3b8ab9..f8cfec9e8a648 100644 --- a/br/tests/br_pitr/run.sh +++ b/br/tests/br_pitr/run.sh @@ -109,7 +109,7 @@ fi echo "run pitr" run_sql "DROP DATABASE __TiDB_BR_Temporary_Log_Restore_Checkpoint;" run_sql "DROP DATABASE __TiDB_BR_Temporary_Custom_SST_Restore_Checkpoint;" -run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" > $res_file 2>&1 +run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" > $res_file 2>&1 || ( cat $res_file && exit 1 ) check_result From 9e989bb615aa657388db9b59deab2f5be7caddd5 Mon Sep 17 00:00:00 2001 From: hillium Date: Mon, 23 Dec 2024 10:43:44 +0800 Subject: [PATCH 3/4] always enable huge index Signed-off-by: hillium --- br/tests/br_pitr/run.sh | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/br/tests/br_pitr/run.sh b/br/tests/br_pitr/run.sh index f8cfec9e8a648..0816537b7b087 100644 --- a/br/tests/br_pitr/run.sh +++ b/br/tests/br_pitr/run.sh @@ -23,10 +23,15 @@ PREFIX="pitr_backup" # NOTICE: don't start with 'br' because `restart services` res_file="$TEST_DIR/sql_res.$TEST_NAME.txt" TASK_NAME="br_pitr" +restart_services_allowing_huge_index() { + echo "restarting services with huge indices enabled..." + stop_services + start_services --tidb-cfg "$CUR/config/tidb-max-index-length.toml" + echo "restart services done..." +} + # start a new cluster -echo "restart a services" -stop_services -start_services --tidb-cfg "$CUR/config/tidb-max-index-length.toml" +restart_services_allowing_huge_index # prepare the data echo "prepare the data" @@ -93,8 +98,7 @@ check_result() { } # start a new cluster -echo "restart services" -restart_services +restart_services_allowing_huge_index # non-compliant operation echo "non compliant operation" @@ -114,8 +118,7 @@ run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-bac check_result # start a new cluster for incremental + log -echo "restart services" -restart_services +restart_services_allowing_huge_index echo "run snapshot restore#2" run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$PREFIX/full" @@ -127,7 +130,7 @@ check_result # start a new cluster for incremental + log echo "restart services" -restart_services +restart_services_allowing_huge_index echo "run snapshot restore#3" run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$PREFIX/full" @@ -141,8 +144,7 @@ if [ $restore_fail -ne 1 ]; then fi # start a new cluster for corruption -echo "restart a services" -restart_services +restart_services_allowing_huge_index file_corruption() { echo "corrupt the whole log files" @@ -168,8 +170,7 @@ if [ $restore_fail -ne 1 ]; then fi # start a new cluster for corruption -echo "restart a services" -restart_services +restart_services_allowing_huge_index file_lost() { echo "lost the whole log files" From 8737e48b53978bef10417df04f30836790dc6f3e Mon Sep 17 00:00:00 2001 From: hillium Date: Mon, 23 Dec 2024 12:10:51 +0800 Subject: [PATCH 4/4] rename enable -> tweak Signed-off-by: hillium --- br/pkg/task/common_test.go | 2 +- br/pkg/task/restore.go | 6 +++--- br/pkg/task/stream.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/br/pkg/task/common_test.go b/br/pkg/task/common_test.go index b58f3f52ee57f..cc0c64e531857 100644 --- a/br/pkg/task/common_test.go +++ b/br/pkg/task/common_test.go @@ -99,7 +99,7 @@ func TestUrlNoQuery(t *testing.T) { func TestTiDBConfigUnchanged(t *testing.T) { cfg := config.GetGlobalConfig() - restoreConfig := enableTiDBConfig() + restoreConfig := tweakLocalConfForRestore() require.NotEqual(t, config.GetGlobalConfig(), cfg) restoreConfig() require.Equal(t, config.GetGlobalConfig(), cfg) diff --git a/br/pkg/task/restore.go b/br/pkg/task/restore.go index 7aa5303e9c6e2..95c5141d82449 100644 --- a/br/pkg/task/restore.go +++ b/br/pkg/task/restore.go @@ -984,7 +984,7 @@ func runSnapshotRestore(c context.Context, mgr *conn.Mgr, g glue.Glue, cmdName s } // pre-set TiDB config for restore - restoreDBConfig := enableTiDBConfig() + restoreDBConfig := tweakLocalConfForRestore() defer restoreDBConfig() if client.GetSupportPolicy() { @@ -1407,9 +1407,9 @@ func filterRestoreFiles( return } -// enableTiDBConfig tweaks some of configs of TiDB to make the restore progress go well. +// tweakLocalConfForRestore tweaks some of configs of TiDB to make the restore progress go well. // return a function that could restore the config to origin. -func enableTiDBConfig() func() { +func tweakLocalConfForRestore() func() { restoreConfig := config.RestoreFunc() config.UpdateGlobal(func(conf *config.Config) { // set max-index-length before execute DDLs and create tables diff --git a/br/pkg/task/stream.go b/br/pkg/task/stream.go index 85b70e8d9aca3..d56b9b021354a 100644 --- a/br/pkg/task/stream.go +++ b/br/pkg/task/stream.go @@ -1321,7 +1321,7 @@ func restoreStream( ctx, cancelFn := context.WithCancel(c) defer cancelFn() - restoreCfg := enableTiDBConfig() + restoreCfg := tweakLocalConfForRestore() defer restoreCfg() if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {