From 4baa1b1a000aaae5c630dddf19e40acafc56f98f Mon Sep 17 00:00:00 2001 From: Austen McClernon Date: Wed, 9 Oct 2024 02:30:00 -0400 Subject: [PATCH 1/8] kvserver: specify flow_control_integration_test_v2 encoding Note to the reader, only files have moved without other changes. We didn't specify the v2 encoding in the testdata file but did for v1 encoding. Now specify both. Part of: #128040 Release note: None --- ...on_post_split_merge => admission_post_split_merge_v2_encoding} | 0 .../flow_control_integration_v2/{basic => basic_v2_encoding} | 0 .../{blocked_admission => blocked_admission_v2_encoding} | 0 .../{class_prioritization => class_prioritization_v2_encoding} | 0 .../{crashed_node => crashed_node_v2_encoding} | 0 ...nter_admit_one_by_one => granter_admit_one_by_one_v2_encoding} | 0 ...{leader_not_leaseholder => leader_not_leaseholder_v2_encoding} | 0 ...ership_remove_self => raft_membership_remove_self_v2_encoding} | 0 .../{raft_membership => raft_membership_v2_encoding} | 0 .../{raft_snapshot => raft_snapshot_v2_encoding} | 0 .../{split_merge => split_merge_v2_encoding} | 0 .../{transfer_lease => transfer_lease_v2_encoding} | 0 .../{unquiesced_range => unquiesced_range_v2_encoding} | 0 13 files changed, 0 insertions(+), 0 deletions(-) rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{admission_post_split_merge => admission_post_split_merge_v2_encoding} (100%) rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{basic => basic_v2_encoding} (100%) rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{blocked_admission => blocked_admission_v2_encoding} (100%) rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{class_prioritization => class_prioritization_v2_encoding} (100%) rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{crashed_node => crashed_node_v2_encoding} (100%) rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{granter_admit_one_by_one => granter_admit_one_by_one_v2_encoding} (100%) rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{leader_not_leaseholder => leader_not_leaseholder_v2_encoding} (100%) rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{raft_membership_remove_self => raft_membership_remove_self_v2_encoding} (100%) rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{raft_membership => raft_membership_v2_encoding} (100%) rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{raft_snapshot => raft_snapshot_v2_encoding} (100%) rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{split_merge => split_merge_v2_encoding} (100%) rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{transfer_lease => transfer_lease_v2_encoding} (100%) rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{unquiesced_range => unquiesced_range_v2_encoding} (100%) diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge b/pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v2_encoding similarity index 100% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v2_encoding diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/basic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v2_encoding similarity index 100% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/basic rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v2_encoding diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission b/pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v2_encoding similarity index 100% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v2_encoding diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization b/pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v2_encoding similarity index 100% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v2_encoding diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node b/pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v2_encoding similarity index 100% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v2_encoding diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one b/pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v2_encoding similarity index 100% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v2_encoding diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder b/pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v2_encoding similarity index 100% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v2_encoding diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v2_encoding similarity index 100% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v2_encoding diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v2_encoding similarity index 100% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v2_encoding diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v2_encoding similarity index 100% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v2_encoding diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge b/pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v2_encoding similarity index 100% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v2_encoding diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease b/pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v2_encoding similarity index 100% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v2_encoding diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range b/pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range_v2_encoding similarity index 100% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range_v2_encoding From c8e30410fad35ef64286f48aed1cd0e1a7bcac59 Mon Sep 17 00:00:00 2001 From: Austen McClernon Date: Wed, 9 Oct 2024 03:22:53 -0400 Subject: [PATCH 2/8] kvserver: add rac2 pull mode integration test variations Introduce pull mode (send queue) variants to the existing TestFlowControl.*V2 integration tests. The tests specify `apply_to_all`, rather than `apply_to_elastic` for the cluster setting `kvadmission.flow_control.mode` -- which enables pull mode in raft and send queue management in RACv2. These tests use their own file, as the behavior differs in some cases between pull and push mode. Part of: #128040 Release note: None --- .../kvserver/flow_control_integration_test.go | 1965 +++++++++-------- .../kvserver/kvflowcontrol/testing_knobs.go | 11 + ...post_split_merge_v1_encoding_apply_to_all} | 4 +- ...t_split_merge_v1_encoding_apply_to_elastic | 181 ++ ...post_split_merge_v2_encoding_apply_to_all} | 4 +- ...t_split_merge_v2_encoding_apply_to_elastic | 174 ++ ...ncoding => basic_v1_encoding_apply_to_all} | 6 +- .../basic_v1_encoding_apply_to_elastic | 109 + ...ncoding => basic_v2_encoding_apply_to_all} | 6 +- .../basic_v2_encoding_apply_to_elastic | 102 + ...locked_admission_v1_encoding_apply_to_all} | 4 +- ...ked_admission_v1_encoding_apply_to_elastic | 118 + ...locked_admission_v2_encoding_apply_to_all} | 38 +- ...ked_admission_v2_encoding_apply_to_elastic | 111 + ...s_prioritization_v1_encoding_apply_to_all} | 4 +- ...rioritization_v1_encoding_apply_to_elastic | 119 + ...s_prioritization_v2_encoding_apply_to_all} | 28 +- ...rioritization_v2_encoding_apply_to_elastic | 112 + ... => crashed_node_v1_encoding_apply_to_all} | 4 +- .../crashed_node_v1_encoding_apply_to_elastic | 103 + ... => crashed_node_v2_encoding_apply_to_all} | 4 +- .../crashed_node_v2_encoding_apply_to_elastic | 96 + ...admit_one_by_one_v1_encoding_apply_to_all} | 2 +- ...it_one_by_one_v1_encoding_apply_to_elastic | 95 + ...admit_one_by_one_v2_encoding_apply_to_all} | 2 +- ...it_one_by_one_v2_encoding_apply_to_elastic | 88 + ..._not_leaseholder_v1_encoding_apply_to_all} | 4 +- ...t_leaseholder_v1_encoding_apply_to_elastic | 182 ++ ..._not_leaseholder_v2_encoding_apply_to_all} | 4 +- ...t_leaseholder_v2_encoding_apply_to_elastic | 175 ++ ...ship_remove_self_v1_encoding_apply_to_all} | 4 +- ...p_remove_self_v1_encoding_apply_to_elastic | 143 ++ ...ship_remove_self_v2_encoding_apply_to_all} | 4 +- ...p_remove_self_v2_encoding_apply_to_elastic | 136 ++ ... raft_membership_v1_encoding_apply_to_all} | 4 +- ...ft_membership_v1_encoding_apply_to_elastic | 153 ++ ... raft_membership_v2_encoding_apply_to_all} | 4 +- ...ft_membership_v2_encoding_apply_to_elastic | 146 ++ ...=> raft_snapshot_v1_encoding_apply_to_all} | 8 +- ...raft_snapshot_v1_encoding_apply_to_elastic | 244 ++ ...=> raft_snapshot_v2_encoding_apply_to_all} | 8 +- ...raft_snapshot_v2_encoding_apply_to_elastic | 237 ++ ...g => split_merge_v1_encoding_apply_to_all} | 4 +- .../split_merge_v1_encoding_apply_to_elastic | 141 ++ ...g => split_merge_v2_encoding_apply_to_all} | 4 +- .../split_merge_v2_encoding_apply_to_elastic | 134 ++ ...> transfer_lease_v1_encoding_apply_to_all} | 4 +- ...ransfer_lease_v1_encoding_apply_to_elastic | 83 + ...> transfer_lease_v2_encoding_apply_to_all} | 4 +- ...ransfer_lease_v2_encoding_apply_to_elastic | 76 + ...unquiesced_range_v1_encoding_apply_to_all} | 0 ...uiesced_range_v1_encoding_apply_to_elastic | 148 ++ ...unquiesced_range_v2_encoding_apply_to_all} | 0 ...uiesced_range_v2_encoding_apply_to_elastic | 141 ++ 54 files changed, 4665 insertions(+), 1020 deletions(-) rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{admission_post_split_merge_v1_encoding => admission_post_split_merge_v1_encoding_apply_to_all} (98%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v1_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{admission_post_split_merge_v2_encoding => admission_post_split_merge_v2_encoding_apply_to_all} (98%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v2_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{basic_v1_encoding => basic_v1_encoding_apply_to_all} (96%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v1_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{basic_v2_encoding => basic_v2_encoding_apply_to_all} (95%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v2_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{blocked_admission_v1_encoding => blocked_admission_v1_encoding_apply_to_all} (97%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v1_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{blocked_admission_v2_encoding => blocked_admission_v2_encoding_apply_to_all} (84%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v2_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{class_prioritization_v1_encoding => class_prioritization_v1_encoding_apply_to_all} (97%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v1_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{class_prioritization_v2_encoding => class_prioritization_v2_encoding_apply_to_all} (95%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v2_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{crashed_node_v1_encoding => crashed_node_v1_encoding_apply_to_all} (97%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v1_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{crashed_node_v2_encoding => crashed_node_v2_encoding_apply_to_all} (97%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v2_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{granter_admit_one_by_one_v1_encoding => granter_admit_one_by_one_v1_encoding_apply_to_all} (98%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v1_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{granter_admit_one_by_one_v2_encoding => granter_admit_one_by_one_v2_encoding_apply_to_all} (98%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v2_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{leader_not_leaseholder_v1_encoding => leader_not_leaseholder_v1_encoding_apply_to_all} (98%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v1_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{leader_not_leaseholder_v2_encoding => leader_not_leaseholder_v2_encoding_apply_to_all} (98%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v2_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{raft_membership_remove_self_v1_encoding => raft_membership_remove_self_v1_encoding_apply_to_all} (97%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v1_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{raft_membership_remove_self_v2_encoding => raft_membership_remove_self_v2_encoding_apply_to_all} (97%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v2_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{raft_membership_v1_encoding => raft_membership_v1_encoding_apply_to_all} (97%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v1_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{raft_membership_v2_encoding => raft_membership_v2_encoding_apply_to_all} (97%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v2_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{raft_snapshot_v1_encoding => raft_snapshot_v1_encoding_apply_to_all} (97%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v1_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{raft_snapshot_v2_encoding => raft_snapshot_v2_encoding_apply_to_all} (97%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v2_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{split_merge_v1_encoding => split_merge_v1_encoding_apply_to_all} (97%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v1_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{split_merge_v2_encoding => split_merge_v2_encoding_apply_to_all} (97%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v2_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{transfer_lease_v1_encoding => transfer_lease_v1_encoding_apply_to_all} (96%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v1_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{transfer_lease_v2_encoding => transfer_lease_v2_encoding_apply_to_all} (96%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v2_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{unquiesced_range_v1_encoding => unquiesced_range_v1_encoding_apply_to_all} (100%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range_v1_encoding_apply_to_elastic rename pkg/kv/kvserver/testdata/flow_control_integration_v2/{unquiesced_range_v2_encoding => unquiesced_range_v2_encoding_apply_to_all} (100%) create mode 100644 pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range_v2_encoding_apply_to_elastic diff --git a/pkg/kv/kvserver/flow_control_integration_test.go b/pkg/kv/kvserver/flow_control_integration_test.go index e541e1e38574..18c733108b43 100644 --- a/pkg/kv/kvserver/flow_control_integration_test.go +++ b/pkg/kv/kvserver/flow_control_integration_test.go @@ -101,7 +101,7 @@ func TestFlowControlBasic(t *testing.T) { n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) h := newFlowControlTestHelperV1(t, tc) - h.init() + h.init(kvflowcontrol.ApplyToAll) defer h.close("basic") // this test behaves identically with or without the fast path h.enableVerboseRaftMsgLoggingForRange(desc.RangeID) @@ -246,7 +246,7 @@ func TestFlowControlRangeSplitMerge(t *testing.T) { n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) h := newFlowControlTestHelperV1(t, tc) - h.init() + h.init(kvflowcontrol.ApplyToAll) defer h.close("split_merge") desc, err := tc.LookupRange(k) @@ -375,7 +375,7 @@ func TestFlowControlBlockedAdmission(t *testing.T) { n2 := sqlutils.MakeSQLRunner(tc.ServerConn(1)) h := newFlowControlTestHelperV1(t, tc) - h.init() + h.init(kvflowcontrol.ApplyToAll) defer h.close("blocked_admission") desc, err := tc.LookupRange(k) @@ -486,7 +486,7 @@ func TestFlowControlAdmissionPostSplitMerge(t *testing.T) { n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) h := newFlowControlTestHelperV1(t, tc) - h.init() + h.init(kvflowcontrol.ApplyToAll) defer h.close("admission_post_split_merge") desc, err := tc.LookupRange(k) @@ -639,7 +639,7 @@ func TestFlowControlCrashedNode(t *testing.T) { n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) h := newFlowControlTestHelperV1(t, tc) - h.init() + h.init(kvflowcontrol.ApplyToAll) defer h.close("crashed_node") desc, err := tc.LookupRange(k) @@ -788,7 +788,7 @@ func TestFlowControlRaftSnapshot(t *testing.T) { n4 := sqlutils.MakeSQLRunner(tc.ServerConn(3)) h := newFlowControlTestHelperV1(t, tc) - h.init() + h.init(kvflowcontrol.ApplyToAll) defer h.close("raft_snapshot") store := tc.GetFirstStoreFromServer(t, 0) @@ -1047,7 +1047,7 @@ func TestFlowControlRaftTransportBreak(t *testing.T) { n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) h := newFlowControlTestHelperV1(t, tc) - h.init() + h.init(kvflowcontrol.ApplyToAll) defer h.close("raft_transport_break") desc, err := tc.LookupRange(k) @@ -1176,7 +1176,7 @@ func TestFlowControlRaftTransportCulled(t *testing.T) { n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) h := newFlowControlTestHelperV1(t, tc) - h.init() + h.init(kvflowcontrol.ApplyToAll) defer h.close("raft_transport_culled") desc, err := tc.LookupRange(k) @@ -1285,7 +1285,7 @@ func TestFlowControlRaftMembership(t *testing.T) { n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) h := newFlowControlTestHelperV1(t, tc) - h.init() + h.init(kvflowcontrol.ApplyToAll) defer h.close("raft_membership") desc, err := tc.LookupRange(k) @@ -1436,7 +1436,7 @@ func TestFlowControlRaftMembershipRemoveSelf(t *testing.T) { n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) h := newFlowControlTestHelperV1(t, tc) - h.init() + h.init(kvflowcontrol.ApplyToAll) defer h.close("raft_membership_remove_self") // this test behaves identically independent of we transfer the lease first desc, err := tc.LookupRange(k) @@ -1544,7 +1544,7 @@ func TestFlowControlClassPrioritization(t *testing.T) { n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) h := newFlowControlTestHelperV1(t, tc) - h.init() + h.init(kvflowcontrol.ApplyToAll) defer h.close("class_prioritization") desc, err := tc.LookupRange(k) @@ -1651,7 +1651,7 @@ func TestFlowControlQuiescedRange(t *testing.T) { n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) h := newFlowControlTestHelperV1(t, tc) - h.init() + h.init(kvflowcontrol.ApplyToAll) defer h.close("quiesced_range") desc, err := tc.LookupRange(k) @@ -1800,7 +1800,7 @@ func TestFlowControlUnquiescedRange(t *testing.T) { n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) h := newFlowControlTestHelperV1(t, tc) - h.init() + h.init(kvflowcontrol.ApplyToAll) defer h.close("unquiesced_range") desc, err := tc.LookupRange(k) @@ -1920,7 +1920,7 @@ func TestFlowControlTransferLease(t *testing.T) { n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) h := newFlowControlTestHelperV1(t, tc) - h.init() + h.init(kvflowcontrol.ApplyToAll) defer h.close("transfer_lease") desc, err := tc.LookupRange(k) @@ -2013,7 +2013,7 @@ func TestFlowControlLeaderNotLeaseholder(t *testing.T) { n2 := sqlutils.MakeSQLRunner(tc.ServerConn(1)) h := newFlowControlTestHelperV1(t, tc) - h.init() + h.init(kvflowcontrol.ApplyToAll) defer h.close("leader_not_leaseholder") desc, err := tc.LookupRange(k) @@ -2160,7 +2160,7 @@ func TestFlowControlGranterAdmitOneByOne(t *testing.T) { n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) h := newFlowControlTestHelperV1(t, tc) - h.init() + h.init(kvflowcontrol.ApplyToAll) defer h.close("granter_admit_one_by_one") desc, err := tc.LookupRange(k) @@ -2200,7 +2200,7 @@ func TestFlowControlGranterAdmitOneByOne(t *testing.T) { } // TestFlowControlBasicV2 runs a basic end-to-end test of the v2 kvflowcontrol -// machinery, replicating + admitting a single 1MiB regular write. The vmodule +// machinery, replicating + admitting a single 1MiB write. The vmodule // flags for running these tests with full logging are: // // --vmodule='replica_raft=1,replica_proposal_buf=1,raft_transport=2, @@ -2215,81 +2215,86 @@ func TestFlowControlBasicV2(t *testing.T) { kvflowcontrol.V2EnabledWhenLeaderV1Encoding, kvflowcontrol.V2EnabledWhenLeaderV2Encoding, }, func(t *testing.T, v2EnabledWhenLeaderLevel kvflowcontrol.V2EnabledWhenLeaderLevel) { - testutils.RunTrueAndFalse(t, "always-enqueue", func(t *testing.T, alwaysEnqueue bool) { - ctx := context.Background() - settings := cluster.MakeTestingClusterSettings() - tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{ - ReplicationMode: base.ReplicationManual, - ServerArgs: base.TestServerArgs{ - Settings: settings, - Knobs: base.TestingKnobs{ - Store: &kvserver.StoreTestingKnobs{ - FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ - UseOnlyForScratchRanges: true, - OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { - return v2EnabledWhenLeaderLevel + testutils.RunValues(t, "kvadmission.flow_control.mode", []kvflowcontrol.ModeT{ + kvflowcontrol.ApplyToElastic, + kvflowcontrol.ApplyToAll, + }, func(t *testing.T, mode kvflowcontrol.ModeT) { + testutils.RunTrueAndFalse(t, "always-enqueue", func(t *testing.T, alwaysEnqueue bool) { + ctx := context.Background() + settings := cluster.MakeTestingClusterSettings() + tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{ + ReplicationMode: base.ReplicationManual, + ServerArgs: base.TestServerArgs{ + Settings: settings, + Knobs: base.TestingKnobs{ + Store: &kvserver.StoreTestingKnobs{ + FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ + UseOnlyForScratchRanges: true, + OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { + return v2EnabledWhenLeaderLevel + }, }, }, - }, - AdmissionControl: &admission.TestingKnobs{ - DisableWorkQueueFastPath: alwaysEnqueue, + AdmissionControl: &admission.TestingKnobs{ + DisableWorkQueueFastPath: alwaysEnqueue, + }, }, }, - }, - }) - defer tc.Stopper().Stop(ctx) - - // Setup the test state with 3 voters, one on each of the three - // node/stores. - k := tc.ScratchRange(t) - tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) - h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) - h.init() - defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, "basic")) - - desc, err := tc.LookupRange(k) - require.NoError(t, err) - h.enableVerboseRaftMsgLoggingForRange(desc.RangeID) - n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) - - h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) - - h.comment(`-- Flow token metrics, before issuing the regular 1MiB replicated write.`) - h.query(n1, v2FlowTokensQueryStr) - - h.comment(`-- (Issuing + admitting a regular 1MiB, triply replicated write...)`) - h.log("sending put request") - h.put(ctx, k, 1<<20 /* 1MiB */, admissionpb.NormalPri) - h.log("sent put request") - - h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) - h.comment(` + }) + defer tc.Stopper().Stop(ctx) + + // Setup the test state with 3 voters, one on each of the three + // node/stores. + k := tc.ScratchRange(t) + tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) + h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) + h.init(mode) + defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, mode, "basic")) + + desc, err := tc.LookupRange(k) + require.NoError(t, err) + h.enableVerboseRaftMsgLoggingForRange(desc.RangeID) + n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) + + h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) + + h.comment(`-- Flow token metrics, before issuing the 1MiB replicated write.`) + h.query(n1, v2FlowTokensQueryStr) + + h.comment(`-- (Issuing + admitting a 1MiB, triply replicated write...)`) + h.log("sending put request") + h.put(ctx, k, 1<<20 /* 1MiB */, testFlowModeToPri(mode)) + h.log("sent put request") + + h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) + h.comment(` -- Stream counts as seen by n1 post-write. We should see three {regular,elastic} -- streams given there are three nodes and we're using a replication factor of -- three. `) - h.query(n1, ` + h.query(n1, ` SELECT name, value FROM crdb_internal.node_metrics WHERE name LIKE '%kvflowcontrol%stream%' ORDER BY name ASC; `) - h.comment(`-- Another view of the stream count, using /inspectz-backed vtables.`) - h.query(n1, ` + h.comment(`-- Another view of the stream count, using /inspectz-backed vtables.`) + h.query(n1, ` SELECT range_id, count(*) AS streams FROM crdb_internal.kv_flow_control_handles_v2 GROUP BY (range_id) ORDER BY streams DESC; `, "range_id", "stream_count") - h.comment(` --- Flow token metrics from n1 after issuing the regular 1MiB replicated write, + h.comment(` +-- Flow token metrics from n1 after issuing the 1MiB replicated write, -- and it being admitted on n1, n2 and n3. We should see 3*1MiB = 3MiB of -- {regular,elastic} tokens deducted and returned, and {8*3=24MiB,16*3=48MiB} of -- {regular,elastic} tokens available. Everything should be accounted for. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) + }) }) }) } @@ -2304,102 +2309,107 @@ func TestFlowControlRangeSplitMergeV2(t *testing.T) { kvflowcontrol.V2EnabledWhenLeaderV1Encoding, kvflowcontrol.V2EnabledWhenLeaderV2Encoding, }, func(t *testing.T, v2EnabledWhenLeaderLevel kvflowcontrol.V2EnabledWhenLeaderLevel) { - ctx := context.Background() - settings := cluster.MakeTestingClusterSettings() - tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{ - ReplicationMode: base.ReplicationManual, - ServerArgs: base.TestServerArgs{ - Settings: settings, - Knobs: base.TestingKnobs{ - Store: &kvserver.StoreTestingKnobs{ - FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ - UseOnlyForScratchRanges: true, - OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { - return v2EnabledWhenLeaderLevel + testutils.RunValues(t, "kvadmission.flow_control.mode", []kvflowcontrol.ModeT{ + kvflowcontrol.ApplyToElastic, + kvflowcontrol.ApplyToAll, + }, func(t *testing.T, mode kvflowcontrol.ModeT) { + ctx := context.Background() + settings := cluster.MakeTestingClusterSettings() + tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{ + ReplicationMode: base.ReplicationManual, + ServerArgs: base.TestServerArgs{ + Settings: settings, + Knobs: base.TestingKnobs{ + Store: &kvserver.StoreTestingKnobs{ + FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ + UseOnlyForScratchRanges: true, + OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { + return v2EnabledWhenLeaderLevel + }, }, }, }, }, - }, - }) - defer tc.Stopper().Stop(ctx) + }) + defer tc.Stopper().Stop(ctx) - k := tc.ScratchRange(t) - tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) + k := tc.ScratchRange(t) + tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) - n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) + n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) - h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) - h.init() - defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, "split_merge")) + h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) + h.init(mode) + defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, mode, "split_merge")) - desc, err := tc.LookupRange(k) - require.NoError(t, err) + desc, err := tc.LookupRange(k) + require.NoError(t, err) - h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) - h.log("sending put request to pre-split range") - h.put(ctx, k, 1<<20 /* 1MiB */, admissionpb.NormalPri) - h.log("sent put request to pre-split range") + h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) + h.log("sending put request to pre-split range") + h.put(ctx, k, 1<<20 /* 1MiB */, testFlowModeToPri(mode)) + h.log("sent put request to pre-split range") - h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) - h.comment(` --- Flow token metrics from n1 after issuing + admitting the regular 1MiB 3x + h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) + h.comment(` +-- Flow token metrics from n1 after issuing + admitting the 1MiB 3x -- replicated write to the pre-split range. There should be 3MiB of -- {regular,elastic} tokens {deducted,returned}. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(`-- (Splitting range.)`) - left, right := tc.SplitRangeOrFatal(t, k.Next()) - h.waitForConnectedStreams(ctx, right.RangeID, 3, 0 /* serverIdx */) + h.comment(`-- (Splitting range.)`) + left, right := tc.SplitRangeOrFatal(t, k.Next()) + h.waitForConnectedStreams(ctx, right.RangeID, 3, 0 /* serverIdx */) - h.log("sending 2MiB put request to post-split LHS") - h.put(ctx, k, 2<<20 /* 2MiB */, admissionpb.NormalPri) - h.log("sent 2MiB put request to post-split LHS") + h.log("sending 2MiB put request to post-split LHS") + h.put(ctx, k, 2<<20 /* 2MiB */, testFlowModeToPri(mode)) + h.log("sent 2MiB put request to post-split LHS") - h.log("sending 3MiB put request to post-split RHS") - h.put(ctx, roachpb.Key(right.StartKey), 3<<20 /* 3MiB */, admissionpb.NormalPri) - h.log("sent 3MiB put request to post-split RHS") + h.log("sending 3MiB put request to post-split RHS") + h.put(ctx, roachpb.Key(right.StartKey), 3<<20 /* 3MiB */, testFlowModeToPri(mode)) + h.log("sent 3MiB put request to post-split RHS") - h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) - h.comment(` + h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) + h.comment(` -- Flow token metrics from n1 after further issuing 2MiB and 3MiB writes to -- post-split LHS and RHS ranges respectively. We should see 15MiB extra tokens -- {deducted,returned}, which comes from (2MiB+3MiB)*3=15MiB. So we stand at -- 3MiB+15MiB=18MiB now. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(`-- Observe the newly split off replica, with its own three streams.`) - h.query(n1, ` + h.comment(`-- Observe the newly split off replica, with its own three streams.`) + h.query(n1, ` SELECT range_id, count(*) AS streams FROM crdb_internal.kv_flow_control_handles_v2 GROUP BY (range_id) ORDER BY streams DESC; `, "range_id", "stream_count") - h.comment(`-- (Merging ranges.)`) - merged := tc.MergeRangesOrFatal(t, left.StartKey.AsRawKey()) + h.comment(`-- (Merging ranges.)`) + merged := tc.MergeRangesOrFatal(t, left.StartKey.AsRawKey()) - h.log("sending 4MiB put request to post-merge range") - h.put(ctx, roachpb.Key(merged.StartKey), 4<<20 /* 4MiB */, admissionpb.NormalPri) - h.log("sent 4MiB put request to post-merged range") + h.log("sending 4MiB put request to post-merge range") + h.put(ctx, roachpb.Key(merged.StartKey), 4<<20 /* 4MiB */, testFlowModeToPri(mode)) + h.log("sent 4MiB put request to post-merged range") - h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) - h.comment(` --- Flow token metrics from n1 after issuing 4MiB of regular replicated writes to + h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) + h.comment(` +-- Flow token metrics from n1 after issuing 4MiB of replicated writes to -- the post-merged range. We should see 12MiB extra tokens {deducted,returned}, -- which comes from 4MiB*3=12MiB. So we stand at 18MiB+12MiB=30MiB now. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(`-- Observe only the merged replica with its own three streams.`) - h.query(n1, ` + h.comment(`-- Observe only the merged replica with its own three streams.`) + h.query(n1, ` SELECT range_id, count(*) AS streams FROM crdb_internal.kv_flow_control_handles_v2 GROUP BY (range_id) ORDER BY streams DESC; `, "range_id", "stream_count") + }) }) } @@ -2413,84 +2423,89 @@ func TestFlowControlBlockedAdmissionV2(t *testing.T) { kvflowcontrol.V2EnabledWhenLeaderV1Encoding, kvflowcontrol.V2EnabledWhenLeaderV2Encoding, }, func(t *testing.T, v2EnabledWhenLeaderLevel kvflowcontrol.V2EnabledWhenLeaderLevel) { - ctx := context.Background() - var disableWorkQueueGranting atomic.Bool - disableWorkQueueGranting.Store(true) - settings := cluster.MakeTestingClusterSettings() - tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{ - ReplicationMode: base.ReplicationManual, - ServerArgs: base.TestServerArgs{ - Settings: settings, - Knobs: base.TestingKnobs{ - Store: &kvserver.StoreTestingKnobs{ - FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ - UseOnlyForScratchRanges: true, - OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { - return v2EnabledWhenLeaderLevel + testutils.RunValues(t, "kvadmission.flow_control.mode", []kvflowcontrol.ModeT{ + kvflowcontrol.ApplyToElastic, + kvflowcontrol.ApplyToAll, + }, func(t *testing.T, mode kvflowcontrol.ModeT) { + ctx := context.Background() + var disableWorkQueueGranting atomic.Bool + disableWorkQueueGranting.Store(true) + settings := cluster.MakeTestingClusterSettings() + tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{ + ReplicationMode: base.ReplicationManual, + ServerArgs: base.TestServerArgs{ + Settings: settings, + Knobs: base.TestingKnobs{ + Store: &kvserver.StoreTestingKnobs{ + FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ + UseOnlyForScratchRanges: true, + OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { + return v2EnabledWhenLeaderLevel + }, }, }, - }, - AdmissionControl: &admission.TestingKnobs{ - DisableWorkQueueFastPath: true, - DisableWorkQueueGranting: func() bool { - return disableWorkQueueGranting.Load() + AdmissionControl: &admission.TestingKnobs{ + DisableWorkQueueFastPath: true, + DisableWorkQueueGranting: func() bool { + return disableWorkQueueGranting.Load() + }, }, }, }, - }, - }) - defer tc.Stopper().Stop(ctx) + }) + defer tc.Stopper().Stop(ctx) - k := tc.ScratchRange(t) - tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) + k := tc.ScratchRange(t) + tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) - n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) + n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) - h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) - h.init() - defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, "blocked_admission")) + h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) + h.init(mode) + defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, mode, "blocked_admission")) - desc, err := tc.LookupRange(k) - require.NoError(t, err) - h.enableVerboseRaftMsgLoggingForRange(desc.RangeID) - h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) + desc, err := tc.LookupRange(k) + require.NoError(t, err) + h.enableVerboseRaftMsgLoggingForRange(desc.RangeID) + h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) - h.comment(`-- (Issuing 5 regular 1MiB, 3x replicated write that's not admitted.)`) - h.log("sending put requests") - for i := 0; i < 5; i++ { - h.put(ctx, k, 1<<20 /* 1MiB */, admissionpb.NormalPri) - } - h.log("sent put requests") + h.comment(`-- (Issuing 5 1MiB, 3x replicated write that's not admitted.)`) + h.log("sending put requests") + for i := 0; i < 5; i++ { + h.put(ctx, k, 1<<20 /* 1MiB */, testFlowModeToPri(mode)) + } + h.log("sent put requests") - h.comment(` --- Flow token metrics from n1 after issuing 5 regular 1MiB 3x replicated writes + h.comment(` +-- Flow token metrics from n1 after issuing 5 1MiB 3x replicated writes -- that are yet to get admitted. We see 5*1MiB*3=15MiB deductions of -- {regular,elastic} tokens with no corresponding returns. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(`-- Observe the total tracked tokens per-stream on n1.`) - h.query(n1, ` + h.comment(`-- Observe the total tracked tokens per-stream on n1.`) + h.query(n1, ` SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) FROM crdb_internal.kv_flow_control_handles_v2 `, "range_id", "store_id", "total_tracked_tokens") - h.comment(`-- Observe the individual tracked tokens per-stream on the scratch range.`) - h.query(n1, ` + h.comment(`-- Observe the individual tracked tokens per-stream on the scratch range.`) + h.query(n1, ` SELECT range_id, store_id, priority, crdb_internal.humanize_bytes(tokens::INT8) FROM crdb_internal.kv_flow_token_deductions_v2 `, "range_id", "store_id", "priority", "tokens") - h.comment(`-- (Allow below-raft admission to proceed.)`) - disableWorkQueueGranting.Store(false) - h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) // wait for admission + h.comment(`-- (Allow below-raft admission to proceed.)`) + disableWorkQueueGranting.Store(false) + h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) // wait for admission - h.comment(` + h.comment(` -- Flow token metrics from n1 after work gets admitted. We see 15MiB returns of -- {regular,elastic} tokens, and the available capacities going back to what -- they were. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) + }) }) } @@ -2509,138 +2524,143 @@ func TestFlowControlAdmissionPostSplitMergeV2(t *testing.T) { kvflowcontrol.V2EnabledWhenLeaderV1Encoding, kvflowcontrol.V2EnabledWhenLeaderV2Encoding, }, func(t *testing.T, v2EnabledWhenLeaderLevel kvflowcontrol.V2EnabledWhenLeaderLevel) { - ctx := context.Background() - var disableWorkQueueGranting atomic.Bool - disableWorkQueueGranting.Store(true) - settings := cluster.MakeTestingClusterSettings() - tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{ - ReplicationMode: base.ReplicationManual, - ServerArgs: base.TestServerArgs{ - Settings: settings, - RaftConfig: base.RaftConfig{ - // Suppress timeout-based elections. This test doesn't want to - // deal with leadership changing hands. - RaftElectionTimeoutTicks: 1000000, - }, - Knobs: base.TestingKnobs{ - Store: &kvserver.StoreTestingKnobs{ - FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ - UseOnlyForScratchRanges: true, - OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { - return v2EnabledWhenLeaderLevel - }, - OverrideTokenDeduction: func(tokens kvflowcontrol.Tokens) kvflowcontrol.Tokens { - // This test sends several puts, with each put potentially - // diverging by a few bytes between runs, in aggregate this - // can accumulate to enough tokens to produce a diff in - // metrics. Round the token deductions to the nearest MiB avoid - // this. - return kvflowcontrol.Tokens( - int64(math.Round(float64(tokens)/float64(1<<20))) * 1 << 20) + testutils.RunValues(t, "kvadmission.flow_control.mode", []kvflowcontrol.ModeT{ + kvflowcontrol.ApplyToElastic, + kvflowcontrol.ApplyToAll, + }, func(t *testing.T, mode kvflowcontrol.ModeT) { + ctx := context.Background() + var disableWorkQueueGranting atomic.Bool + disableWorkQueueGranting.Store(true) + settings := cluster.MakeTestingClusterSettings() + tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{ + ReplicationMode: base.ReplicationManual, + ServerArgs: base.TestServerArgs{ + Settings: settings, + RaftConfig: base.RaftConfig{ + // Suppress timeout-based elections. This test doesn't want to + // deal with leadership changing hands. + RaftElectionTimeoutTicks: 1000000, + }, + Knobs: base.TestingKnobs{ + Store: &kvserver.StoreTestingKnobs{ + FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ + UseOnlyForScratchRanges: true, + OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { + return v2EnabledWhenLeaderLevel + }, + OverrideTokenDeduction: func(tokens kvflowcontrol.Tokens) kvflowcontrol.Tokens { + // This test sends several puts, with each put potentially + // diverging by a few bytes between runs, in aggregate this + // can accumulate to enough tokens to produce a diff in + // metrics. Round the token deductions to the nearest MiB avoid + // this. + return kvflowcontrol.Tokens( + int64(math.Round(float64(tokens)/float64(1<<20))) * 1 << 20) + }, }, }, - }, - AdmissionControl: &admission.TestingKnobs{ - DisableWorkQueueFastPath: true, - DisableWorkQueueGranting: func() bool { - return disableWorkQueueGranting.Load() + AdmissionControl: &admission.TestingKnobs{ + DisableWorkQueueFastPath: true, + DisableWorkQueueGranting: func() bool { + return disableWorkQueueGranting.Load() + }, }, }, }, - }, - }) - defer tc.Stopper().Stop(ctx) + }) + defer tc.Stopper().Stop(ctx) - k := tc.ScratchRange(t) - tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) + k := tc.ScratchRange(t) + tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) - n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) + n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) - h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) - h.init() - defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, "admission_post_split_merge")) + h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) + h.init(mode) + defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, mode, "admission_post_split_merge")) - desc, err := tc.LookupRange(k) - require.NoError(t, err) + desc, err := tc.LookupRange(k) + require.NoError(t, err) - h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) + h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) - h.log("sending put request to pre-split range") - h.put(ctx, k, 1<<20 /* 1MiB */, admissionpb.NormalPri) - h.put(ctx, k.Next(), 1<<20 /* 1MiB */, admissionpb.NormalPri) - h.log("sent put request to pre-split range") + h.log("sending put request to pre-split range") + h.put(ctx, k, 1<<20 /* 1MiB */, testFlowModeToPri(mode)) + h.put(ctx, k.Next(), 1<<20 /* 1MiB */, testFlowModeToPri(mode)) + h.log("sent put request to pre-split range") - h.comment(` --- Flow token metrics from n1 after issuing a regular 2*1MiB 3x replicated write + h.comment(` +-- Flow token metrics from n1 after issuing a 2*1MiB 3x replicated write -- that are yet to get admitted. We see 2*3*1MiB=6MiB deductions of -- {regular,elastic} tokens with no corresponding returns. The 2*1MiB writes -- happened on what is soon going to be the LHS and RHS of a range being split. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(`-- (Splitting range.)`) - left, right := tc.SplitRangeOrFatal(t, k.Next()) - h.waitForConnectedStreams(ctx, right.RangeID, 3, 0 /* serverIdx */) + h.comment(`-- (Splitting range.)`) + left, right := tc.SplitRangeOrFatal(t, k.Next()) + h.waitForConnectedStreams(ctx, right.RangeID, 3, 0 /* serverIdx */) - h.log("sending 2MiB put request to post-split LHS") - h.put(ctx, k, 2<<20 /* 2MiB */, admissionpb.NormalPri) - h.log("sent 2MiB put request to post-split LHS") + h.log("sending 2MiB put request to post-split LHS") + h.put(ctx, k, 2<<20 /* 2MiB */, testFlowModeToPri(mode)) + h.log("sent 2MiB put request to post-split LHS") - h.log("sending 3MiB put request to post-split RHS") - h.put(ctx, roachpb.Key(right.StartKey), 3<<20 /* 3MiB */, admissionpb.NormalPri) - h.log("sent 3MiB put request to post-split RHS") + h.log("sending 3MiB put request to post-split RHS") + h.put(ctx, roachpb.Key(right.StartKey), 3<<20 /* 3MiB */, testFlowModeToPri(mode)) + h.log("sent 3MiB put request to post-split RHS") - h.comment(` + h.comment(` -- Flow token metrics from n1 after further issuing 2MiB and 3MiB writes to -- post-split LHS and RHS ranges respectively. We should see 15MiB extra tokens -- deducted which comes from (2MiB+3MiB)*3=15MiB. So we stand at -- 6MiB+15MiB=21MiB now. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(`-- Observe the newly split off replica, with its own three streams.`) - h.query(n1, ` + h.comment(`-- Observe the newly split off replica, with its own three streams.`) + h.query(n1, ` SELECT range_id, count(*) AS streams FROM crdb_internal.kv_flow_control_handles_v2 GROUP BY (range_id) ORDER BY streams DESC; `, "range_id", "stream_count") - h.comment(`-- (Merging ranges.)`) - merged := tc.MergeRangesOrFatal(t, left.StartKey.AsRawKey()) + h.comment(`-- (Merging ranges.)`) + merged := tc.MergeRangesOrFatal(t, left.StartKey.AsRawKey()) - h.log("sending 4MiB put request to post-merge range") - h.put(ctx, roachpb.Key(merged.StartKey), 4<<20 /* 4MiB */, admissionpb.NormalPri) - h.log("sent 4MiB put request to post-merged range") + h.log("sending 4MiB put request to post-merge range") + h.put(ctx, roachpb.Key(merged.StartKey), 4<<20 /* 4MiB */, testFlowModeToPri(mode)) + h.log("sent 4MiB put request to post-merged range") - h.comment(` --- Flow token metrics from n1 after issuing 4MiB of regular replicated writes to + h.comment(` +-- Flow token metrics from n1 after issuing 4MiB of replicated writes to -- the post-merged range. We should see 12MiB extra tokens deducted which comes -- from 4MiB*3=12MiB. So we stand at 21MiB+12MiB=33MiB tokens deducted now. The -- RHS of the range is gone now, and the previously 3*3MiB=9MiB of tokens -- deducted for it are released at the subsuming LHS leaseholder. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(`-- Observe only the merged replica with its own three streams.`) - h.query(n1, ` + h.comment(`-- Observe only the merged replica with its own three streams.`) + h.query(n1, ` SELECT range_id, count(*) AS streams FROM crdb_internal.kv_flow_control_handles_v2 GROUP BY (range_id) ORDER BY streams DESC; `, "range_id", "stream_count") - h.comment(`-- (Allow below-raft admission to proceed.)`) - disableWorkQueueGranting.Store(false) - h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) // wait for admission + h.comment(`-- (Allow below-raft admission to proceed.)`) + disableWorkQueueGranting.Store(false) + h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) // wait for admission - h.comment(` + h.comment(` -- Flow token metrics from n1 after work gets admitted. We see all outstanding -- {regular,elastic} tokens returned, including those from: -- - the LHS before the merge, and -- - the LHS and RHS before the original split. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) + }) }) } @@ -2654,92 +2674,97 @@ func TestFlowControlCrashedNodeV2(t *testing.T) { kvflowcontrol.V2EnabledWhenLeaderV1Encoding, kvflowcontrol.V2EnabledWhenLeaderV2Encoding, }, func(t *testing.T, v2EnabledWhenLeaderLevel kvflowcontrol.V2EnabledWhenLeaderLevel) { - ctx := context.Background() - settings := cluster.MakeTestingClusterSettings() - kvserver.ExpirationLeasesOnly.Override(ctx, &settings.SV, true) - tc := testcluster.StartTestCluster(t, 2, base.TestClusterArgs{ - ReplicationMode: base.ReplicationManual, - ServerArgs: base.TestServerArgs{ - Settings: settings, - RaftConfig: base.RaftConfig{ - // Suppress timeout-based elections. This test doesn't want to - // deal with leadership changing hands. - RaftElectionTimeoutTicks: 1000000, - // Reduce the RangeLeaseDuration to speeds up failure detection - // below. - RangeLeaseDuration: time.Second, - }, - Knobs: base.TestingKnobs{ - Store: &kvserver.StoreTestingKnobs{ - FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ - UseOnlyForScratchRanges: true, - OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { - return v2EnabledWhenLeaderLevel + testutils.RunValues(t, "kvadmission.flow_control.mode", []kvflowcontrol.ModeT{ + kvflowcontrol.ApplyToElastic, + kvflowcontrol.ApplyToAll, + }, func(t *testing.T, mode kvflowcontrol.ModeT) { + ctx := context.Background() + settings := cluster.MakeTestingClusterSettings() + kvserver.ExpirationLeasesOnly.Override(ctx, &settings.SV, true) + tc := testcluster.StartTestCluster(t, 2, base.TestClusterArgs{ + ReplicationMode: base.ReplicationManual, + ServerArgs: base.TestServerArgs{ + Settings: settings, + RaftConfig: base.RaftConfig{ + // Suppress timeout-based elections. This test doesn't want to + // deal with leadership changing hands. + RaftElectionTimeoutTicks: 1000000, + // Reduce the RangeLeaseDuration to speeds up failure detection + // below. + RangeLeaseDuration: time.Second, + }, + Knobs: base.TestingKnobs{ + Store: &kvserver.StoreTestingKnobs{ + FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ + UseOnlyForScratchRanges: true, + OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { + return v2EnabledWhenLeaderLevel + }, }, }, - }, - AdmissionControl: &admission.TestingKnobs{ - DisableWorkQueueFastPath: true, - DisableWorkQueueGranting: func() bool { - return true + AdmissionControl: &admission.TestingKnobs{ + DisableWorkQueueFastPath: true, + DisableWorkQueueGranting: func() bool { + return true + }, }, }, }, - }, - }) - defer tc.Stopper().Stop(ctx) + }) + defer tc.Stopper().Stop(ctx) - k := tc.ScratchRange(t) - tc.AddVotersOrFatal(t, k, tc.Targets(1)...) + k := tc.ScratchRange(t) + tc.AddVotersOrFatal(t, k, tc.Targets(1)...) - n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) + n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) - h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) - h.init() - defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, "crashed_node")) + h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) + h.init(mode) + defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, mode, "crashed_node")) - desc, err := tc.LookupRange(k) - require.NoError(t, err) - tc.TransferRangeLeaseOrFatal(t, desc, tc.Target(0)) - h.waitForConnectedStreams(ctx, desc.RangeID, 2, 0 /* serverIdx */) + desc, err := tc.LookupRange(k) + require.NoError(t, err) + tc.TransferRangeLeaseOrFatal(t, desc, tc.Target(0)) + h.waitForConnectedStreams(ctx, desc.RangeID, 2, 0 /* serverIdx */) - h.comment(`-- (Issuing regular 5x1MiB, 2x replicated writes that are not admitted.)`) - h.log("sending put requests") - for i := 0; i < 5; i++ { - h.put(ctx, k, 1<<20 /* 1MiB */, admissionpb.NormalPri) - } - h.log("sent put requests") + h.comment(`-- (Issuing 5x1MiB, 2x replicated writes that are not admitted.)`) + h.log("sending put requests") + for i := 0; i < 5; i++ { + h.put(ctx, k, 1<<20 /* 1MiB */, testFlowModeToPri(mode)) + } + h.log("sent put requests") - h.comment(` --- Flow token metrics from n1 after issuing 5 regular 1MiB 2x replicated writes + h.comment(` +-- Flow token metrics from n1 after issuing 5 1MiB 2x replicated writes -- that are yet to get admitted. We see 5*1MiB*2=10MiB deductions of -- {regular,elastic} tokens with no corresponding returns. `) - h.query(n1, v2FlowTokensQueryStr) - h.comment(`-- Observe the per-stream tracked tokens on n1, before n2 is crashed.`) - h.query(n1, ` + h.query(n1, v2FlowTokensQueryStr) + h.comment(`-- Observe the per-stream tracked tokens on n1, before n2 is crashed.`) + h.query(n1, ` SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) FROM crdb_internal.kv_flow_control_handles_v2 `, "range_id", "store_id", "total_tracked_tokens") - h.comment(`-- (Crashing n2)`) - tc.StopServer(1) - h.waitForConnectedStreams(ctx, desc.RangeID, 1, 0 /* serverIdx */) + h.comment(`-- (Crashing n2)`) + tc.StopServer(1) + h.waitForConnectedStreams(ctx, desc.RangeID, 1, 0 /* serverIdx */) - h.comment(` + h.comment(` -- Observe the per-stream tracked tokens on n1, after n2 crashed. We're no -- longer tracking the 5MiB held by n2. `) - h.query(n1, ` + h.query(n1, ` SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) FROM crdb_internal.kv_flow_control_handles_v2 `, "range_id", "store_id", "total_tracked_tokens") - h.comment(` + h.comment(` -- Flow token metrics from n1 after n2 crashed. Observe that we've returned the -- 5MiB previously held by n2. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) + }) }) } @@ -2755,245 +2780,250 @@ func TestFlowControlRaftSnapshotV2(t *testing.T) { kvflowcontrol.V2EnabledWhenLeaderV1Encoding, kvflowcontrol.V2EnabledWhenLeaderV2Encoding, }, func(t *testing.T, v2EnabledWhenLeaderLevel kvflowcontrol.V2EnabledWhenLeaderLevel) { - stickyServerArgs := make(map[int]base.TestServerArgs) - var disableWorkQueueGranting atomic.Bool - disableWorkQueueGranting.Store(true) - var bypassReplicaUnreachable atomic.Bool - bypassReplicaUnreachable.Store(false) - ctx := context.Background() - settings := cluster.MakeTestingClusterSettings() - for i := 0; i < numServers; i++ { - stickyServerArgs[i] = base.TestServerArgs{ - Settings: settings, - StoreSpecs: []base.StoreSpec{ - { - InMemory: true, - StickyVFSID: strconv.FormatInt(int64(i), 10), + testutils.RunValues(t, "kvadmission.flow_control.mode", []kvflowcontrol.ModeT{ + kvflowcontrol.ApplyToElastic, + kvflowcontrol.ApplyToAll, + }, func(t *testing.T, mode kvflowcontrol.ModeT) { + stickyServerArgs := make(map[int]base.TestServerArgs) + var disableWorkQueueGranting atomic.Bool + disableWorkQueueGranting.Store(true) + var bypassReplicaUnreachable atomic.Bool + bypassReplicaUnreachable.Store(false) + ctx := context.Background() + settings := cluster.MakeTestingClusterSettings() + for i := 0; i < numServers; i++ { + stickyServerArgs[i] = base.TestServerArgs{ + Settings: settings, + StoreSpecs: []base.StoreSpec{ + { + InMemory: true, + StickyVFSID: strconv.FormatInt(int64(i), 10), + }, }, - }, - RaftConfig: base.RaftConfig{ - // Suppress timeout-based elections. This test doesn't want to - // deal with leadership changing hands. - RaftElectionTimeoutTicks: 1000000, - }, - Knobs: base.TestingKnobs{ - Server: &server.TestingKnobs{ - StickyVFSRegistry: fs.NewStickyRegistry(), + RaftConfig: base.RaftConfig{ + // Suppress timeout-based elections. This test doesn't want to + // deal with leadership changing hands. + RaftElectionTimeoutTicks: 1000000, }, - Store: &kvserver.StoreTestingKnobs{ - RaftReportUnreachableBypass: func(_ roachpb.ReplicaID) bool { - // This test is going to crash nodes, then truncate the raft log - // and assert that tokens are returned upon an replica entering - // StateSnapshot. To avoid the stopped replicas entering - // StateProbe returning tokens, we disable reporting a replica - // as unreachable while nodes are down. - return bypassReplicaUnreachable.Load() + Knobs: base.TestingKnobs{ + Server: &server.TestingKnobs{ + StickyVFSRegistry: fs.NewStickyRegistry(), }, - FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ - UseOnlyForScratchRanges: true, - OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { - return v2EnabledWhenLeaderLevel + Store: &kvserver.StoreTestingKnobs{ + RaftReportUnreachableBypass: func(_ roachpb.ReplicaID) bool { + // This test is going to crash nodes, then truncate the raft log + // and assert that tokens are returned upon an replica entering + // StateSnapshot. To avoid the stopped replicas entering + // StateProbe returning tokens, we disable reporting a replica + // as unreachable while nodes are down. + return bypassReplicaUnreachable.Load() }, - OverrideTokenDeduction: func(_ kvflowcontrol.Tokens) kvflowcontrol.Tokens { - // This test makes use of (small) increment - // requests, but wants to see large token - // deductions/returns. - return kvflowcontrol.Tokens(1 << 20 /* 1MiB */) + FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ + UseOnlyForScratchRanges: true, + OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { + return v2EnabledWhenLeaderLevel + }, + OverrideTokenDeduction: func(_ kvflowcontrol.Tokens) kvflowcontrol.Tokens { + // This test makes use of (small) increment + // requests, but wants to see large token + // deductions/returns. + return kvflowcontrol.Tokens(1 << 20 /* 1MiB */) + }, }, }, - }, - AdmissionControl: &admission.TestingKnobs{ - DisableWorkQueueFastPath: true, - DisableWorkQueueGranting: func() bool { - return disableWorkQueueGranting.Load() + AdmissionControl: &admission.TestingKnobs{ + DisableWorkQueueFastPath: true, + DisableWorkQueueGranting: func() bool { + return disableWorkQueueGranting.Load() + }, }, - }, - RaftTransport: &kvserver.RaftTransportTestingKnobs{ - OverrideIdleTimeout: func() time.Duration { - // Effectively disable token returns due to underlying - // raft transport streams disconnecting due to - // inactivity. - return time.Hour + RaftTransport: &kvserver.RaftTransportTestingKnobs{ + OverrideIdleTimeout: func() time.Duration { + // Effectively disable token returns due to underlying + // raft transport streams disconnecting due to + // inactivity. + return time.Hour + }, }, }, - }, + } } - } - tc := testcluster.StartTestCluster(t, numServers, - base.TestClusterArgs{ - ReplicationMode: base.ReplicationManual, - ServerArgsPerNode: stickyServerArgs, - }) - defer tc.Stopper().Stop(ctx) + tc := testcluster.StartTestCluster(t, numServers, + base.TestClusterArgs{ + ReplicationMode: base.ReplicationManual, + ServerArgsPerNode: stickyServerArgs, + }) + defer tc.Stopper().Stop(ctx) - n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) - h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) - h.init() - defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, "raft_snapshot")) + n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) + h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) + h.init(mode) + defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, mode, "raft_snapshot")) - store := tc.GetFirstStoreFromServer(t, 0) + store := tc.GetFirstStoreFromServer(t, 0) - incA := int64(5) - incB := int64(7) - incAB := incA + incB + incA := int64(5) + incB := int64(7) + incAB := incA + incB - k := tc.ScratchRange(t) - tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) + k := tc.ScratchRange(t) + tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) - tc.AddVotersOrFatal(t, k, tc.Targets(3, 4)...) - repl := store.LookupReplica(roachpb.RKey(k)) - require.NotNil(t, repl) - h.waitForConnectedStreams(ctx, repl.RangeID, 5, 0 /* serverIdx */) - - // Set up a key to replicate across the cluster. We're going to modify this - // key and truncate the raft logs from that command after killing one of the - // nodes to check that it gets the new value after it comes up. - incArgs := incrementArgs(k, incA) - if _, err := kv.SendWrappedWithAdmission(ctx, tc.Server(0).DB().NonTransactionalSender(), kvpb.Header{}, kvpb.AdmissionHeader{ - Priority: int32(admissionpb.HighPri), - Source: kvpb.AdmissionHeader_FROM_SQL, - }, incArgs); err != nil { - t.Fatal(err) - } + tc.AddVotersOrFatal(t, k, tc.Targets(3, 4)...) + repl := store.LookupReplica(roachpb.RKey(k)) + require.NotNil(t, repl) + h.waitForConnectedStreams(ctx, repl.RangeID, 5, 0 /* serverIdx */) + + // Set up a key to replicate across the cluster. We're going to modify this + // key and truncate the raft logs from that command after killing one of the + // nodes to check that it gets the new value after it comes up. + incArgs := incrementArgs(k, incA) + if _, err := kv.SendWrappedWithAdmission(ctx, tc.Server(0).DB().NonTransactionalSender(), kvpb.Header{}, kvpb.AdmissionHeader{ + Priority: int32(testFlowModeToPri(mode)), + Source: kvpb.AdmissionHeader_FROM_SQL, + }, incArgs); err != nil { + t.Fatal(err) + } - h.comment(` --- Flow token metrics from n1 after issuing 1 regular 1MiB 5x replicated write + h.comment(` +-- Flow token metrics from n1 after issuing 1 1MiB 5x replicated write -- that's not admitted. Since this test is ignoring crashed nodes for token --- deduction purposes, we see a deduction of 5MiB {regular,elastic} tokens. +-- deduction purposes, we see a deduction of 5MiB tokens. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(` + h.comment(` -- Observe the total tracked tokens per-stream on n1. 1MiB is tracked for n1-n5. `) - h.query(n1, ` + h.query(n1, ` SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) FROM crdb_internal.kv_flow_control_handles_v2 `, "range_id", "store_id", "total_tracked_tokens") - tc.WaitForValues(t, k, []int64{incA, incA, incA, incA, incA}) + tc.WaitForValues(t, k, []int64{incA, incA, incA, incA, incA}) - h.comment(` + h.comment(` -- (Killing n2 and n3, but preventing their tokens from being returned + -- artificially allowing tokens to get deducted.)`) - // Kill stores 1 + 2, increment the key on the other stores and truncate - // their logs to make sure that when store 1 + 2 comes back up they will - // require a snapshot from Raft. - // - // Also prevent replicas on the killed nodes from being marked as - // unreachable, in order to prevent them from returning tokens via - // entering StateProbe, before we're able to truncate the log and assert - // on the snapshot behavior. - bypassReplicaUnreachable.Store(true) - tc.StopServer(1) - tc.StopServer(2) + // Kill stores 1 + 2, increment the key on the other stores and truncate + // their logs to make sure that when store 1 + 2 comes back up they will + // require a snapshot from Raft. + // + // Also prevent replicas on the killed nodes from being marked as + // unreachable, in order to prevent them from returning tokens via + // entering StateProbe, before we're able to truncate the log and assert + // on the snapshot behavior. + bypassReplicaUnreachable.Store(true) + tc.StopServer(1) + tc.StopServer(2) - h.comment(` + h.comment(` -- Observe the total tracked tokens per-stream on n1. 1MiB is (still) tracked -- for n1-n5, because they are not in StateSnapshot yet and have likely been -- in StateProbe for less than the close timer. `) - h.query(n1, ` + h.query(n1, ` SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) FROM crdb_internal.kv_flow_control_handles_v2 `, "range_id", "store_id", "total_tracked_tokens") - h.comment(` + h.comment(` -- (Issuing another 1MiB of 5x replicated writes while n2 and n3 are down and -- below-raft admission is paused.) `) - incArgs = incrementArgs(k, incB) - if _, err := kv.SendWrappedWithAdmission(ctx, tc.Server(0).DB().NonTransactionalSender(), kvpb.Header{}, kvpb.AdmissionHeader{ - Priority: int32(admissionpb.HighPri), - Source: kvpb.AdmissionHeader_FROM_SQL, - }, incArgs); err != nil { - t.Fatal(err) - } + incArgs = incrementArgs(k, incB) + if _, err := kv.SendWrappedWithAdmission(ctx, tc.Server(0).DB().NonTransactionalSender(), kvpb.Header{}, kvpb.AdmissionHeader{ + Priority: int32(testFlowModeToPri(mode)), + Source: kvpb.AdmissionHeader_FROM_SQL, + }, incArgs); err != nil { + t.Fatal(err) + } - h.comment(` --- Flow token metrics from n1 after issuing 1 regular 1MiB 5x replicated write + h.comment(` +-- Flow token metrics from n1 after issuing 1 1MiB 5x replicated write -- that's not admitted. We'll have deducted another 5*1MiB=5MiB worth of tokens. `) - h.query(n1, v2FlowTokensQueryStr) - h.comment(` + h.query(n1, v2FlowTokensQueryStr) + h.comment(` -- Observe the total tracked tokens per-stream on n1. 2MiB is tracked for n1-n5; -- see last comment for an explanation why we're still deducting for n2, n3. `) - h.query(n1, ` + h.query(n1, ` SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) FROM crdb_internal.kv_flow_control_handles_v2 `, "range_id", "store_id", "total_tracked_tokens") - tc.WaitForValues(t, k, []int64{incAB, 0 /* stopped */, 0 /* stopped */, incAB, incAB}) + tc.WaitForValues(t, k, []int64{incAB, 0 /* stopped */, 0 /* stopped */, incAB, incAB}) - index := repl.GetLastIndex() - h.comment(`-- (Truncating raft log.)`) + index := repl.GetLastIndex() + h.comment(`-- (Truncating raft log.)`) - // Truncate the log at index+1 (log entries < N are removed, so this - // includes the increment). - truncArgs := truncateLogArgs(index+1, repl.GetRangeID()) - if _, err := kv.SendWrappedWithAdmission(ctx, tc.Server(0).DB().NonTransactionalSender(), kvpb.Header{}, kvpb.AdmissionHeader{ - Priority: int32(admissionpb.HighPri), - Source: kvpb.AdmissionHeader_FROM_SQL, - }, truncArgs); err != nil { - t.Fatal(err) - } + // Truncate the log at index+1 (log entries < N are removed, so this + // includes the increment). + truncArgs := truncateLogArgs(index+1, repl.GetRangeID()) + if _, err := kv.SendWrappedWithAdmission(ctx, tc.Server(0).DB().NonTransactionalSender(), kvpb.Header{}, kvpb.AdmissionHeader{ + Priority: int32(testFlowModeToPri(mode)), + Source: kvpb.AdmissionHeader_FROM_SQL, + }, truncArgs); err != nil { + t.Fatal(err) + } - h.comment(`-- (Restarting n2 and n3.)`) - require.NoError(t, tc.RestartServer(1)) - require.NoError(t, tc.RestartServer(2)) - bypassReplicaUnreachable.Store(false) + h.comment(`-- (Restarting n2 and n3.)`) + require.NoError(t, tc.RestartServer(1)) + require.NoError(t, tc.RestartServer(2)) + bypassReplicaUnreachable.Store(false) - tc.WaitForValues(t, k, []int64{incAB, incAB, incAB, incAB, incAB}) + tc.WaitForValues(t, k, []int64{incAB, incAB, incAB, incAB, incAB}) - h.comment(` + h.comment(` -- Flow token metrics from n1 after restarting n2 and n3. We've returned the -- 2MiB previously held by those nodes (2MiB each). We're reacting to it's raft -- progress state, noting that since we've truncated our log, we need to catch -- it up via snapshot. So we release all held tokens. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(` + h.comment(` -- Observe the total tracked tokens per-stream on n1. There's nothing tracked -- for n2 and n3 anymore. `) - h.query(n1, ` + h.query(n1, ` SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) FROM crdb_internal.kv_flow_control_handles_v2 WHERE total_tracked_tokens > 0 `, "range_id", "store_id", "total_tracked_tokens") - h.waitForConnectedStreams(ctx, repl.RangeID, 5, 0 /* serverIdx */) - h.comment(`-- (Allow below-raft admission to proceed.)`) - disableWorkQueueGranting.Store(false) + h.waitForConnectedStreams(ctx, repl.RangeID, 5, 0 /* serverIdx */) + h.comment(`-- (Allow below-raft admission to proceed.)`) + disableWorkQueueGranting.Store(false) - h.waitForAllTokensReturned(ctx, 5, 0 /* serverIdx */) + h.waitForAllTokensReturned(ctx, 5, 0 /* serverIdx */) - h.comment(` + h.comment(` -- Flow token metrics from n1 after work gets admitted. We see the remaining --- 6MiB of {regular,elastic} tokens returned. +-- 6MiB of tokens returned. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(` + h.comment(` -- Observe the total tracked tokens per-stream on n1; there should be nothing. `) - h.query(n1, ` + h.query(n1, ` SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) FROM crdb_internal.kv_flow_control_handles_v2 `, "range_id", "store_id", "total_tracked_tokens") - h.comment(`-- Another view of tokens, using /inspectz-backed vtables.`) - h.query(n1, ` + h.comment(`-- Another view of tokens, using /inspectz-backed vtables.`) + h.query(n1, ` SELECT store_id, crdb_internal.humanize_bytes(available_eval_regular_tokens), crdb_internal.humanize_bytes(available_eval_elastic_tokens) FROM crdb_internal.kv_flow_controller_v2 ORDER BY store_id ASC; `, "range_id", "eval_regular_available", "eval_elastic_available") + }) }) } @@ -3007,120 +3037,125 @@ func TestFlowControlRaftMembershipV2(t *testing.T) { kvflowcontrol.V2EnabledWhenLeaderV1Encoding, kvflowcontrol.V2EnabledWhenLeaderV2Encoding, }, func(t *testing.T, v2EnabledWhenLeaderLevel kvflowcontrol.V2EnabledWhenLeaderLevel) { - ctx := context.Background() - settings := cluster.MakeTestingClusterSettings() - var disableWorkQueueGranting atomic.Bool - disableWorkQueueGranting.Store(true) - tc := testcluster.StartTestCluster(t, 5, base.TestClusterArgs{ - ReplicationMode: base.ReplicationManual, - ServerArgs: base.TestServerArgs{ - Settings: settings, - Knobs: base.TestingKnobs{ - Store: &kvserver.StoreTestingKnobs{ - FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ - UseOnlyForScratchRanges: true, - OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { - return v2EnabledWhenLeaderLevel + testutils.RunValues(t, "kvadmission.flow_control.mode", []kvflowcontrol.ModeT{ + kvflowcontrol.ApplyToElastic, + kvflowcontrol.ApplyToAll, + }, func(t *testing.T, mode kvflowcontrol.ModeT) { + ctx := context.Background() + settings := cluster.MakeTestingClusterSettings() + var disableWorkQueueGranting atomic.Bool + disableWorkQueueGranting.Store(true) + tc := testcluster.StartTestCluster(t, 5, base.TestClusterArgs{ + ReplicationMode: base.ReplicationManual, + ServerArgs: base.TestServerArgs{ + Settings: settings, + Knobs: base.TestingKnobs{ + Store: &kvserver.StoreTestingKnobs{ + FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ + UseOnlyForScratchRanges: true, + OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { + return v2EnabledWhenLeaderLevel + }, }, }, - }, - AdmissionControl: &admission.TestingKnobs{ - DisableWorkQueueFastPath: true, - DisableWorkQueueGranting: func() bool { - return disableWorkQueueGranting.Load() + AdmissionControl: &admission.TestingKnobs{ + DisableWorkQueueFastPath: true, + DisableWorkQueueGranting: func() bool { + return disableWorkQueueGranting.Load() + }, }, }, }, - }, - }) - defer tc.Stopper().Stop(ctx) - - k := tc.ScratchRange(t) - tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) + }) + defer tc.Stopper().Stop(ctx) - n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) + k := tc.ScratchRange(t) + tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) - h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) - h.init() - defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, "raft_membership")) + n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) - desc, err := tc.LookupRange(k) - require.NoError(t, err) - h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) + h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) + h.init(mode) + defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, mode, "raft_membership")) - h.comment(`-- (Issuing 1x1MiB, 3x replicated write that's not admitted.)`) - h.put(ctx, k, 1<<20 /* 1MiB */, admissionpb.NormalPri) + desc, err := tc.LookupRange(k) + require.NoError(t, err) + h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) - h.comment(` --- Flow token metrics from n1 after issuing 1x1MiB regular 3x replicated write --- that's not admitted. We see 1*1MiB*3=3MiB deductions of regular tokens with + h.comment(`-- (Issuing 1x1MiB, 3x replicated write that's not admitted.)`) + h.put(ctx, k, 1<<20 /* 1MiB */, testFlowModeToPri(mode)) + + h.comment(` +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with -- no corresponding returns. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(`-- (Adding a voting replica on n4.)`) - tc.AddVotersOrFatal(t, k, tc.Target(3)) - h.waitForConnectedStreams(ctx, desc.RangeID, 4, 0 /* serverIdx */) + h.comment(`-- (Adding a voting replica on n4.)`) + tc.AddVotersOrFatal(t, k, tc.Target(3)) + h.waitForConnectedStreams(ctx, desc.RangeID, 4, 0 /* serverIdx */) - h.comment(` + h.comment(` -- Observe the total tracked tokens per-stream on n1. s1-s3 should have 1MiB -- tracked each, and s4 should have none.`) - h.query(n1, ` + h.query(n1, ` SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) FROM crdb_internal.kv_flow_control_handles_v2 `, "range_id", "store_id", "total_tracked_tokens") - h.comment(`-- (Issuing 1x1MiB, 4x replicated write that's not admitted.)`) - h.put(ctx, k, 1<<20 /* 1MiB */, admissionpb.NormalPri) + h.comment(`-- (Issuing 1x1MiB, 4x replicated write that's not admitted.)`) + h.put(ctx, k, 1<<20 /* 1MiB */, testFlowModeToPri(mode)) - h.comment(` + h.comment(` -- Observe the individual tracked tokens per-stream on the scratch range. s1-s3 -- should have 2MiB tracked (they've observed 2x1MiB writes), s4 should have -- 1MiB. `) - h.query(n1, ` + h.query(n1, ` SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) FROM crdb_internal.kv_flow_control_handles_v2 `, "range_id", "store_id", "total_tracked_tokens") - h.comment(`-- (Removing voting replica from n3.)`) - tc.RemoveVotersOrFatal(t, k, tc.Target(2)) - h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) + h.comment(`-- (Removing voting replica from n3.)`) + tc.RemoveVotersOrFatal(t, k, tc.Target(2)) + h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) - h.comment(`-- (Adding non-voting replica to n5.)`) - tc.AddNonVotersOrFatal(t, k, tc.Target(4)) - h.waitForConnectedStreams(ctx, desc.RangeID, 4, 0 /* serverIdx */) + h.comment(`-- (Adding non-voting replica to n5.)`) + tc.AddNonVotersOrFatal(t, k, tc.Target(4)) + h.waitForConnectedStreams(ctx, desc.RangeID, 4, 0 /* serverIdx */) - h.comment(`-- (Issuing 1x1MiB, 4x replicated write (w/ one non-voter) that's not admitted.`) - h.put(ctx, k, 1<<20 /* 1MiB */, admissionpb.NormalPri) + h.comment(`-- (Issuing 1x1MiB, 4x replicated write (w/ one non-voter) that's not admitted.`) + h.put(ctx, k, 1<<20 /* 1MiB */, testFlowModeToPri(mode)) - h.comment(` + h.comment(` -- Observe the individual tracked tokens per-stream on the scratch range. s1-s2 -- should have 3MiB tracked (they've observed 3x1MiB writes), there should be -- no s3 since it was removed, s4 and s5 should have 2MiB and 1MiB -- respectively. `) - h.query(n1, ` + h.query(n1, ` SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) FROM crdb_internal.kv_flow_control_handles_v2 `, "range_id", "store_id", "total_tracked_tokens") - h.comment(`-- (Allow below-raft admission to proceed.)`) - disableWorkQueueGranting.Store(false) - h.waitForAllTokensReturned(ctx, 5, 0 /* serverIdx */) + h.comment(`-- (Allow below-raft admission to proceed.)`) + disableWorkQueueGranting.Store(false) + h.waitForAllTokensReturned(ctx, 5, 0 /* serverIdx */) - h.comment(`-- Observe that there no tracked tokens across s1,s2,s4,s5.`) - h.query(n1, ` + h.comment(`-- Observe that there no tracked tokens across s1,s2,s4,s5.`) + h.query(n1, ` SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) FROM crdb_internal.kv_flow_control_handles_v2 `, "range_id", "store_id", "total_tracked_tokens") - h.comment(` + h.comment(` -- Flow token metrics from n1 after work gets admitted. All {regular,elastic} -- tokens deducted are returned, including from when s3 was removed as a raft -- member. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) + }) }) } @@ -3134,122 +3169,127 @@ func TestFlowControlRaftMembershipRemoveSelfV2(t *testing.T) { kvflowcontrol.V2EnabledWhenLeaderV1Encoding, kvflowcontrol.V2EnabledWhenLeaderV2Encoding, }, func(t *testing.T, v2EnabledWhenLeaderLevel kvflowcontrol.V2EnabledWhenLeaderLevel) { - testutils.RunTrueAndFalse(t, "transfer-lease-first", func(t *testing.T, transferLeaseFirst bool) { - ctx := context.Background() - settings := cluster.MakeTestingClusterSettings() - var disableWorkQueueGranting atomic.Bool - disableWorkQueueGranting.Store(true) - tc := testcluster.StartTestCluster(t, 4, base.TestClusterArgs{ - ReplicationMode: base.ReplicationManual, - ServerArgs: base.TestServerArgs{ - Settings: settings, - Knobs: base.TestingKnobs{ - Store: &kvserver.StoreTestingKnobs{ - FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ - UseOnlyForScratchRanges: true, - OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { - return v2EnabledWhenLeaderLevel + testutils.RunValues(t, "kvadmission.flow_control.mode", []kvflowcontrol.ModeT{ + kvflowcontrol.ApplyToElastic, + kvflowcontrol.ApplyToAll, + }, func(t *testing.T, mode kvflowcontrol.ModeT) { + testutils.RunTrueAndFalse(t, "transfer-lease-first", func(t *testing.T, transferLeaseFirst bool) { + ctx := context.Background() + settings := cluster.MakeTestingClusterSettings() + var disableWorkQueueGranting atomic.Bool + disableWorkQueueGranting.Store(true) + tc := testcluster.StartTestCluster(t, 4, base.TestClusterArgs{ + ReplicationMode: base.ReplicationManual, + ServerArgs: base.TestServerArgs{ + Settings: settings, + Knobs: base.TestingKnobs{ + Store: &kvserver.StoreTestingKnobs{ + FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ + UseOnlyForScratchRanges: true, + OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { + return v2EnabledWhenLeaderLevel + }, }, }, - }, - AdmissionControl: &admission.TestingKnobs{ - DisableWorkQueueFastPath: true, - DisableWorkQueueGranting: func() bool { - return disableWorkQueueGranting.Load() + AdmissionControl: &admission.TestingKnobs{ + DisableWorkQueueFastPath: true, + DisableWorkQueueGranting: func() bool { + return disableWorkQueueGranting.Load() + }, }, }, }, - }, - }) - defer tc.Stopper().Stop(ctx) + }) + defer tc.Stopper().Stop(ctx) - k := tc.ScratchRange(t) - tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) + k := tc.ScratchRange(t) + tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) - n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) - n4 := sqlutils.MakeSQLRunner(tc.ServerConn(3)) + n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) + n4 := sqlutils.MakeSQLRunner(tc.ServerConn(3)) - h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) - h.init() - // Note this test behaves identically independent of we transfer the lease - // first. - defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, "raft_membership_remove_self")) + h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) + h.init(mode) + // Note this test behaves identically independent of we transfer the lease + // first. + defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, mode, "raft_membership_remove_self")) - desc, err := tc.LookupRange(k) - require.NoError(t, err) + desc, err := tc.LookupRange(k) + require.NoError(t, err) - // Make sure the lease is on n1 and that we're triply connected. - tc.TransferRangeLeaseOrFatal(t, desc, tc.Target(0)) - h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) + // Make sure the lease is on n1 and that we're triply connected. + tc.TransferRangeLeaseOrFatal(t, desc, tc.Target(0)) + h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) - h.comment(`-- (Issuing 1x1MiB, 3x replicated write that's not admitted.)`) - h.put(ctx, k, 1<<20 /* 1MiB */, admissionpb.NormalPri) + h.comment(`-- (Issuing 1x1MiB, 3x replicated write that's not admitted.)`) + h.put(ctx, k, 1<<20 /* 1MiB */, testFlowModeToPri(mode)) - h.comment(` --- Flow token metrics from n1 after issuing 1x1MiB regular 3x replicated write --- that's not admitted. We see 1*1MiB*3=3MiB deductions of regular tokens with + h.comment(` +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with -- no corresponding returns. `) - h.query(n1, v2FlowTokensQueryStr) - - h.comment(`-- (Replacing current raft leader on n1 in raft group with new n4 replica.)`) - testutils.SucceedsSoon(t, func() error { - // Relocate range from n1 -> n4. - if err := tc.Servers[2].DB(). - AdminRelocateRange( - context.Background(), desc.StartKey.AsRawKey(), - tc.Targets(3, 2, 1), nil, transferLeaseFirst); err != nil { - return err - } - leaseHolder, err := tc.FindRangeLeaseHolder(desc, nil) - if err != nil { - return err - } - if !leaseHolder.Equal(tc.Target(3)) { - return errors.Errorf("expected leaseholder to be n4, found %v", leaseHolder) - } - return nil - }) - h.waitForAllTokensReturned(ctx, 4, 0 /* serverIdx */) - h.waitForConnectedStreams(ctx, desc.RangeID, 3, 3 /* serverIdx */) - - h.comment(` + h.query(n1, v2FlowTokensQueryStr) + + h.comment(`-- (Replacing current raft leader on n1 in raft group with new n4 replica.)`) + testutils.SucceedsSoon(t, func() error { + // Relocate range from n1 -> n4. + if err := tc.Servers[2].DB(). + AdminRelocateRange( + context.Background(), desc.StartKey.AsRawKey(), + tc.Targets(3, 2, 1), nil, transferLeaseFirst); err != nil { + return err + } + leaseHolder, err := tc.FindRangeLeaseHolder(desc, nil) + if err != nil { + return err + } + if !leaseHolder.Equal(tc.Target(3)) { + return errors.Errorf("expected leaseholder to be n4, found %v", leaseHolder) + } + return nil + }) + h.waitForAllTokensReturned(ctx, 4, 0 /* serverIdx */) + h.waitForConnectedStreams(ctx, desc.RangeID, 3, 3 /* serverIdx */) + + h.comment(` -- Flow token metrics from n1 after raft leader removed itself from raft group. -- All {regular,elastic} tokens deducted are returned. Note that the available -- tokens increases, as n1 has seen 4 replication streams, s1,s2,s3,s4. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(` + h.comment(` -- n1 should have no connected streams now after transferring the lease to n4. -- While, n4 should have 3 connected streams to s2,s3,s4. Query the stream count -- on n1, then on n4. -- n1 connected v2 streams: `) - h.query(n1, ` + h.query(n1, ` SELECT range_id, count(*) AS streams FROM crdb_internal.kv_flow_control_handles_v2 GROUP BY (range_id) ORDER BY streams DESC; `, "range_id", "stream_count") - h.comment(`-- n4 connected v2 streams:`) - h.query(n4, ` + h.comment(`-- n4 connected v2 streams:`) + h.query(n4, ` SELECT range_id, count(*) AS streams FROM crdb_internal.kv_flow_control_handles_v2 GROUP BY (range_id) ORDER BY streams DESC; `, "range_id", "stream_count") - h.comment(`-- (Allow below-raft admission to proceed.)`) - disableWorkQueueGranting.Store(false) - h.waitForAllTokensReturned(ctx, 4, 0 /* serverIdx */) + h.comment(`-- (Allow below-raft admission to proceed.)`) + disableWorkQueueGranting.Store(false) + h.waitForAllTokensReturned(ctx, 4, 0 /* serverIdx */) - h.comment(` + h.comment(` -- Flow token metrics from n1 after work gets admitted. Tokens were already -- returned earlier, so there's no change. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) + }) }) }) } @@ -3265,76 +3305,81 @@ func TestFlowControlClassPrioritizationV2(t *testing.T) { kvflowcontrol.V2EnabledWhenLeaderV1Encoding, kvflowcontrol.V2EnabledWhenLeaderV2Encoding, }, func(t *testing.T, v2EnabledWhenLeaderLevel kvflowcontrol.V2EnabledWhenLeaderLevel) { - ctx := context.Background() - var disableWorkQueueGranting atomic.Bool - disableWorkQueueGranting.Store(true) - settings := cluster.MakeTestingClusterSettings() - tc := testcluster.StartTestCluster(t, 5, base.TestClusterArgs{ - ReplicationMode: base.ReplicationManual, - ServerArgs: base.TestServerArgs{ - Settings: settings, - Knobs: base.TestingKnobs{ - Store: &kvserver.StoreTestingKnobs{ - FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ - UseOnlyForScratchRanges: true, - OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { - return v2EnabledWhenLeaderLevel + testutils.RunValues(t, "kvadmission.flow_control.mode", []kvflowcontrol.ModeT{ + kvflowcontrol.ApplyToElastic, + kvflowcontrol.ApplyToAll, + }, func(t *testing.T, mode kvflowcontrol.ModeT) { + ctx := context.Background() + var disableWorkQueueGranting atomic.Bool + disableWorkQueueGranting.Store(true) + settings := cluster.MakeTestingClusterSettings() + tc := testcluster.StartTestCluster(t, 5, base.TestClusterArgs{ + ReplicationMode: base.ReplicationManual, + ServerArgs: base.TestServerArgs{ + Settings: settings, + Knobs: base.TestingKnobs{ + Store: &kvserver.StoreTestingKnobs{ + FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ + UseOnlyForScratchRanges: true, + OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { + return v2EnabledWhenLeaderLevel + }, }, }, - }, - AdmissionControl: &admission.TestingKnobs{ - DisableWorkQueueFastPath: true, - DisableWorkQueueGranting: func() bool { - return disableWorkQueueGranting.Load() + AdmissionControl: &admission.TestingKnobs{ + DisableWorkQueueFastPath: true, + DisableWorkQueueGranting: func() bool { + return disableWorkQueueGranting.Load() + }, }, }, }, - }, - }) - defer tc.Stopper().Stop(ctx) + }) + defer tc.Stopper().Stop(ctx) - k := tc.ScratchRange(t) - tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) + k := tc.ScratchRange(t) + tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) - n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) + n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) - h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) - h.init() - defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, "class_prioritization")) + h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) + h.init(mode) + defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, mode, "class_prioritization")) - desc, err := tc.LookupRange(k) - require.NoError(t, err) - h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) + desc, err := tc.LookupRange(k) + require.NoError(t, err) + h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) - h.comment(`-- (Issuing 1x1MiB, 3x replicated elastic write that's not admitted.)`) - h.put(ctx, k, 1<<20 /* 1MiB */, admissionpb.BulkNormalPri) + h.comment(`-- (Issuing 1x1MiB, 3x replicated elastic write that's not admitted.)`) + h.put(ctx, k, 1<<20 /* 1MiB */, testFlowModeToPri(mode)) - h.comment(` + h.comment(` -- Flow token metrics from n1 after issuing 1x1MiB elastic 3x replicated write -- that's not admitted. We see 1*1MiB*3=3MiB deductions of elastic tokens with -- no corresponding returns. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(`-- (Issuing 1x1MiB, 3x replicated regular write that's not admitted.)`) - h.put(ctx, k, 1<<20 /* 1MiB */, admissionpb.NormalPri) + h.comment(`-- (Issuing 1x1MiB, 3x replicated write that's not admitted.)`) + h.put(ctx, k, 1<<20 /* 1MiB */, testFlowModeToPri(mode)) - h.comment(` --- Flow token metrics from n1 after issuing 1x1MiB regular 3x replicated write + h.comment(` +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write -- that's not admitted. We see 1*1MiB*3=3MiB deductions of {regular,elastic} -- tokens with no corresponding returns. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(`-- (Allow below-raft admission to proceed.)`) - disableWorkQueueGranting.Store(false) - h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) + h.comment(`-- (Allow below-raft admission to proceed.)`) + disableWorkQueueGranting.Store(false) + h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) - h.comment(` + h.comment(` -- Flow token metrics from n1 after work gets admitted. All {regular,elastic} -- tokens deducted are returned. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) + }) }) } @@ -3350,121 +3395,126 @@ func TestFlowControlUnquiescedRangeV2(t *testing.T) { kvflowcontrol.V2EnabledWhenLeaderV1Encoding, kvflowcontrol.V2EnabledWhenLeaderV2Encoding, }, func(t *testing.T, v2EnabledWhenLeaderLevel kvflowcontrol.V2EnabledWhenLeaderLevel) { - ctx := context.Background() - var disableWorkQueueGranting atomic.Bool - var disablePiggybackTokenDispatch atomic.Bool - disableWorkQueueGranting.Store(true) - disablePiggybackTokenDispatch.Store(true) + testutils.RunValues(t, "kvadmission.flow_control.mode", []kvflowcontrol.ModeT{ + kvflowcontrol.ApplyToElastic, + kvflowcontrol.ApplyToAll, + }, func(t *testing.T, mode kvflowcontrol.ModeT) { + ctx := context.Background() + var disableWorkQueueGranting atomic.Bool + var disablePiggybackTokenDispatch atomic.Bool + disableWorkQueueGranting.Store(true) + disablePiggybackTokenDispatch.Store(true) - settings := cluster.MakeTestingClusterSettings() - // Override metamorphism to allow range quiescence. - kvserver.ExpirationLeasesOnly.Override(ctx, &settings.SV, false) - pinnedLease := kvserver.NewPinnedLeases() - tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{ - ReplicationMode: base.ReplicationManual, - ServerArgs: base.TestServerArgs{ - Settings: settings, - RaftConfig: base.RaftConfig{ - // Suppress timeout-based elections. This test doesn't want to deal - // with leadership changing hands. - RaftElectionTimeoutTicks: 1000000, - }, - Knobs: base.TestingKnobs{ - Store: &kvserver.StoreTestingKnobs{ - // Pin the lease to the first store to prevent lease and leader - // moves which disrupt this test. - PinnedLeases: pinnedLease, + settings := cluster.MakeTestingClusterSettings() + // Override metamorphism to allow range quiescence. + kvserver.ExpirationLeasesOnly.Override(ctx, &settings.SV, false) + pinnedLease := kvserver.NewPinnedLeases() + tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{ + ReplicationMode: base.ReplicationManual, + ServerArgs: base.TestServerArgs{ + Settings: settings, + RaftConfig: base.RaftConfig{ + // Suppress timeout-based elections. This test doesn't want to deal + // with leadership changing hands. + RaftElectionTimeoutTicks: 1000000, + }, + Knobs: base.TestingKnobs{ + Store: &kvserver.StoreTestingKnobs{ + // Pin the lease to the first store to prevent lease and leader + // moves which disrupt this test. + PinnedLeases: pinnedLease, - FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ - UseOnlyForScratchRanges: true, - OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { - return v2EnabledWhenLeaderLevel - }, - OverrideTokenDeduction: func(_ kvflowcontrol.Tokens) kvflowcontrol.Tokens { - // This test asserts on the exact values of tracked tokens. In - // non-test code, the tokens deducted are a few bytes off (give - // or take) from the size of the proposals. We don't care about - // such differences. - return kvflowcontrol.Tokens(1 << 20 /* 1MiB */) + FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ + UseOnlyForScratchRanges: true, + OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { + return v2EnabledWhenLeaderLevel + }, + OverrideTokenDeduction: func(_ kvflowcontrol.Tokens) kvflowcontrol.Tokens { + // This test asserts on the exact values of tracked tokens. In + // non-test code, the tokens deducted are a few bytes off (give + // or take) from the size of the proposals. We don't care about + // such differences. + return kvflowcontrol.Tokens(1 << 20 /* 1MiB */) + }, }, }, - }, - AdmissionControl: &admission.TestingKnobs{ - DisableWorkQueueFastPath: true, - DisableWorkQueueGranting: func() bool { - return disableWorkQueueGranting.Load() - }, - }, - RaftTransport: &kvserver.RaftTransportTestingKnobs{ - DisableFallbackFlowTokenDispatch: func() bool { - return disablePiggybackTokenDispatch.Load() + AdmissionControl: &admission.TestingKnobs{ + DisableWorkQueueFastPath: true, + DisableWorkQueueGranting: func() bool { + return disableWorkQueueGranting.Load() + }, }, - DisablePiggyBackedFlowTokenDispatch: func() bool { - return disablePiggybackTokenDispatch.Load() + RaftTransport: &kvserver.RaftTransportTestingKnobs{ + DisableFallbackFlowTokenDispatch: func() bool { + return disablePiggybackTokenDispatch.Load() + }, + DisablePiggyBackedFlowTokenDispatch: func() bool { + return disablePiggybackTokenDispatch.Load() + }, }, }, }, - }, - }) - defer tc.Stopper().Stop(ctx) + }) + defer tc.Stopper().Stop(ctx) - k := tc.ScratchRange(t) - desc, err := tc.LookupRange(k) - require.NoError(t, err) - pinnedLease.PinLease(desc.RangeID, tc.GetFirstStoreFromServer(t, 0).StoreID()) + k := tc.ScratchRange(t) + desc, err := tc.LookupRange(k) + require.NoError(t, err) + pinnedLease.PinLease(desc.RangeID, tc.GetFirstStoreFromServer(t, 0).StoreID()) - tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) - h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) - h.init() - defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, "unquiesced_range")) + tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) + h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) + h.init(mode) + defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, mode, "unquiesced_range")) - h.enableVerboseRaftMsgLoggingForRange(desc.RangeID) - n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) + h.enableVerboseRaftMsgLoggingForRange(desc.RangeID) + n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) - h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) + h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) - h.comment(`-- (Issuing 1x1MiB, 3x replicated elastic write that's not admitted.)`) - h.put(ctx, k, 1<<20 /* 1MiB */, admissionpb.BulkNormalPri) - h.comment(` + h.comment(`-- (Issuing 1x1MiB, 3x replicated elastic write that's not admitted.)`) + h.put(ctx, k, 1<<20 /* 1MiB */, admissionpb.BulkNormalPri) + h.comment(` -- Flow token metrics from n1 after issuing 1x1MiB elastic 3x replicated write -- that's not admitted. We see 1*1MiB*3=3MiB deductions of elastic tokens with -- no corresponding returns. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - // The range must not quiesce because the leader holds send tokens. - leader := tc.GetRaftLeader(t, roachpb.RKey(k)) - require.NotNil(t, leader) - require.False(t, leader.IsQuiescent()) + // The range must not quiesce because the leader holds send tokens. + leader := tc.GetRaftLeader(t, roachpb.RKey(k)) + require.NotNil(t, leader) + require.False(t, leader.IsQuiescent()) - h.comment(` + h.comment(` -- (Allow below-raft admission to proceed. We've disabled the piggybacked token -- return mechanism so no tokens are returned via this path. But the tokens will -- be returned anyway because the range is not quiesced and keeps pinging.)`) - disableWorkQueueGranting.Store(false) - h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) - h.query(n1, v2FlowTokensQueryStr) + disableWorkQueueGranting.Store(false) + h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) + h.query(n1, v2FlowTokensQueryStr) - h.comment(`-- (Issuing another 1x1MiB 3x elastic write.)`) - disableWorkQueueGranting.Store(true) - h.put(ctx, k, 1<<20 /* 1MiB */, admissionpb.BulkNormalPri) - h.query(n1, v2FlowTokensQueryStr) + h.comment(`-- (Issuing another 1x1MiB 3x elastic write.)`) + disableWorkQueueGranting.Store(true) + h.put(ctx, k, 1<<20 /* 1MiB */, admissionpb.BulkNormalPri) + h.query(n1, v2FlowTokensQueryStr) - h.comment(` + h.comment(` -- (Allow below-raft admission to proceed. We've enabled the piggybacked token -- return mechanism so tokens are returned either via this path, or the normal -- MsgAppResp flow, depending on which is exercised first.)`) - disablePiggybackTokenDispatch.Store(false) - disableWorkQueueGranting.Store(false) - h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) - h.query(n1, v2FlowTokensQueryStr) + disablePiggybackTokenDispatch.Store(false) + disableWorkQueueGranting.Store(false) + h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) + h.query(n1, v2FlowTokensQueryStr) - h.comment(`-- (Now the range can quiesce. Wait for it.)`) - testutils.SucceedsSoon(t, func() error { - if !leader.IsQuiescent() { - return errors.Errorf("%s not quiescent", leader) - } - return nil + h.comment(`-- (Now the range can quiesce. Wait for it.)`) + testutils.SucceedsSoon(t, func() error { + if !leader.IsQuiescent() { + return errors.Errorf("%s not quiescent", leader) + } + return nil + }) }) }) } @@ -3479,72 +3529,77 @@ func TestFlowControlTransferLeaseV2(t *testing.T) { kvflowcontrol.V2EnabledWhenLeaderV1Encoding, kvflowcontrol.V2EnabledWhenLeaderV2Encoding, }, func(t *testing.T, v2EnabledWhenLeaderLevel kvflowcontrol.V2EnabledWhenLeaderLevel) { - ctx := context.Background() - var disableWorkQueueGranting atomic.Bool - disableWorkQueueGranting.Store(true) - settings := cluster.MakeTestingClusterSettings() - tc := testcluster.StartTestCluster(t, 5, base.TestClusterArgs{ - ReplicationMode: base.ReplicationManual, - ServerArgs: base.TestServerArgs{ - Settings: settings, - Knobs: base.TestingKnobs{ - Store: &kvserver.StoreTestingKnobs{ - FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ - UseOnlyForScratchRanges: true, - OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { - return v2EnabledWhenLeaderLevel + testutils.RunValues(t, "kvadmission.flow_control.mode", []kvflowcontrol.ModeT{ + kvflowcontrol.ApplyToElastic, + kvflowcontrol.ApplyToAll, + }, func(t *testing.T, mode kvflowcontrol.ModeT) { + ctx := context.Background() + var disableWorkQueueGranting atomic.Bool + disableWorkQueueGranting.Store(true) + settings := cluster.MakeTestingClusterSettings() + tc := testcluster.StartTestCluster(t, 5, base.TestClusterArgs{ + ReplicationMode: base.ReplicationManual, + ServerArgs: base.TestServerArgs{ + Settings: settings, + Knobs: base.TestingKnobs{ + Store: &kvserver.StoreTestingKnobs{ + FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ + UseOnlyForScratchRanges: true, + OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { + return v2EnabledWhenLeaderLevel + }, }, }, - }, - AdmissionControl: &admission.TestingKnobs{ - DisableWorkQueueFastPath: true, - DisableWorkQueueGranting: func() bool { - return disableWorkQueueGranting.Load() + AdmissionControl: &admission.TestingKnobs{ + DisableWorkQueueFastPath: true, + DisableWorkQueueGranting: func() bool { + return disableWorkQueueGranting.Load() + }, }, }, }, - }, - }) - defer tc.Stopper().Stop(ctx) + }) + defer tc.Stopper().Stop(ctx) - k := tc.ScratchRange(t) - tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) + k := tc.ScratchRange(t) + tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) - n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) + n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) - h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) - h.init() - defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, "transfer_lease")) + h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) + h.init(mode) + defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, mode, "transfer_lease")) - desc, err := tc.LookupRange(k) - require.NoError(t, err) - h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) + desc, err := tc.LookupRange(k) + require.NoError(t, err) + h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) - h.comment(`-- (Issuing 1x1MiB, 3x replicated write that's not admitted.)`) - h.put(ctx, k, 1<<20 /* 1MiB */, admissionpb.NormalPri) + h.comment(`-- (Issuing 1x1MiB, 3x replicated write that's not admitted.)`) + h.put(ctx, k, 1<<20 /* 1MiB */, testFlowModeToPri(mode)) - h.comment(` --- Flow token metrics from n1 after issuing 1x1MiB regular 3x replicated write --- that's not admitted. We see 1*1MiB*3=3MiB deductions of regular tokens with + h.comment(` +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with -- no corresponding returns. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(`-- (Transferring range lease to n2 and allowing leadership to follow.)`) - tc.TransferRangeLeaseOrFatal(t, desc, tc.Target(1)) - testutils.SucceedsSoon(t, func() error { - if leader := tc.GetRaftLeader(t, roachpb.RKey(k)); leader.NodeID() != tc.Target(1).NodeID { - return errors.Errorf("expected raft leadership to transfer to n1, found n%d", leader.NodeID()) - } - return nil - }) - h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) + h.comment(`-- (Transferring range lease to n2 and allowing leadership to follow.)`) + tc.TransferRangeLeaseOrFatal(t, desc, tc.Target(1)) + testutils.SucceedsSoon(t, func() error { + if leader := tc.GetRaftLeader(t, roachpb.RKey(k)); leader.NodeID() != tc.Target(1).NodeID { + return errors.Errorf("expected raft leadership to transfer to n1, found n%d", leader.NodeID()) + } + return nil + }) + h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) - h.comment(` + h.comment(` -- Flow token metrics from n1 having lost the lease and raft leadership. All -- deducted tokens are returned. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) + }) }) } @@ -3562,96 +3617,101 @@ func TestFlowControlLeaderNotLeaseholderV2(t *testing.T) { kvflowcontrol.V2EnabledWhenLeaderV1Encoding, kvflowcontrol.V2EnabledWhenLeaderV2Encoding, }, func(t *testing.T, v2EnabledWhenLeaderLevel kvflowcontrol.V2EnabledWhenLeaderLevel) { - ctx := context.Background() - var disableWorkQueueGranting atomic.Bool - disableWorkQueueGranting.Store(true) - settings := cluster.MakeTestingClusterSettings() - tc := testcluster.StartTestCluster(t, 5, base.TestClusterArgs{ - ReplicationMode: base.ReplicationManual, - ServerArgs: base.TestServerArgs{ - Settings: settings, - Knobs: base.TestingKnobs{ - Store: &kvserver.StoreTestingKnobs{ - // Disable leader transfers during leaseholder changes so - // that we can easily create leader-not-leaseholder - // scenarios. - DisableLeaderFollowsLeaseholder: true, - FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ - UseOnlyForScratchRanges: true, - OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { - return v2EnabledWhenLeaderLevel + testutils.RunValues(t, "kvadmission.flow_control.mode", []kvflowcontrol.ModeT{ + kvflowcontrol.ApplyToElastic, + kvflowcontrol.ApplyToAll, + }, func(t *testing.T, mode kvflowcontrol.ModeT) { + ctx := context.Background() + var disableWorkQueueGranting atomic.Bool + disableWorkQueueGranting.Store(true) + settings := cluster.MakeTestingClusterSettings() + tc := testcluster.StartTestCluster(t, 5, base.TestClusterArgs{ + ReplicationMode: base.ReplicationManual, + ServerArgs: base.TestServerArgs{ + Settings: settings, + Knobs: base.TestingKnobs{ + Store: &kvserver.StoreTestingKnobs{ + // Disable leader transfers during leaseholder changes so + // that we can easily create leader-not-leaseholder + // scenarios. + DisableLeaderFollowsLeaseholder: true, + FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ + UseOnlyForScratchRanges: true, + OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { + return v2EnabledWhenLeaderLevel + }, }, }, - }, - AdmissionControl: &admission.TestingKnobs{ - DisableWorkQueueFastPath: true, - DisableWorkQueueGranting: func() bool { - return disableWorkQueueGranting.Load() + AdmissionControl: &admission.TestingKnobs{ + DisableWorkQueueFastPath: true, + DisableWorkQueueGranting: func() bool { + return disableWorkQueueGranting.Load() + }, }, }, }, - }, - }) - defer tc.Stopper().Stop(ctx) + }) + defer tc.Stopper().Stop(ctx) - k := tc.ScratchRange(t) - tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) + k := tc.ScratchRange(t) + tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) - n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) - n2 := sqlutils.MakeSQLRunner(tc.ServerConn(1)) + n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) + n2 := sqlutils.MakeSQLRunner(tc.ServerConn(1)) - h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) - h.init() - defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, "leader_not_leaseholder")) + h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) + h.init(mode) + defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, mode, "leader_not_leaseholder")) - desc, err := tc.LookupRange(k) - require.NoError(t, err) - h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) + desc, err := tc.LookupRange(k) + require.NoError(t, err) + h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) - h.comment(`-- (Issuing 1x1MiB, 3x replicated write that's not admitted.)`) - h.put(ctx, k, 1<<20 /* 1MiB */, admissionpb.NormalPri) + h.comment(`-- (Issuing 1x1MiB, 3x replicated write that's not admitted.)`) + h.put(ctx, k, 1<<20 /* 1MiB */, testFlowModeToPri(mode)) - h.comment(` --- Flow token metrics from n1 after issuing 1x1MiB regular 3x replicated write --- that's not admitted. We see 1*1MiB*3=3MiB deductions of regular tokens with + h.comment(` +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with -- no corresponding returns. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(`-- (Transferring only range lease, not raft leadership, to n2.)`) - tc.TransferRangeLeaseOrFatal(t, desc, tc.Target(1)) - require.Equal(t, tc.GetRaftLeader(t, roachpb.RKey(k)).NodeID(), tc.Target(0).NodeID) + h.comment(`-- (Transferring only range lease, not raft leadership, to n2.)`) + tc.TransferRangeLeaseOrFatal(t, desc, tc.Target(1)) + require.Equal(t, tc.GetRaftLeader(t, roachpb.RKey(k)).NodeID(), tc.Target(0).NodeID) - h.comment(` + h.comment(` -- Flow token metrics from n1 having lost the lease but retained raft -- leadership. No deducted tokens are released. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(` + h.comment(` -- (Allow below-raft admission to proceed. All tokens should be returned.) `) - disableWorkQueueGranting.Store(false) - h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) - h.query(n1, v2FlowTokensQueryStr) + disableWorkQueueGranting.Store(false) + h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) + h.query(n1, v2FlowTokensQueryStr) - h.comment(` + h.comment(` -- (Issuing another 1x1MiB, 3x replicated write that's admitted via -- the work queue on the leaseholder. It shouldn't deduct any tokens.) `) - h.put(ctx, k, 1<<20 /* 1MiB */, admissionpb.NormalPri) + h.put(ctx, k, 1<<20 /* 1MiB */, testFlowModeToPri(mode)) - h.comment(` + h.comment(` -- Looking at n1's flow token metrics, there's no change. No additional tokens -- are deducted since the write is not being proposed here. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(` + h.comment(` -- Looking at n2's flow token metrics, there's no activity. n2 never acquired -- the raft leadership. `) - h.query(n2, v2FlowTokensQueryStr) + h.query(n2, v2FlowTokensQueryStr) + }) }) } @@ -3669,91 +3729,96 @@ func TestFlowControlGranterAdmitOneByOneV2(t *testing.T) { kvflowcontrol.V2EnabledWhenLeaderV1Encoding, kvflowcontrol.V2EnabledWhenLeaderV2Encoding, }, func(t *testing.T, v2EnabledWhenLeaderLevel kvflowcontrol.V2EnabledWhenLeaderLevel) { - ctx := context.Background() - var disableWorkQueueGranting atomic.Bool - disableWorkQueueGranting.Store(true) - settings := cluster.MakeTestingClusterSettings() - tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{ - ReplicationMode: base.ReplicationManual, - ServerArgs: base.TestServerArgs{ - Settings: settings, - Knobs: base.TestingKnobs{ - Store: &kvserver.StoreTestingKnobs{ - FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ - UseOnlyForScratchRanges: true, - OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { - return v2EnabledWhenLeaderLevel - }, - OverrideTokenDeduction: func(_ kvflowcontrol.Tokens) kvflowcontrol.Tokens { - // This test asserts on the exact values of tracked - // tokens. In non-test code, the tokens deducted are - // a few bytes off (give or take) from the size of - // the proposals. We don't care about such - // differences. - return kvflowcontrol.Tokens(1 << 10 /* 1KiB */) + testutils.RunValues(t, "kvadmission.flow_control.mode", []kvflowcontrol.ModeT{ + kvflowcontrol.ApplyToElastic, + kvflowcontrol.ApplyToAll, + }, func(t *testing.T, mode kvflowcontrol.ModeT) { + ctx := context.Background() + var disableWorkQueueGranting atomic.Bool + disableWorkQueueGranting.Store(true) + settings := cluster.MakeTestingClusterSettings() + tc := testcluster.StartTestCluster(t, 3, base.TestClusterArgs{ + ReplicationMode: base.ReplicationManual, + ServerArgs: base.TestServerArgs{ + Settings: settings, + Knobs: base.TestingKnobs{ + Store: &kvserver.StoreTestingKnobs{ + FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ + UseOnlyForScratchRanges: true, + OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { + return v2EnabledWhenLeaderLevel + }, + OverrideTokenDeduction: func(_ kvflowcontrol.Tokens) kvflowcontrol.Tokens { + // This test asserts on the exact values of tracked + // tokens. In non-test code, the tokens deducted are + // a few bytes off (give or take) from the size of + // the proposals. We don't care about such + // differences. + return kvflowcontrol.Tokens(1 << 10 /* 1KiB */) + }, }, }, - }, - AdmissionControl: &admission.TestingKnobs{ - DisableWorkQueueFastPath: true, - DisableWorkQueueGranting: func() bool { - return disableWorkQueueGranting.Load() + AdmissionControl: &admission.TestingKnobs{ + DisableWorkQueueFastPath: true, + DisableWorkQueueGranting: func() bool { + return disableWorkQueueGranting.Load() + }, + AlwaysTryGrantWhenAdmitted: true, }, - AlwaysTryGrantWhenAdmitted: true, }, }, - }, - }) - defer tc.Stopper().Stop(ctx) + }) + defer tc.Stopper().Stop(ctx) - k := tc.ScratchRange(t) - tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) + k := tc.ScratchRange(t) + tc.AddVotersOrFatal(t, k, tc.Targets(1, 2)...) - n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) + n1 := sqlutils.MakeSQLRunner(tc.ServerConn(0)) - h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) - h.init() - defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, "granter_admit_one_by_one")) + h := newFlowControlTestHelperV2(t, tc, v2EnabledWhenLeaderLevel) + h.init(mode) + defer h.close(makeV2EnabledTestFileName(v2EnabledWhenLeaderLevel, mode, "granter_admit_one_by_one")) - desc, err := tc.LookupRange(k) - require.NoError(t, err) - h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) + desc, err := tc.LookupRange(k) + require.NoError(t, err) + h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) - h.comment(`-- (Issuing regular 1024*1KiB, 3x replicated writes that are not admitted.)`) - h.log("sending put requests") - for i := 0; i < 1024; i++ { - // TODO(kvoli): This sleep is necessary because we fill up the (raft) - // send queue and delay sending + tracking. We need to determine why this - // occasionally occurs under race. - time.Sleep(1 * time.Millisecond) - h.put(ctx, k, 1<<10 /* 1KiB */, admissionpb.NormalPri) - } - h.log("sent put requests") + h.comment(`-- (Issuing 1024*1KiB, 3x replicated writes that are not admitted.)`) + h.log("sending put requests") + for i := 0; i < 1024; i++ { + // TODO(kvoli): This sleep is necessary because we fill up the (raft) + // send queue and delay sending + tracking. We need to determine why this + // occasionally occurs under race. + time.Sleep(1 * time.Millisecond) + h.put(ctx, k, 1<<10 /* 1KiB */, testFlowModeToPri(mode)) + } + h.log("sent put requests") - h.comment(` + h.comment(` -- Flow token metrics from n1 after issuing 1024KiB, i.e. 1MiB 3x replicated writes -- that are yet to get admitted. We see 3*1MiB=3MiB deductions of -- {regular,elastic} tokens with no corresponding returns. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) - h.comment(`-- Observe the total tracked tokens per-stream on n1.`) - h.query(n1, ` + h.comment(`-- Observe the total tracked tokens per-stream on n1.`) + h.query(n1, ` SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) FROM crdb_internal.kv_flow_control_handles_v2 `, "range_id", "store_id", "total_tracked_tokens") - h.comment(`-- (Allow below-raft admission to proceed.)`) - disableWorkQueueGranting.Store(false) - h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) // wait for admission + h.comment(`-- (Allow below-raft admission to proceed.)`) + disableWorkQueueGranting.Store(false) + h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) // wait for admission - h.comment(` + h.comment(` -- Flow token metrics from n1 after work gets admitted. We see 3MiB returns of -- {regular,elastic} tokens, and the available capacities going back to what -- they were. In #105185, by now we would've observed panics. `) - h.query(n1, v2FlowTokensQueryStr) + h.query(n1, v2FlowTokensQueryStr) + }) }) } @@ -3805,6 +3870,10 @@ func TestFlowControlV1ToV2Transition(t *testing.T) { Store: &kvserver.StoreTestingKnobs{ FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ UseOnlyForScratchRanges: true, + OverridePullPushMode: func() bool { + // Push mode. + return false + }, OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { return serverLevels[i].Load() }, @@ -3843,7 +3912,7 @@ func TestFlowControlV1ToV2Transition(t *testing.T) { kvflowcontrol.V2NotEnabledWhenLeader, false, /* isStatic */ ) - h.init() + h.init(kvflowcontrol.ApplyToAll) defer h.close("v1_to_v2_transition") desc, err := tc.LookupRange(k) @@ -4435,14 +4504,25 @@ func newFlowControlTestHelperV2( ) } -func (h *flowControlTestHelper) init() { +func testFlowModeToPri(mode kvflowcontrol.ModeT) admissionpb.WorkPriority { + switch mode { + case kvflowcontrol.ApplyToElastic: + return admissionpb.UserLowPri + case kvflowcontrol.ApplyToAll: + return admissionpb.UserHighPri + default: + panic("unknown flow control mode") + } +} + +func (h *flowControlTestHelper) init(mode kvflowcontrol.ModeT) { // Reach into each server's cluster setting and override. This causes any // registered change callbacks to run immediately, which is important since // running them with some lag (which happens when using SQL and `SET CLUSTER // SETTING`) interferes with the later activities in these tests. for _, s := range h.tc.Servers { kvflowcontrol.Enabled.Override(context.Background(), &s.ClusterSettings().SV, true) - kvflowcontrol.Mode.Override(context.Background(), &s.ClusterSettings().SV, kvflowcontrol.ApplyToAll) + kvflowcontrol.Mode.Override(context.Background(), &s.ClusterSettings().SV, mode) } } @@ -4725,20 +4805,22 @@ func (h *flowControlTestHelper) enableVerboseRaftMsgLoggingForRange(rangeID roac // makeV2EnabledTestFileName is a utility function which returns an updated // filename for the testdata file based on the v2EnabledWhenLeaderLevel. func makeV2EnabledTestFileName( - v2EnabledWhenLeaderLevel kvflowcontrol.V2EnabledWhenLeaderLevel, filename string, + v2EnabledWhenLeaderLevel kvflowcontrol.V2EnabledWhenLeaderLevel, + mode kvflowcontrol.ModeT, + filename string, ) string { - var s string + var enabledPart string switch v2EnabledWhenLeaderLevel { case kvflowcontrol.V2NotEnabledWhenLeader: panic("unused") case kvflowcontrol.V2EnabledWhenLeaderV1Encoding: - s = "_v1_encoding" + enabledPart = "_v1_encoding" case kvflowcontrol.V2EnabledWhenLeaderV2Encoding: - s = "" + enabledPart = "_v2_encoding" default: panic("unknown v2EnabledWhenLeaderLevel") } - return filename + s + return filename + enabledPart + "_" + mode.String() } func BenchmarkFlowControlV2Basic(b *testing.B) { @@ -4748,45 +4830,50 @@ func BenchmarkFlowControlV2Basic(b *testing.B) { kvflowcontrol.V2EnabledWhenLeaderV1Encoding, kvflowcontrol.V2EnabledWhenLeaderV2Encoding, }, func(b *testing.B, v2EnabledWhenLeaderLevel kvflowcontrol.V2EnabledWhenLeaderLevel) { - ctx := context.Background() - settings := cluster.MakeTestingClusterSettings() - tc := testcluster.StartTestCluster(b, 3, base.TestClusterArgs{ - ReplicationMode: base.ReplicationManual, - ServerArgs: base.TestServerArgs{ - Settings: settings, - Knobs: base.TestingKnobs{ - Store: &kvserver.StoreTestingKnobs{ - FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ - UseOnlyForScratchRanges: true, - OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { - return v2EnabledWhenLeaderLevel + testutils.RunValues(b, "kvadmission.flow_control.mode", []kvflowcontrol.ModeT{ + kvflowcontrol.ApplyToElastic, + kvflowcontrol.ApplyToAll, + }, func(t *testing.B, mode kvflowcontrol.ModeT) { + ctx := context.Background() + settings := cluster.MakeTestingClusterSettings() + tc := testcluster.StartTestCluster(b, 3, base.TestClusterArgs{ + ReplicationMode: base.ReplicationManual, + ServerArgs: base.TestServerArgs{ + Settings: settings, + Knobs: base.TestingKnobs{ + Store: &kvserver.StoreTestingKnobs{ + FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{ + UseOnlyForScratchRanges: true, + OverrideV2EnabledWhenLeaderLevel: func() kvflowcontrol.V2EnabledWhenLeaderLevel { + return v2EnabledWhenLeaderLevel + }, }, }, - }, - AdmissionControl: &admission.TestingKnobs{ - DisableWorkQueueFastPath: false, + AdmissionControl: &admission.TestingKnobs{ + DisableWorkQueueFastPath: false, + }, }, }, - }, - }) - defer tc.Stopper().Stop(ctx) + }) + defer tc.Stopper().Stop(ctx) - // Set up the benchmark state with 3 voters, one on each of the three - // node/stores. - k := tc.ScratchRange(b) - tc.AddVotersOrFatal(b, k, tc.Targets(1, 2)...) - h := newFlowControlTestHelperV2(b, tc, v2EnabledWhenLeaderLevel) - h.init() + // Set up the benchmark state with 3 voters, one on each of the three + // node/stores. + k := tc.ScratchRange(b) + tc.AddVotersOrFatal(b, k, tc.Targets(1, 2)...) + h := newFlowControlTestHelperV2(b, tc, v2EnabledWhenLeaderLevel) + h.init(mode) - desc, err := tc.LookupRange(k) - require.NoError(b, err) - h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) + desc, err := tc.LookupRange(k) + require.NoError(b, err) + h.waitForConnectedStreams(ctx, desc.RangeID, 3, 0 /* serverIdx */) - b.ResetTimer() - for i := 0; i < b.N; i++ { - h.put(ctx, k, 1<<10 /* 1KiB */, admissionpb.UserLowPri) - } - h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) - b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + h.put(ctx, k, 1<<10 /* 1KiB */, testFlowModeToPri(mode)) + } + h.waitForAllTokensReturned(ctx, 3, 0 /* serverIdx */) + b.StopTimer() + }) }) } diff --git a/pkg/kv/kvserver/kvflowcontrol/testing_knobs.go b/pkg/kv/kvserver/kvflowcontrol/testing_knobs.go index fb76f13b4b58..1da9ca425079 100644 --- a/pkg/kv/kvserver/kvflowcontrol/testing_knobs.go +++ b/pkg/kv/kvserver/kvflowcontrol/testing_knobs.go @@ -21,6 +21,17 @@ type TestingKnobs struct { // OverrideV2EnabledWhenLeaderLevel is used to override the level at which // RACv2 is enabled when a replica is the leader. OverrideV2EnabledWhenLeaderLevel func() V2EnabledWhenLeaderLevel + // OverridePullPushMode is used to override whether the pull mode, or push + // mode is enabled. + // + // - when set to true, pull mode is enabled + // - when set to false, push mode is enabled + // - when left unset the otherwise set mode is used + // + // This is used to test the behavior of the flow control in push and pull + // mode, while also having the ability to switch between the two + // apply_to_(elastic|all) modes. + OverridePullPushMode func() bool } // TestingKnobsV1 are the testing knobs that appply to replication flow control diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v1_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v1_encoding_apply_to_all similarity index 98% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v1_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v1_encoding_apply_to_all index 216e2d4200e3..c16d8cdfe2bf 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v1_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v1_encoding_apply_to_all @@ -8,7 +8,7 @@ echo ---- ---- --- Flow token metrics from n1 after issuing a regular 2*1MiB 3x replicated write +-- Flow token metrics from n1 after issuing a 2*1MiB 3x replicated write -- that are yet to get admitted. We see 2*3*1MiB=6MiB deductions of -- {regular,elastic} tokens with no corresponding returns. The 2*1MiB writes -- happened on what is soon going to be the LHS and RHS of a range being split. @@ -94,7 +94,7 @@ ORDER BY streams DESC; -- (Merging ranges.) --- Flow token metrics from n1 after issuing 4MiB of regular replicated writes to +-- Flow token metrics from n1 after issuing 4MiB of replicated writes to -- the post-merged range. We should see 12MiB extra tokens deducted which comes -- from 4MiB*3=12MiB. So we stand at 21MiB+12MiB=33MiB tokens deducted now. The -- RHS of the range is gone now, and the previously 3*3MiB=9MiB of tokens diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v1_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v1_encoding_apply_to_elastic new file mode 100644 index 000000000000..c16d8cdfe2bf --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v1_encoding_apply_to_elastic @@ -0,0 +1,181 @@ +# -- When using v1 encoding (V2EnabledWhenLeaderV1Encoding), all entries which +# -- are subject to admission control are encoded as `raftpb.LowPri`, +# -- regardless of their original priority; to avoid the overhead of +# -- deserializing the raft admission metadata. Therefore, as the underlying +# -- test is shared between the v1 and v2 encoding testdata files, the reader +# -- should interpret any comments referring to regular tokens as referring to +# -- elastic token. +echo +---- +---- +-- Flow token metrics from n1 after issuing a 2*1MiB 3x replicated write +-- that are yet to get admitted. We see 2*3*1MiB=6MiB deductions of +-- {regular,elastic} tokens with no corresponding returns. The 2*1MiB writes +-- happened on what is soon going to be the LHS and RHS of a range being split. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 18 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 18 MiB + kvflowcontrol.tokens.send.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Splitting range.) + + +-- Flow token metrics from n1 after further issuing 2MiB and 3MiB writes to +-- post-split LHS and RHS ranges respectively. We should see 15MiB extra tokens +-- deducted which comes from (2MiB+3MiB)*3=15MiB. So we stand at +-- 6MiB+15MiB=21MiB now. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 21 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe the newly split off replica, with its own three streams. +SELECT range_id, count(*) AS streams + FROM crdb_internal.kv_flow_control_handles_v2 +GROUP BY (range_id) +ORDER BY streams DESC; + + range_id | stream_count +-----------+--------------- + 70 | 3 + 71 | 3 + + +-- (Merging ranges.) + + +-- Flow token metrics from n1 after issuing 4MiB of replicated writes to +-- the post-merged range. We should see 12MiB extra tokens deducted which comes +-- from 4MiB*3=12MiB. So we stand at 21MiB+12MiB=33MiB tokens deducted now. The +-- RHS of the range is gone now, and the previously 3*3MiB=9MiB of tokens +-- deducted for it are released at the subsuming LHS leaseholder. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 0 B + kvflowcontrol.tokens.eval.elastic.deducted | 33 MiB + kvflowcontrol.tokens.eval.elastic.returned | 9.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 9.0 MiB + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 0 B + kvflowcontrol.tokens.send.elastic.deducted | 33 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 9.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 9.0 MiB + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe only the merged replica with its own three streams. +SELECT range_id, count(*) AS streams + FROM crdb_internal.kv_flow_control_handles_v2 +GROUP BY (range_id) +ORDER BY streams DESC; + + range_id | stream_count +-----------+--------------- + 70 | 3 + + +-- (Allow below-raft admission to proceed.) + + +-- Flow token metrics from n1 after work gets admitted. We see all outstanding +-- {regular,elastic} tokens returned, including those from: +-- - the LHS before the merge, and +-- - the LHS and RHS before the original split. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 33 MiB + kvflowcontrol.tokens.eval.elastic.returned | 33 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 9.0 MiB + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 33 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 33 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 9.0 MiB + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v2_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v2_encoding_apply_to_all similarity index 98% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v2_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v2_encoding_apply_to_all index e84eae8f5cd8..d2a16d3a8194 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v2_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v2_encoding_apply_to_all @@ -1,7 +1,7 @@ echo ---- ---- --- Flow token metrics from n1 after issuing a regular 2*1MiB 3x replicated write +-- Flow token metrics from n1 after issuing a 2*1MiB 3x replicated write -- that are yet to get admitted. We see 2*3*1MiB=6MiB deductions of -- {regular,elastic} tokens with no corresponding returns. The 2*1MiB writes -- happened on what is soon going to be the LHS and RHS of a range being split. @@ -87,7 +87,7 @@ ORDER BY streams DESC; -- (Merging ranges.) --- Flow token metrics from n1 after issuing 4MiB of regular replicated writes to +-- Flow token metrics from n1 after issuing 4MiB of replicated writes to -- the post-merged range. We should see 12MiB extra tokens deducted which comes -- from 4MiB*3=12MiB. So we stand at 21MiB+12MiB=33MiB tokens deducted now. The -- RHS of the range is gone now, and the previously 3*3MiB=9MiB of tokens diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v2_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v2_encoding_apply_to_elastic new file mode 100644 index 000000000000..875f65abff97 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/admission_post_split_merge_v2_encoding_apply_to_elastic @@ -0,0 +1,174 @@ +echo +---- +---- +-- Flow token metrics from n1 after issuing a 2*1MiB 3x replicated write +-- that are yet to get admitted. We see 2*3*1MiB=6MiB deductions of +-- {regular,elastic} tokens with no corresponding returns. The 2*1MiB writes +-- happened on what is soon going to be the LHS and RHS of a range being split. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 18 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 18 MiB + kvflowcontrol.tokens.send.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Splitting range.) + + +-- Flow token metrics from n1 after further issuing 2MiB and 3MiB writes to +-- post-split LHS and RHS ranges respectively. We should see 15MiB extra tokens +-- deducted which comes from (2MiB+3MiB)*3=15MiB. So we stand at +-- 6MiB+15MiB=21MiB now. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 21 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe the newly split off replica, with its own three streams. +SELECT range_id, count(*) AS streams + FROM crdb_internal.kv_flow_control_handles_v2 +GROUP BY (range_id) +ORDER BY streams DESC; + + range_id | stream_count +-----------+--------------- + 70 | 3 + 71 | 3 + + +-- (Merging ranges.) + + +-- Flow token metrics from n1 after issuing 4MiB of replicated writes to +-- the post-merged range. We should see 12MiB extra tokens deducted which comes +-- from 4MiB*3=12MiB. So we stand at 21MiB+12MiB=33MiB tokens deducted now. The +-- RHS of the range is gone now, and the previously 3*3MiB=9MiB of tokens +-- deducted for it are released at the subsuming LHS leaseholder. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 0 B + kvflowcontrol.tokens.eval.elastic.deducted | 33 MiB + kvflowcontrol.tokens.eval.elastic.returned | 9.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 9.0 MiB + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 0 B + kvflowcontrol.tokens.send.elastic.deducted | 33 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 9.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 9.0 MiB + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe only the merged replica with its own three streams. +SELECT range_id, count(*) AS streams + FROM crdb_internal.kv_flow_control_handles_v2 +GROUP BY (range_id) +ORDER BY streams DESC; + + range_id | stream_count +-----------+--------------- + 70 | 3 + + +-- (Allow below-raft admission to proceed.) + + +-- Flow token metrics from n1 after work gets admitted. We see all outstanding +-- {regular,elastic} tokens returned, including those from: +-- - the LHS before the merge, and +-- - the LHS and RHS before the original split. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 33 MiB + kvflowcontrol.tokens.eval.elastic.returned | 33 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 9.0 MiB + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 33 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 33 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 9.0 MiB + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v1_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v1_encoding_apply_to_all similarity index 96% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v1_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v1_encoding_apply_to_all index 4fabfb1e4cad..810841c79646 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v1_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v1_encoding_apply_to_all @@ -8,7 +8,7 @@ echo ---- ---- --- Flow token metrics, before issuing the regular 1MiB replicated write. +-- Flow token metrics, before issuing the 1MiB replicated write. SELECT name, crdb_internal.humanize_bytes(value::INT8) FROM crdb_internal.node_metrics WHERE name LIKE '%kvflowcontrol%tokens%' @@ -39,7 +39,7 @@ ORDER BY name ASC; kvflowcontrol.tokens.send.regular.unaccounted | 0 B --- (Issuing + admitting a regular 1MiB, triply replicated write...) +-- (Issuing + admitting a 1MiB, triply replicated write...) -- Stream counts as seen by n1 post-write. We should see three {regular,elastic} @@ -71,7 +71,7 @@ ORDER BY streams DESC; 70 | 3 --- Flow token metrics from n1 after issuing the regular 1MiB replicated write, +-- Flow token metrics from n1 after issuing the 1MiB replicated write, -- and it being admitted on n1, n2 and n3. We should see 3*1MiB = 3MiB of -- {regular,elastic} tokens deducted and returned, and {8*3=24MiB,16*3=48MiB} of -- {regular,elastic} tokens available. Everything should be accounted for. diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v1_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v1_encoding_apply_to_elastic new file mode 100644 index 000000000000..810841c79646 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v1_encoding_apply_to_elastic @@ -0,0 +1,109 @@ +# -- When using v1 encoding (V2EnabledWhenLeaderV1Encoding), all entries which +# -- are subject to admission control are encoded as `raftpb.LowPri`, +# -- regardless of their original priority; to avoid the overhead of +# -- deserializing the raft admission metadata. Therefore, as the underlying +# -- test is shared between the v1 and v2 encoding testdata files, the reader +# -- should interpret any comments referring to regular tokens as referring to +# -- elastic token. +echo +---- +---- +-- Flow token metrics, before issuing the 1MiB replicated write. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 0 B + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 0 B + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Issuing + admitting a 1MiB, triply replicated write...) + + +-- Stream counts as seen by n1 post-write. We should see three {regular,elastic} +-- streams given there are three nodes and we're using a replication factor of +-- three. +SELECT name, value + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%stream%' +ORDER BY name ASC; + + kvflowcontrol.streams.eval.elastic.blocked_count | 0 + kvflowcontrol.streams.eval.elastic.total_count | 3 + kvflowcontrol.streams.eval.regular.blocked_count | 0 + kvflowcontrol.streams.eval.regular.total_count | 3 + kvflowcontrol.streams.send.elastic.blocked_count | 0 + kvflowcontrol.streams.send.elastic.total_count | 3 + kvflowcontrol.streams.send.regular.blocked_count | 0 + kvflowcontrol.streams.send.regular.total_count | 3 + + +-- Another view of the stream count, using /inspectz-backed vtables. +SELECT range_id, count(*) AS streams + FROM crdb_internal.kv_flow_control_handles_v2 +GROUP BY (range_id) +ORDER BY streams DESC; + + range_id | stream_count +-----------+--------------- + 70 | 3 + + +-- Flow token metrics from n1 after issuing the 1MiB replicated write, +-- and it being admitted on n1, n2 and n3. We should see 3*1MiB = 3MiB of +-- {regular,elastic} tokens deducted and returned, and {8*3=24MiB,16*3=48MiB} of +-- {regular,elastic} tokens available. Everything should be accounted for. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v2_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v2_encoding_apply_to_all similarity index 95% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v2_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v2_encoding_apply_to_all index 33f453a7d133..d423c3c9de63 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v2_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v2_encoding_apply_to_all @@ -1,7 +1,7 @@ echo ---- ---- --- Flow token metrics, before issuing the regular 1MiB replicated write. +-- Flow token metrics, before issuing the 1MiB replicated write. SELECT name, crdb_internal.humanize_bytes(value::INT8) FROM crdb_internal.node_metrics WHERE name LIKE '%kvflowcontrol%tokens%' @@ -32,7 +32,7 @@ ORDER BY name ASC; kvflowcontrol.tokens.send.regular.unaccounted | 0 B --- (Issuing + admitting a regular 1MiB, triply replicated write...) +-- (Issuing + admitting a 1MiB, triply replicated write...) -- Stream counts as seen by n1 post-write. We should see three {regular,elastic} @@ -64,7 +64,7 @@ ORDER BY streams DESC; 70 | 3 --- Flow token metrics from n1 after issuing the regular 1MiB replicated write, +-- Flow token metrics from n1 after issuing the 1MiB replicated write, -- and it being admitted on n1, n2 and n3. We should see 3*1MiB = 3MiB of -- {regular,elastic} tokens deducted and returned, and {8*3=24MiB,16*3=48MiB} of -- {regular,elastic} tokens available. Everything should be accounted for. diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v2_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v2_encoding_apply_to_elastic new file mode 100644 index 000000000000..53a9e7621f2d --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/basic_v2_encoding_apply_to_elastic @@ -0,0 +1,102 @@ +echo +---- +---- +-- Flow token metrics, before issuing the 1MiB replicated write. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 0 B + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 0 B + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Issuing + admitting a 1MiB, triply replicated write...) + + +-- Stream counts as seen by n1 post-write. We should see three {regular,elastic} +-- streams given there are three nodes and we're using a replication factor of +-- three. +SELECT name, value + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%stream%' +ORDER BY name ASC; + + kvflowcontrol.streams.eval.elastic.blocked_count | 0 + kvflowcontrol.streams.eval.elastic.total_count | 3 + kvflowcontrol.streams.eval.regular.blocked_count | 0 + kvflowcontrol.streams.eval.regular.total_count | 3 + kvflowcontrol.streams.send.elastic.blocked_count | 0 + kvflowcontrol.streams.send.elastic.total_count | 3 + kvflowcontrol.streams.send.regular.blocked_count | 0 + kvflowcontrol.streams.send.regular.total_count | 3 + + +-- Another view of the stream count, using /inspectz-backed vtables. +SELECT range_id, count(*) AS streams + FROM crdb_internal.kv_flow_control_handles_v2 +GROUP BY (range_id) +ORDER BY streams DESC; + + range_id | stream_count +-----------+--------------- + 70 | 3 + + +-- Flow token metrics from n1 after issuing the 1MiB replicated write, +-- and it being admitted on n1, n2 and n3. We should see 3*1MiB = 3MiB of +-- {regular,elastic} tokens deducted and returned, and {8*3=24MiB,16*3=48MiB} of +-- {regular,elastic} tokens available. Everything should be accounted for. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v1_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v1_encoding_apply_to_all similarity index 97% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v1_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v1_encoding_apply_to_all index 80fbd272cca3..6fe8826ca02a 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v1_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v1_encoding_apply_to_all @@ -8,10 +8,10 @@ echo ---- ---- --- (Issuing 5 regular 1MiB, 3x replicated write that's not admitted.) +-- (Issuing 5 1MiB, 3x replicated write that's not admitted.) --- Flow token metrics from n1 after issuing 5 regular 1MiB 3x replicated writes +-- Flow token metrics from n1 after issuing 5 1MiB 3x replicated writes -- that are yet to get admitted. We see 5*1MiB*3=15MiB deductions of -- {regular,elastic} tokens with no corresponding returns. SELECT name, crdb_internal.humanize_bytes(value::INT8) diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v1_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v1_encoding_apply_to_elastic new file mode 100644 index 000000000000..6fe8826ca02a --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v1_encoding_apply_to_elastic @@ -0,0 +1,118 @@ +# -- When using v1 encoding (V2EnabledWhenLeaderV1Encoding), all entries which +# -- are subject to admission control are encoded as `raftpb.LowPri`, +# -- regardless of their original priority; to avoid the overhead of +# -- deserializing the raft admission metadata. Therefore, as the underlying +# -- test is shared between the v1 and v2 encoding testdata files, the reader +# -- should interpret any comments referring to regular tokens as referring to +# -- elastic token. +echo +---- +---- +-- (Issuing 5 1MiB, 3x replicated write that's not admitted.) + + +-- Flow token metrics from n1 after issuing 5 1MiB 3x replicated writes +-- that are yet to get admitted. We see 5*1MiB*3=15MiB deductions of +-- {regular,elastic} tokens with no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 9.0 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 15 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 9.0 MiB + kvflowcontrol.tokens.send.elastic.deducted | 15 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe the total tracked tokens per-stream on n1. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 5.0 MiB + 70 | 2 | 5.0 MiB + 70 | 3 | 5.0 MiB + + +-- Observe the individual tracked tokens per-stream on the scratch range. +SELECT range_id, store_id, priority, crdb_internal.humanize_bytes(tokens::INT8) + FROM crdb_internal.kv_flow_token_deductions_v2 + + range_id | store_id | priority | tokens +-----------+----------+----------+---------- + 70 | 1 | low-pri | 1.0 MiB + 70 | 1 | low-pri | 1.0 MiB + 70 | 1 | low-pri | 1.0 MiB + 70 | 1 | low-pri | 1.0 MiB + 70 | 1 | low-pri | 1.0 MiB + 70 | 2 | low-pri | 1.0 MiB + 70 | 2 | low-pri | 1.0 MiB + 70 | 2 | low-pri | 1.0 MiB + 70 | 2 | low-pri | 1.0 MiB + 70 | 2 | low-pri | 1.0 MiB + 70 | 3 | low-pri | 1.0 MiB + 70 | 3 | low-pri | 1.0 MiB + 70 | 3 | low-pri | 1.0 MiB + 70 | 3 | low-pri | 1.0 MiB + 70 | 3 | low-pri | 1.0 MiB + + +-- (Allow below-raft admission to proceed.) + + +-- Flow token metrics from n1 after work gets admitted. We see 15MiB returns of +-- {regular,elastic} tokens, and the available capacities going back to what +-- they were. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 15 MiB + kvflowcontrol.tokens.eval.elastic.returned | 15 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 15 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 15 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v2_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v2_encoding_apply_to_all similarity index 84% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v2_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v2_encoding_apply_to_all index 8569cde166b5..227fbe8edaf7 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v2_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v2_encoding_apply_to_all @@ -1,10 +1,10 @@ echo ---- ---- --- (Issuing 5 regular 1MiB, 3x replicated write that's not admitted.) +-- (Issuing 5 1MiB, 3x replicated write that's not admitted.) --- Flow token metrics from n1 after issuing 5 regular 1MiB 3x replicated writes +-- Flow token metrics from n1 after issuing 5 1MiB 3x replicated writes -- that are yet to get admitted. We see 5*1MiB*3=15MiB deductions of -- {regular,elastic} tokens with no corresponding returns. SELECT name, crdb_internal.humanize_bytes(value::INT8) @@ -52,23 +52,23 @@ SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::IN SELECT range_id, store_id, priority, crdb_internal.humanize_bytes(tokens::INT8) FROM crdb_internal.kv_flow_token_deductions_v2 - range_id | store_id | priority | tokens ------------+----------+------------+---------- - 70 | 1 | normal-pri | 1.0 MiB - 70 | 1 | normal-pri | 1.0 MiB - 70 | 1 | normal-pri | 1.0 MiB - 70 | 1 | normal-pri | 1.0 MiB - 70 | 1 | normal-pri | 1.0 MiB - 70 | 2 | normal-pri | 1.0 MiB - 70 | 2 | normal-pri | 1.0 MiB - 70 | 2 | normal-pri | 1.0 MiB - 70 | 2 | normal-pri | 1.0 MiB - 70 | 2 | normal-pri | 1.0 MiB - 70 | 3 | normal-pri | 1.0 MiB - 70 | 3 | normal-pri | 1.0 MiB - 70 | 3 | normal-pri | 1.0 MiB - 70 | 3 | normal-pri | 1.0 MiB - 70 | 3 | normal-pri | 1.0 MiB + range_id | store_id | priority | tokens +-----------+----------+----------+---------- + 70 | 1 | high-pri | 1.0 MiB + 70 | 1 | high-pri | 1.0 MiB + 70 | 1 | high-pri | 1.0 MiB + 70 | 1 | high-pri | 1.0 MiB + 70 | 1 | high-pri | 1.0 MiB + 70 | 2 | high-pri | 1.0 MiB + 70 | 2 | high-pri | 1.0 MiB + 70 | 2 | high-pri | 1.0 MiB + 70 | 2 | high-pri | 1.0 MiB + 70 | 2 | high-pri | 1.0 MiB + 70 | 3 | high-pri | 1.0 MiB + 70 | 3 | high-pri | 1.0 MiB + 70 | 3 | high-pri | 1.0 MiB + 70 | 3 | high-pri | 1.0 MiB + 70 | 3 | high-pri | 1.0 MiB -- (Allow below-raft admission to proceed.) diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v2_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v2_encoding_apply_to_elastic new file mode 100644 index 000000000000..2b8d5b2db851 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/blocked_admission_v2_encoding_apply_to_elastic @@ -0,0 +1,111 @@ +echo +---- +---- +-- (Issuing 5 1MiB, 3x replicated write that's not admitted.) + + +-- Flow token metrics from n1 after issuing 5 1MiB 3x replicated writes +-- that are yet to get admitted. We see 5*1MiB*3=15MiB deductions of +-- {regular,elastic} tokens with no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 9.0 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 15 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 9.0 MiB + kvflowcontrol.tokens.send.elastic.deducted | 15 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe the total tracked tokens per-stream on n1. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 5.0 MiB + 70 | 2 | 5.0 MiB + 70 | 3 | 5.0 MiB + + +-- Observe the individual tracked tokens per-stream on the scratch range. +SELECT range_id, store_id, priority, crdb_internal.humanize_bytes(tokens::INT8) + FROM crdb_internal.kv_flow_token_deductions_v2 + + range_id | store_id | priority | tokens +-----------+----------+----------+---------- + 70 | 1 | low-pri | 1.0 MiB + 70 | 1 | low-pri | 1.0 MiB + 70 | 1 | low-pri | 1.0 MiB + 70 | 1 | low-pri | 1.0 MiB + 70 | 1 | low-pri | 1.0 MiB + 70 | 2 | low-pri | 1.0 MiB + 70 | 2 | low-pri | 1.0 MiB + 70 | 2 | low-pri | 1.0 MiB + 70 | 2 | low-pri | 1.0 MiB + 70 | 2 | low-pri | 1.0 MiB + 70 | 3 | low-pri | 1.0 MiB + 70 | 3 | low-pri | 1.0 MiB + 70 | 3 | low-pri | 1.0 MiB + 70 | 3 | low-pri | 1.0 MiB + 70 | 3 | low-pri | 1.0 MiB + + +-- (Allow below-raft admission to proceed.) + + +-- Flow token metrics from n1 after work gets admitted. We see 15MiB returns of +-- {regular,elastic} tokens, and the available capacities going back to what +-- they were. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 15 MiB + kvflowcontrol.tokens.eval.elastic.returned | 15 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 15 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 15 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v1_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v1_encoding_apply_to_all similarity index 97% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v1_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v1_encoding_apply_to_all index d44e51c025a9..e7097dba46e3 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v1_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v1_encoding_apply_to_all @@ -44,10 +44,10 @@ ORDER BY name ASC; kvflowcontrol.tokens.send.regular.unaccounted | 0 B --- (Issuing 1x1MiB, 3x replicated regular write that's not admitted.) +-- (Issuing 1x1MiB, 3x replicated write that's not admitted.) --- Flow token metrics from n1 after issuing 1x1MiB regular 3x replicated write +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write -- that's not admitted. We see 1*1MiB*3=3MiB deductions of {regular,elastic} -- tokens with no corresponding returns. SELECT name, crdb_internal.humanize_bytes(value::INT8) diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v1_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v1_encoding_apply_to_elastic new file mode 100644 index 000000000000..e7097dba46e3 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v1_encoding_apply_to_elastic @@ -0,0 +1,119 @@ +# -- When using v1 encoding (V2EnabledWhenLeaderV1Encoding), all entries which +# -- are subject to admission control are encoded as `raftpb.LowPri`, +# -- regardless of their original priority; to avoid the overhead of +# -- deserializing the raft admission metadata. Therefore, as the underlying +# -- test is shared between the v1 and v2 encoding testdata files, the reader +# -- should interpret any comments referring to regular tokens as referring to +# -- elastic token. +echo +---- +---- +-- (Issuing 1x1MiB, 3x replicated elastic write that's not admitted.) + + +-- Flow token metrics from n1 after issuing 1x1MiB elastic 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of elastic tokens with +-- no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 21 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Issuing 1x1MiB, 3x replicated write that's not admitted.) + + +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of {regular,elastic} +-- tokens with no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 18 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 18 MiB + kvflowcontrol.tokens.send.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Allow below-raft admission to proceed.) + + +-- Flow token metrics from n1 after work gets admitted. All {regular,elastic} +-- tokens deducted are returned. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 6.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 6.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v2_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v2_encoding_apply_to_all similarity index 95% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v2_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v2_encoding_apply_to_all index 2166180b1041..0bbaae93460b 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v2_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v2_encoding_apply_to_all @@ -17,8 +17,8 @@ ORDER BY name ASC; kvflowcontrol.tokens.eval.elastic.returned | 0 B kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B - kvflowcontrol.tokens.eval.regular.available | 48 MiB - kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.available | 45 MiB + kvflowcontrol.tokens.eval.regular.deducted | 3.0 MiB kvflowcontrol.tokens.eval.regular.returned | 0 B kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B kvflowcontrol.tokens.eval.regular.unaccounted | 0 B @@ -29,18 +29,18 @@ ORDER BY name ASC; kvflowcontrol.tokens.send.elastic.returned | 0 B kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B kvflowcontrol.tokens.send.elastic.unaccounted | 0 B - kvflowcontrol.tokens.send.regular.available | 48 MiB - kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.available | 45 MiB + kvflowcontrol.tokens.send.regular.deducted | 3.0 MiB kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B kvflowcontrol.tokens.send.regular.returned | 0 B kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B kvflowcontrol.tokens.send.regular.unaccounted | 0 B --- (Issuing 1x1MiB, 3x replicated regular write that's not admitted.) +-- (Issuing 1x1MiB, 3x replicated write that's not admitted.) --- Flow token metrics from n1 after issuing 1x1MiB regular 3x replicated write +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write -- that's not admitted. We see 1*1MiB*3=3MiB deductions of {regular,elastic} -- tokens with no corresponding returns. SELECT name, crdb_internal.humanize_bytes(value::INT8) @@ -53,8 +53,8 @@ ORDER BY name ASC; kvflowcontrol.tokens.eval.elastic.returned | 0 B kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B - kvflowcontrol.tokens.eval.regular.available | 45 MiB - kvflowcontrol.tokens.eval.regular.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.regular.available | 42 MiB + kvflowcontrol.tokens.eval.regular.deducted | 6.0 MiB kvflowcontrol.tokens.eval.regular.returned | 0 B kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B kvflowcontrol.tokens.eval.regular.unaccounted | 0 B @@ -65,8 +65,8 @@ ORDER BY name ASC; kvflowcontrol.tokens.send.elastic.returned | 0 B kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B kvflowcontrol.tokens.send.elastic.unaccounted | 0 B - kvflowcontrol.tokens.send.regular.available | 45 MiB - kvflowcontrol.tokens.send.regular.deducted | 3.0 MiB + kvflowcontrol.tokens.send.regular.available | 42 MiB + kvflowcontrol.tokens.send.regular.deducted | 6.0 MiB kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B kvflowcontrol.tokens.send.regular.returned | 0 B kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B @@ -89,8 +89,8 @@ ORDER BY name ASC; kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B kvflowcontrol.tokens.eval.regular.available | 48 MiB - kvflowcontrol.tokens.eval.regular.deducted | 3.0 MiB - kvflowcontrol.tokens.eval.regular.returned | 3.0 MiB + kvflowcontrol.tokens.eval.regular.deducted | 6.0 MiB + kvflowcontrol.tokens.eval.regular.returned | 6.0 MiB kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B kvflowcontrol.tokens.eval.regular.unaccounted | 0 B kvflowcontrol.tokens.send.elastic.available | 24 MiB @@ -101,9 +101,9 @@ ORDER BY name ASC; kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B kvflowcontrol.tokens.send.elastic.unaccounted | 0 B kvflowcontrol.tokens.send.regular.available | 48 MiB - kvflowcontrol.tokens.send.regular.deducted | 3.0 MiB + kvflowcontrol.tokens.send.regular.deducted | 6.0 MiB kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B - kvflowcontrol.tokens.send.regular.returned | 3.0 MiB + kvflowcontrol.tokens.send.regular.returned | 6.0 MiB kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B kvflowcontrol.tokens.send.regular.unaccounted | 0 B ---- diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v2_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v2_encoding_apply_to_elastic new file mode 100644 index 000000000000..336764c4c489 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/class_prioritization_v2_encoding_apply_to_elastic @@ -0,0 +1,112 @@ +echo +---- +---- +-- (Issuing 1x1MiB, 3x replicated elastic write that's not admitted.) + + +-- Flow token metrics from n1 after issuing 1x1MiB elastic 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of elastic tokens with +-- no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 21 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Issuing 1x1MiB, 3x replicated write that's not admitted.) + + +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of {regular,elastic} +-- tokens with no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 18 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 18 MiB + kvflowcontrol.tokens.send.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Allow below-raft admission to proceed.) + + +-- Flow token metrics from n1 after work gets admitted. All {regular,elastic} +-- tokens deducted are returned. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 6.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 6.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v1_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v1_encoding_apply_to_all similarity index 97% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v1_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v1_encoding_apply_to_all index e77ea61292dd..77eef5b234b3 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v1_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v1_encoding_apply_to_all @@ -8,10 +8,10 @@ echo ---- ---- --- (Issuing regular 5x1MiB, 2x replicated writes that are not admitted.) +-- (Issuing 5x1MiB, 2x replicated writes that are not admitted.) --- Flow token metrics from n1 after issuing 5 regular 1MiB 2x replicated writes +-- Flow token metrics from n1 after issuing 5 1MiB 2x replicated writes -- that are yet to get admitted. We see 5*1MiB*2=10MiB deductions of -- {regular,elastic} tokens with no corresponding returns. SELECT name, crdb_internal.humanize_bytes(value::INT8) diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v1_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v1_encoding_apply_to_elastic new file mode 100644 index 000000000000..77eef5b234b3 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v1_encoding_apply_to_elastic @@ -0,0 +1,103 @@ +# -- When using v1 encoding (V2EnabledWhenLeaderV1Encoding), all entries which +# -- are subject to admission control are encoded as `raftpb.LowPri`, +# -- regardless of their original priority; to avoid the overhead of +# -- deserializing the raft admission metadata. Therefore, as the underlying +# -- test is shared between the v1 and v2 encoding testdata files, the reader +# -- should interpret any comments referring to regular tokens as referring to +# -- elastic token. +echo +---- +---- +-- (Issuing 5x1MiB, 2x replicated writes that are not admitted.) + + +-- Flow token metrics from n1 after issuing 5 1MiB 2x replicated writes +-- that are yet to get admitted. We see 5*1MiB*2=10MiB deductions of +-- {regular,elastic} tokens with no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 6.0 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 10 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 32 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 6.0 MiB + kvflowcontrol.tokens.send.elastic.deducted | 10 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 32 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe the per-stream tracked tokens on n1, before n2 is crashed. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 5.0 MiB + 70 | 2 | 5.0 MiB + + +-- (Crashing n2) + + +-- Observe the per-stream tracked tokens on n1, after n2 crashed. We're no +-- longer tracking the 5MiB held by n2. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 5.0 MiB + + +-- Flow token metrics from n1 after n2 crashed. Observe that we've returned the +-- 5MiB previously held by n2. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 11 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 10 MiB + kvflowcontrol.tokens.eval.elastic.returned | 5.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 5.0 MiB + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 32 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 11 MiB + kvflowcontrol.tokens.send.elastic.deducted | 10 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 5.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 5.0 MiB + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 32 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v2_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v2_encoding_apply_to_all similarity index 97% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v2_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v2_encoding_apply_to_all index 91c26c57f85b..7ef89b1252c8 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v2_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v2_encoding_apply_to_all @@ -1,10 +1,10 @@ echo ---- ---- --- (Issuing regular 5x1MiB, 2x replicated writes that are not admitted.) +-- (Issuing 5x1MiB, 2x replicated writes that are not admitted.) --- Flow token metrics from n1 after issuing 5 regular 1MiB 2x replicated writes +-- Flow token metrics from n1 after issuing 5 1MiB 2x replicated writes -- that are yet to get admitted. We see 5*1MiB*2=10MiB deductions of -- {regular,elastic} tokens with no corresponding returns. SELECT name, crdb_internal.humanize_bytes(value::INT8) diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v2_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v2_encoding_apply_to_elastic new file mode 100644 index 000000000000..0d85c359ce46 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/crashed_node_v2_encoding_apply_to_elastic @@ -0,0 +1,96 @@ +echo +---- +---- +-- (Issuing 5x1MiB, 2x replicated writes that are not admitted.) + + +-- Flow token metrics from n1 after issuing 5 1MiB 2x replicated writes +-- that are yet to get admitted. We see 5*1MiB*2=10MiB deductions of +-- {regular,elastic} tokens with no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 6.0 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 10 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 32 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 6.0 MiB + kvflowcontrol.tokens.send.elastic.deducted | 10 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 32 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe the per-stream tracked tokens on n1, before n2 is crashed. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 5.0 MiB + 70 | 2 | 5.0 MiB + + +-- (Crashing n2) + + +-- Observe the per-stream tracked tokens on n1, after n2 crashed. We're no +-- longer tracking the 5MiB held by n2. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 5.0 MiB + + +-- Flow token metrics from n1 after n2 crashed. Observe that we've returned the +-- 5MiB previously held by n2. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 11 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 10 MiB + kvflowcontrol.tokens.eval.elastic.returned | 5.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 5.0 MiB + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 32 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 11 MiB + kvflowcontrol.tokens.send.elastic.deducted | 10 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 5.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 5.0 MiB + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 32 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v1_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v1_encoding_apply_to_all similarity index 98% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v1_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v1_encoding_apply_to_all index ed06a48a3aed..784b55a88672 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v1_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v1_encoding_apply_to_all @@ -8,7 +8,7 @@ echo ---- ---- --- (Issuing regular 1024*1KiB, 3x replicated writes that are not admitted.) +-- (Issuing 1024*1KiB, 3x replicated writes that are not admitted.) -- Flow token metrics from n1 after issuing 1024KiB, i.e. 1MiB 3x replicated writes diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v1_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v1_encoding_apply_to_elastic new file mode 100644 index 000000000000..784b55a88672 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v1_encoding_apply_to_elastic @@ -0,0 +1,95 @@ +# -- When using v1 encoding (V2EnabledWhenLeaderV1Encoding), all entries which +# -- are subject to admission control are encoded as `raftpb.LowPri`, +# -- regardless of their original priority; to avoid the overhead of +# -- deserializing the raft admission metadata. Therefore, as the underlying +# -- test is shared between the v1 and v2 encoding testdata files, the reader +# -- should interpret any comments referring to regular tokens as referring to +# -- elastic token. +echo +---- +---- +-- (Issuing 1024*1KiB, 3x replicated writes that are not admitted.) + + +-- Flow token metrics from n1 after issuing 1024KiB, i.e. 1MiB 3x replicated writes +-- that are yet to get admitted. We see 3*1MiB=3MiB deductions of +-- {regular,elastic} tokens with no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 21 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe the total tracked tokens per-stream on n1. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 1.0 MiB + 70 | 2 | 1.0 MiB + 70 | 3 | 1.0 MiB + + +-- (Allow below-raft admission to proceed.) + + +-- Flow token metrics from n1 after work gets admitted. We see 3MiB returns of +-- {regular,elastic} tokens, and the available capacities going back to what +-- they were. In #105185, by now we would've observed panics. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v2_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v2_encoding_apply_to_all similarity index 98% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v2_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v2_encoding_apply_to_all index 9683778500f4..2f93592d68f7 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v2_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v2_encoding_apply_to_all @@ -1,7 +1,7 @@ echo ---- ---- --- (Issuing regular 1024*1KiB, 3x replicated writes that are not admitted.) +-- (Issuing 1024*1KiB, 3x replicated writes that are not admitted.) -- Flow token metrics from n1 after issuing 1024KiB, i.e. 1MiB 3x replicated writes diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v2_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v2_encoding_apply_to_elastic new file mode 100644 index 000000000000..2060e602ed12 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/granter_admit_one_by_one_v2_encoding_apply_to_elastic @@ -0,0 +1,88 @@ +echo +---- +---- +-- (Issuing 1024*1KiB, 3x replicated writes that are not admitted.) + + +-- Flow token metrics from n1 after issuing 1024KiB, i.e. 1MiB 3x replicated writes +-- that are yet to get admitted. We see 3*1MiB=3MiB deductions of +-- {regular,elastic} tokens with no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 21 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe the total tracked tokens per-stream on n1. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 1.0 MiB + 70 | 2 | 1.0 MiB + 70 | 3 | 1.0 MiB + + +-- (Allow below-raft admission to proceed.) + + +-- Flow token metrics from n1 after work gets admitted. We see 3MiB returns of +-- {regular,elastic} tokens, and the available capacities going back to what +-- they were. In #105185, by now we would've observed panics. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v1_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v1_encoding_apply_to_all similarity index 98% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v1_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v1_encoding_apply_to_all index 0fe403459237..92402d898af5 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v1_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v1_encoding_apply_to_all @@ -11,8 +11,8 @@ echo -- (Issuing 1x1MiB, 3x replicated write that's not admitted.) --- Flow token metrics from n1 after issuing 1x1MiB regular 3x replicated write --- that's not admitted. We see 1*1MiB*3=3MiB deductions of regular tokens with +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with -- no corresponding returns. SELECT name, crdb_internal.humanize_bytes(value::INT8) FROM crdb_internal.node_metrics diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v1_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v1_encoding_apply_to_elastic new file mode 100644 index 000000000000..92402d898af5 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v1_encoding_apply_to_elastic @@ -0,0 +1,182 @@ +# -- When using v1 encoding (V2EnabledWhenLeaderV1Encoding), all entries which +# -- are subject to admission control are encoded as `raftpb.LowPri`, +# -- regardless of their original priority; to avoid the overhead of +# -- deserializing the raft admission metadata. Therefore, as the underlying +# -- test is shared between the v1 and v2 encoding testdata files, the reader +# -- should interpret any comments referring to regular tokens as referring to +# -- elastic token. +echo +---- +---- +-- (Issuing 1x1MiB, 3x replicated write that's not admitted.) + + +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with +-- no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 21 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Transferring only range lease, not raft leadership, to n2.) + + +-- Flow token metrics from n1 having lost the lease but retained raft +-- leadership. No deducted tokens are released. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 21 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Allow below-raft admission to proceed. All tokens should be returned.) +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Issuing another 1x1MiB, 3x replicated write that's admitted via +-- the work queue on the leaseholder. It shouldn't deduct any tokens.) + + +-- Looking at n1's flow token metrics, there's no change. No additional tokens +-- are deducted since the write is not being proposed here. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Looking at n2's flow token metrics, there's no activity. n2 never acquired +-- the raft leadership. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 0 B + kvflowcontrol.tokens.eval.elastic.deducted | 0 B + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 0 B + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 0 B + kvflowcontrol.tokens.send.elastic.deducted | 0 B + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 0 B + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v2_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v2_encoding_apply_to_all similarity index 98% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v2_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v2_encoding_apply_to_all index caf3541ae84d..01ba53599b90 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v2_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v2_encoding_apply_to_all @@ -4,8 +4,8 @@ echo -- (Issuing 1x1MiB, 3x replicated write that's not admitted.) --- Flow token metrics from n1 after issuing 1x1MiB regular 3x replicated write --- that's not admitted. We see 1*1MiB*3=3MiB deductions of regular tokens with +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with -- no corresponding returns. SELECT name, crdb_internal.humanize_bytes(value::INT8) FROM crdb_internal.node_metrics diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v2_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v2_encoding_apply_to_elastic new file mode 100644 index 000000000000..1bfc45745927 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/leader_not_leaseholder_v2_encoding_apply_to_elastic @@ -0,0 +1,175 @@ +echo +---- +---- +-- (Issuing 1x1MiB, 3x replicated write that's not admitted.) + + +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with +-- no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 21 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Transferring only range lease, not raft leadership, to n2.) + + +-- Flow token metrics from n1 having lost the lease but retained raft +-- leadership. No deducted tokens are released. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 21 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Allow below-raft admission to proceed. All tokens should be returned.) +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Issuing another 1x1MiB, 3x replicated write that's admitted via +-- the work queue on the leaseholder. It shouldn't deduct any tokens.) + + +-- Looking at n1's flow token metrics, there's no change. No additional tokens +-- are deducted since the write is not being proposed here. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Looking at n2's flow token metrics, there's no activity. n2 never acquired +-- the raft leadership. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 0 B + kvflowcontrol.tokens.eval.elastic.deducted | 0 B + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 0 B + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 0 B + kvflowcontrol.tokens.send.elastic.deducted | 0 B + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 0 B + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v1_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v1_encoding_apply_to_all similarity index 97% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v1_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v1_encoding_apply_to_all index 7a00c9668b0d..d4f7a9ed4fe1 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v1_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v1_encoding_apply_to_all @@ -11,8 +11,8 @@ echo -- (Issuing 1x1MiB, 3x replicated write that's not admitted.) --- Flow token metrics from n1 after issuing 1x1MiB regular 3x replicated write --- that's not admitted. We see 1*1MiB*3=3MiB deductions of regular tokens with +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with -- no corresponding returns. SELECT name, crdb_internal.humanize_bytes(value::INT8) FROM crdb_internal.node_metrics diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v1_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v1_encoding_apply_to_elastic new file mode 100644 index 000000000000..d4f7a9ed4fe1 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v1_encoding_apply_to_elastic @@ -0,0 +1,143 @@ +# -- When using v1 encoding (V2EnabledWhenLeaderV1Encoding), all entries which +# -- are subject to admission control are encoded as `raftpb.LowPri`, +# -- regardless of their original priority; to avoid the overhead of +# -- deserializing the raft admission metadata. Therefore, as the underlying +# -- test is shared between the v1 and v2 encoding testdata files, the reader +# -- should interpret any comments referring to regular tokens as referring to +# -- elastic token. +echo +---- +---- +-- (Issuing 1x1MiB, 3x replicated write that's not admitted.) + + +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with +-- no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 21 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Replacing current raft leader on n1 in raft group with new n4 replica.) + + +-- Flow token metrics from n1 after raft leader removed itself from raft group. +-- All {regular,elastic} tokens deducted are returned. Note that the available +-- tokens increases, as n1 has seen 4 replication streams, s1,s2,s3,s4. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 32 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 64 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 32 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 3.0 MiB + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 64 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- n1 should have no connected streams now after transferring the lease to n4. +-- While, n4 should have 3 connected streams to s2,s3,s4. Query the stream count +-- on n1, then on n4. +-- n1 connected v2 streams: +SELECT range_id, count(*) AS streams + FROM crdb_internal.kv_flow_control_handles_v2 +GROUP BY (range_id) +ORDER BY streams DESC; + + range_id | stream_count +-----------+--------------- + + +-- n4 connected v2 streams: +SELECT range_id, count(*) AS streams + FROM crdb_internal.kv_flow_control_handles_v2 +GROUP BY (range_id) +ORDER BY streams DESC; + + range_id | stream_count +-----------+--------------- + 70 | 3 + + +-- (Allow below-raft admission to proceed.) + + +-- Flow token metrics from n1 after work gets admitted. Tokens were already +-- returned earlier, so there's no change. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 32 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 64 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 32 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 3.0 MiB + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 64 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v2_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v2_encoding_apply_to_all similarity index 97% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v2_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v2_encoding_apply_to_all index 856e3c93a996..e1f53449b81d 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v2_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v2_encoding_apply_to_all @@ -4,8 +4,8 @@ echo -- (Issuing 1x1MiB, 3x replicated write that's not admitted.) --- Flow token metrics from n1 after issuing 1x1MiB regular 3x replicated write --- that's not admitted. We see 1*1MiB*3=3MiB deductions of regular tokens with +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with -- no corresponding returns. SELECT name, crdb_internal.humanize_bytes(value::INT8) FROM crdb_internal.node_metrics diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v2_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v2_encoding_apply_to_elastic new file mode 100644 index 000000000000..3cc9b15f943b --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_remove_self_v2_encoding_apply_to_elastic @@ -0,0 +1,136 @@ +echo +---- +---- +-- (Issuing 1x1MiB, 3x replicated write that's not admitted.) + + +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with +-- no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 21 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Replacing current raft leader on n1 in raft group with new n4 replica.) + + +-- Flow token metrics from n1 after raft leader removed itself from raft group. +-- All {regular,elastic} tokens deducted are returned. Note that the available +-- tokens increases, as n1 has seen 4 replication streams, s1,s2,s3,s4. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 32 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 64 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 32 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 3.0 MiB + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 64 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- n1 should have no connected streams now after transferring the lease to n4. +-- While, n4 should have 3 connected streams to s2,s3,s4. Query the stream count +-- on n1, then on n4. +-- n1 connected v2 streams: +SELECT range_id, count(*) AS streams + FROM crdb_internal.kv_flow_control_handles_v2 +GROUP BY (range_id) +ORDER BY streams DESC; + + range_id | stream_count +-----------+--------------- + + +-- n4 connected v2 streams: +SELECT range_id, count(*) AS streams + FROM crdb_internal.kv_flow_control_handles_v2 +GROUP BY (range_id) +ORDER BY streams DESC; + + range_id | stream_count +-----------+--------------- + 70 | 3 + + +-- (Allow below-raft admission to proceed.) + + +-- Flow token metrics from n1 after work gets admitted. Tokens were already +-- returned earlier, so there's no change. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 32 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 64 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 32 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 3.0 MiB + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 64 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v1_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v1_encoding_apply_to_all similarity index 97% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v1_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v1_encoding_apply_to_all index 42c61145e99b..404193fde651 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v1_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v1_encoding_apply_to_all @@ -11,8 +11,8 @@ echo -- (Issuing 1x1MiB, 3x replicated write that's not admitted.) --- Flow token metrics from n1 after issuing 1x1MiB regular 3x replicated write --- that's not admitted. We see 1*1MiB*3=3MiB deductions of regular tokens with +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with -- no corresponding returns. SELECT name, crdb_internal.humanize_bytes(value::INT8) FROM crdb_internal.node_metrics diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v1_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v1_encoding_apply_to_elastic new file mode 100644 index 000000000000..404193fde651 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v1_encoding_apply_to_elastic @@ -0,0 +1,153 @@ +# -- When using v1 encoding (V2EnabledWhenLeaderV1Encoding), all entries which +# -- are subject to admission control are encoded as `raftpb.LowPri`, +# -- regardless of their original priority; to avoid the overhead of +# -- deserializing the raft admission metadata. Therefore, as the underlying +# -- test is shared between the v1 and v2 encoding testdata files, the reader +# -- should interpret any comments referring to regular tokens as referring to +# -- elastic token. +echo +---- +---- +-- (Issuing 1x1MiB, 3x replicated write that's not admitted.) + + +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with +-- no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 21 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Adding a voting replica on n4.) + + +-- Observe the total tracked tokens per-stream on n1. s1-s3 should have 1MiB +-- tracked each, and s4 should have none. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 1.0 MiB + 70 | 2 | 1.0 MiB + 70 | 3 | 1.0 MiB + 70 | 4 | 0 B + + +-- (Issuing 1x1MiB, 4x replicated write that's not admitted.) + + +-- Observe the individual tracked tokens per-stream on the scratch range. s1-s3 +-- should have 2MiB tracked (they've observed 2x1MiB writes), s4 should have +-- 1MiB. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 2.0 MiB + 70 | 2 | 2.0 MiB + 70 | 3 | 2.0 MiB + 70 | 4 | 1.0 MiB + + +-- (Removing voting replica from n3.) + + +-- (Adding non-voting replica to n5.) + + +-- (Issuing 1x1MiB, 4x replicated write (w/ one non-voter) that's not admitted. + + +-- Observe the individual tracked tokens per-stream on the scratch range. s1-s2 +-- should have 3MiB tracked (they've observed 3x1MiB writes), there should be +-- no s3 since it was removed, s4 and s5 should have 2MiB and 1MiB +-- respectively. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 3.0 MiB + 70 | 2 | 3.0 MiB + 70 | 4 | 2.0 MiB + 70 | 5 | 1.0 MiB + + +-- (Allow below-raft admission to proceed.) + + +-- Observe that there no tracked tokens across s1,s2,s4,s5. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 0 B + 70 | 2 | 0 B + 70 | 4 | 0 B + 70 | 5 | 0 B + + +-- Flow token metrics from n1 after work gets admitted. All {regular,elastic} +-- tokens deducted are returned, including from when s3 was removed as a raft +-- member. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 40 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 11 MiB + kvflowcontrol.tokens.eval.elastic.returned | 11 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 2.0 MiB + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 80 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 40 MiB + kvflowcontrol.tokens.send.elastic.deducted | 11 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 11 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 2.0 MiB + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 80 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v2_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v2_encoding_apply_to_all similarity index 97% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v2_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v2_encoding_apply_to_all index 9bbe7e27a78e..be3e63b11579 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v2_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v2_encoding_apply_to_all @@ -4,8 +4,8 @@ echo -- (Issuing 1x1MiB, 3x replicated write that's not admitted.) --- Flow token metrics from n1 after issuing 1x1MiB regular 3x replicated write --- that's not admitted. We see 1*1MiB*3=3MiB deductions of regular tokens with +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with -- no corresponding returns. SELECT name, crdb_internal.humanize_bytes(value::INT8) FROM crdb_internal.node_metrics diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v2_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v2_encoding_apply_to_elastic new file mode 100644 index 000000000000..3552c2a6a719 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_membership_v2_encoding_apply_to_elastic @@ -0,0 +1,146 @@ +echo +---- +---- +-- (Issuing 1x1MiB, 3x replicated write that's not admitted.) + + +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with +-- no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 21 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Adding a voting replica on n4.) + + +-- Observe the total tracked tokens per-stream on n1. s1-s3 should have 1MiB +-- tracked each, and s4 should have none. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 1.0 MiB + 70 | 2 | 1.0 MiB + 70 | 3 | 1.0 MiB + 70 | 4 | 0 B + + +-- (Issuing 1x1MiB, 4x replicated write that's not admitted.) + + +-- Observe the individual tracked tokens per-stream on the scratch range. s1-s3 +-- should have 2MiB tracked (they've observed 2x1MiB writes), s4 should have +-- 1MiB. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 2.0 MiB + 70 | 2 | 2.0 MiB + 70 | 3 | 2.0 MiB + 70 | 4 | 1.0 MiB + + +-- (Removing voting replica from n3.) + + +-- (Adding non-voting replica to n5.) + + +-- (Issuing 1x1MiB, 4x replicated write (w/ one non-voter) that's not admitted. + + +-- Observe the individual tracked tokens per-stream on the scratch range. s1-s2 +-- should have 3MiB tracked (they've observed 3x1MiB writes), there should be +-- no s3 since it was removed, s4 and s5 should have 2MiB and 1MiB +-- respectively. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 3.0 MiB + 70 | 2 | 3.0 MiB + 70 | 4 | 2.0 MiB + 70 | 5 | 1.0 MiB + + +-- (Allow below-raft admission to proceed.) + + +-- Observe that there no tracked tokens across s1,s2,s4,s5. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 0 B + 70 | 2 | 0 B + 70 | 4 | 0 B + 70 | 5 | 0 B + + +-- Flow token metrics from n1 after work gets admitted. All {regular,elastic} +-- tokens deducted are returned, including from when s3 was removed as a raft +-- member. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 40 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 11 MiB + kvflowcontrol.tokens.eval.elastic.returned | 11 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 2.0 MiB + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 80 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 40 MiB + kvflowcontrol.tokens.send.elastic.deducted | 11 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 11 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 2.0 MiB + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 80 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v1_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v1_encoding_apply_to_all similarity index 97% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v1_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v1_encoding_apply_to_all index 09245c7514e2..c1f874858793 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v1_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v1_encoding_apply_to_all @@ -8,9 +8,9 @@ echo ---- ---- --- Flow token metrics from n1 after issuing 1 regular 1MiB 5x replicated write +-- Flow token metrics from n1 after issuing 1 1MiB 5x replicated write -- that's not admitted. Since this test is ignoring crashed nodes for token --- deduction purposes, we see a deduction of 5MiB {regular,elastic} tokens. +-- deduction purposes, we see a deduction of 5MiB tokens. SELECT name, crdb_internal.humanize_bytes(value::INT8) FROM crdb_internal.node_metrics WHERE name LIKE '%kvflowcontrol%tokens%' @@ -77,7 +77,7 @@ SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::IN -- below-raft admission is paused.) --- Flow token metrics from n1 after issuing 1 regular 1MiB 5x replicated write +-- Flow token metrics from n1 after issuing 1 1MiB 5x replicated write -- that's not admitted. We'll have deducted another 5*1MiB=5MiB worth of tokens. SELECT name, crdb_internal.humanize_bytes(value::INT8) FROM crdb_internal.node_metrics @@ -180,7 +180,7 @@ SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::IN -- Flow token metrics from n1 after work gets admitted. We see the remaining --- 6MiB of {regular,elastic} tokens returned. +-- 6MiB of tokens returned. SELECT name, crdb_internal.humanize_bytes(value::INT8) FROM crdb_internal.node_metrics WHERE name LIKE '%kvflowcontrol%tokens%' diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v1_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v1_encoding_apply_to_elastic new file mode 100644 index 000000000000..c1f874858793 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v1_encoding_apply_to_elastic @@ -0,0 +1,244 @@ +# -- When using v1 encoding (V2EnabledWhenLeaderV1Encoding), all entries which +# -- are subject to admission control are encoded as `raftpb.LowPri`, +# -- regardless of their original priority; to avoid the overhead of +# -- deserializing the raft admission metadata. Therefore, as the underlying +# -- test is shared between the v1 and v2 encoding testdata files, the reader +# -- should interpret any comments referring to regular tokens as referring to +# -- elastic token. +echo +---- +---- +-- Flow token metrics from n1 after issuing 1 1MiB 5x replicated write +-- that's not admitted. Since this test is ignoring crashed nodes for token +-- deduction purposes, we see a deduction of 5MiB tokens. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 35 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 5.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 80 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 35 MiB + kvflowcontrol.tokens.send.elastic.deducted | 5.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 80 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe the total tracked tokens per-stream on n1. 1MiB is tracked for n1-n5. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 1.0 MiB + 70 | 2 | 1.0 MiB + 70 | 3 | 1.0 MiB + 70 | 4 | 1.0 MiB + 70 | 5 | 1.0 MiB + + +-- (Killing n2 and n3, but preventing their tokens from being returned + +-- artificially allowing tokens to get deducted.) + + +-- Observe the total tracked tokens per-stream on n1. 1MiB is (still) tracked +-- for n1-n5, because they are not in StateSnapshot yet and have likely been +-- in StateProbe for less than the close timer. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 1.0 MiB + 70 | 2 | 1.0 MiB + 70 | 3 | 1.0 MiB + 70 | 4 | 1.0 MiB + 70 | 5 | 1.0 MiB + + +-- (Issuing another 1MiB of 5x replicated writes while n2 and n3 are down and +-- below-raft admission is paused.) + + +-- Flow token metrics from n1 after issuing 1 1MiB 5x replicated write +-- that's not admitted. We'll have deducted another 5*1MiB=5MiB worth of tokens. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 30 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 10 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 80 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 30 MiB + kvflowcontrol.tokens.send.elastic.deducted | 10 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 80 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe the total tracked tokens per-stream on n1. 2MiB is tracked for n1-n5; +-- see last comment for an explanation why we're still deducting for n2, n3. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 2.0 MiB + 70 | 2 | 2.0 MiB + 70 | 3 | 2.0 MiB + 70 | 4 | 2.0 MiB + 70 | 5 | 2.0 MiB + + +-- (Truncating raft log.) + + +-- (Restarting n2 and n3.) + + +-- Flow token metrics from n1 after restarting n2 and n3. We've returned the +-- 2MiB previously held by those nodes (2MiB each). We're reacting to it's raft +-- progress state, noting that since we've truncated our log, we need to catch +-- it up via snapshot. So we release all held tokens. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 34 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 10 MiB + kvflowcontrol.tokens.eval.elastic.returned | 4.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 4.0 MiB + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 80 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 34 MiB + kvflowcontrol.tokens.send.elastic.deducted | 10 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 4.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 4.0 MiB + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 80 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe the total tracked tokens per-stream on n1. There's nothing tracked +-- for n2 and n3 anymore. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + WHERE total_tracked_tokens > 0 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 2.0 MiB + 70 | 4 | 2.0 MiB + 70 | 5 | 2.0 MiB + + +-- (Allow below-raft admission to proceed.) + + +-- Flow token metrics from n1 after work gets admitted. We see the remaining +-- 6MiB of tokens returned. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 40 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 10 MiB + kvflowcontrol.tokens.eval.elastic.returned | 10 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 4.0 MiB + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 80 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 40 MiB + kvflowcontrol.tokens.send.elastic.deducted | 10 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 10 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 4.0 MiB + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 80 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe the total tracked tokens per-stream on n1; there should be nothing. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 0 B + 70 | 2 | 0 B + 70 | 3 | 0 B + 70 | 4 | 0 B + 70 | 5 | 0 B + + +-- Another view of tokens, using /inspectz-backed vtables. +SELECT store_id, + crdb_internal.humanize_bytes(available_eval_regular_tokens), + crdb_internal.humanize_bytes(available_eval_elastic_tokens) + FROM crdb_internal.kv_flow_controller_v2 + ORDER BY store_id ASC; + + range_id | eval_regular_available | eval_elastic_available +-----------+------------------------+------------------------- + 1 | 16 MiB | 8.0 MiB + 2 | 16 MiB | 8.0 MiB + 3 | 16 MiB | 8.0 MiB + 4 | 16 MiB | 8.0 MiB + 5 | 16 MiB | 8.0 MiB +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v2_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v2_encoding_apply_to_all similarity index 97% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v2_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v2_encoding_apply_to_all index c1dd8ff9c8bf..3ec8a6969ef7 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v2_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v2_encoding_apply_to_all @@ -1,9 +1,9 @@ echo ---- ---- --- Flow token metrics from n1 after issuing 1 regular 1MiB 5x replicated write +-- Flow token metrics from n1 after issuing 1 1MiB 5x replicated write -- that's not admitted. Since this test is ignoring crashed nodes for token --- deduction purposes, we see a deduction of 5MiB {regular,elastic} tokens. +-- deduction purposes, we see a deduction of 5MiB tokens. SELECT name, crdb_internal.humanize_bytes(value::INT8) FROM crdb_internal.node_metrics WHERE name LIKE '%kvflowcontrol%tokens%' @@ -70,7 +70,7 @@ SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::IN -- below-raft admission is paused.) --- Flow token metrics from n1 after issuing 1 regular 1MiB 5x replicated write +-- Flow token metrics from n1 after issuing 1 1MiB 5x replicated write -- that's not admitted. We'll have deducted another 5*1MiB=5MiB worth of tokens. SELECT name, crdb_internal.humanize_bytes(value::INT8) FROM crdb_internal.node_metrics @@ -173,7 +173,7 @@ SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::IN -- Flow token metrics from n1 after work gets admitted. We see the remaining --- 6MiB of {regular,elastic} tokens returned. +-- 6MiB of tokens returned. SELECT name, crdb_internal.humanize_bytes(value::INT8) FROM crdb_internal.node_metrics WHERE name LIKE '%kvflowcontrol%tokens%' diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v2_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v2_encoding_apply_to_elastic new file mode 100644 index 000000000000..9315162fcb63 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/raft_snapshot_v2_encoding_apply_to_elastic @@ -0,0 +1,237 @@ +echo +---- +---- +-- Flow token metrics from n1 after issuing 1 1MiB 5x replicated write +-- that's not admitted. Since this test is ignoring crashed nodes for token +-- deduction purposes, we see a deduction of 5MiB tokens. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 35 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 5.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 80 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 35 MiB + kvflowcontrol.tokens.send.elastic.deducted | 5.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 80 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe the total tracked tokens per-stream on n1. 1MiB is tracked for n1-n5. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 1.0 MiB + 70 | 2 | 1.0 MiB + 70 | 3 | 1.0 MiB + 70 | 4 | 1.0 MiB + 70 | 5 | 1.0 MiB + + +-- (Killing n2 and n3, but preventing their tokens from being returned + +-- artificially allowing tokens to get deducted.) + + +-- Observe the total tracked tokens per-stream on n1. 1MiB is (still) tracked +-- for n1-n5, because they are not in StateSnapshot yet and have likely been +-- in StateProbe for less than the close timer. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 1.0 MiB + 70 | 2 | 1.0 MiB + 70 | 3 | 1.0 MiB + 70 | 4 | 1.0 MiB + 70 | 5 | 1.0 MiB + + +-- (Issuing another 1MiB of 5x replicated writes while n2 and n3 are down and +-- below-raft admission is paused.) + + +-- Flow token metrics from n1 after issuing 1 1MiB 5x replicated write +-- that's not admitted. We'll have deducted another 5*1MiB=5MiB worth of tokens. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 30 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 10 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 80 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 30 MiB + kvflowcontrol.tokens.send.elastic.deducted | 10 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 80 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe the total tracked tokens per-stream on n1. 2MiB is tracked for n1-n5; +-- see last comment for an explanation why we're still deducting for n2, n3. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 2.0 MiB + 70 | 2 | 2.0 MiB + 70 | 3 | 2.0 MiB + 70 | 4 | 2.0 MiB + 70 | 5 | 2.0 MiB + + +-- (Truncating raft log.) + + +-- (Restarting n2 and n3.) + + +-- Flow token metrics from n1 after restarting n2 and n3. We've returned the +-- 2MiB previously held by those nodes (2MiB each). We're reacting to it's raft +-- progress state, noting that since we've truncated our log, we need to catch +-- it up via snapshot. So we release all held tokens. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 34 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 10 MiB + kvflowcontrol.tokens.eval.elastic.returned | 4.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 4.0 MiB + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 80 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 34 MiB + kvflowcontrol.tokens.send.elastic.deducted | 10 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 4.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 4.0 MiB + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 80 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe the total tracked tokens per-stream on n1. There's nothing tracked +-- for n2 and n3 anymore. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + WHERE total_tracked_tokens > 0 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 2.0 MiB + 70 | 4 | 2.0 MiB + 70 | 5 | 2.0 MiB + + +-- (Allow below-raft admission to proceed.) + + +-- Flow token metrics from n1 after work gets admitted. We see the remaining +-- 6MiB of tokens returned. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 40 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 10 MiB + kvflowcontrol.tokens.eval.elastic.returned | 10 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 4.0 MiB + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 80 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 40 MiB + kvflowcontrol.tokens.send.elastic.deducted | 10 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 10 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 4.0 MiB + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 80 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe the total tracked tokens per-stream on n1; there should be nothing. +SELECT range_id, store_id, crdb_internal.humanize_bytes(total_tracked_tokens::INT8) + FROM crdb_internal.kv_flow_control_handles_v2 + + range_id | store_id | total_tracked_tokens +-----------+----------+----------------------- + 70 | 1 | 0 B + 70 | 2 | 0 B + 70 | 3 | 0 B + 70 | 4 | 0 B + 70 | 5 | 0 B + + +-- Another view of tokens, using /inspectz-backed vtables. +SELECT store_id, + crdb_internal.humanize_bytes(available_eval_regular_tokens), + crdb_internal.humanize_bytes(available_eval_elastic_tokens) + FROM crdb_internal.kv_flow_controller_v2 + ORDER BY store_id ASC; + + range_id | eval_regular_available | eval_elastic_available +-----------+------------------------+------------------------- + 1 | 16 MiB | 8.0 MiB + 2 | 16 MiB | 8.0 MiB + 3 | 16 MiB | 8.0 MiB + 4 | 16 MiB | 8.0 MiB + 5 | 16 MiB | 8.0 MiB +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v1_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v1_encoding_apply_to_all similarity index 97% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v1_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v1_encoding_apply_to_all index fac8b292284f..87e963f43e62 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v1_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v1_encoding_apply_to_all @@ -8,7 +8,7 @@ echo ---- ---- --- Flow token metrics from n1 after issuing + admitting the regular 1MiB 3x +-- Flow token metrics from n1 after issuing + admitting the 1MiB 3x -- replicated write to the pre-split range. There should be 3MiB of -- {regular,elastic} tokens {deducted,returned}. SELECT name, crdb_internal.humanize_bytes(value::INT8) @@ -93,7 +93,7 @@ ORDER BY streams DESC; -- (Merging ranges.) --- Flow token metrics from n1 after issuing 4MiB of regular replicated writes to +-- Flow token metrics from n1 after issuing 4MiB of replicated writes to -- the post-merged range. We should see 12MiB extra tokens {deducted,returned}, -- which comes from 4MiB*3=12MiB. So we stand at 18MiB+12MiB=30MiB now. SELECT name, crdb_internal.humanize_bytes(value::INT8) diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v1_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v1_encoding_apply_to_elastic new file mode 100644 index 000000000000..87e963f43e62 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v1_encoding_apply_to_elastic @@ -0,0 +1,141 @@ +# -- When using v1 encoding (V2EnabledWhenLeaderV1Encoding), all entries which +# -- are subject to admission control are encoded as `raftpb.LowPri`, +# -- regardless of their original priority; to avoid the overhead of +# -- deserializing the raft admission metadata. Therefore, as the underlying +# -- test is shared between the v1 and v2 encoding testdata files, the reader +# -- should interpret any comments referring to regular tokens as referring to +# -- elastic token. +echo +---- +---- +-- Flow token metrics from n1 after issuing + admitting the 1MiB 3x +-- replicated write to the pre-split range. There should be 3MiB of +-- {regular,elastic} tokens {deducted,returned}. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Splitting range.) + + +-- Flow token metrics from n1 after further issuing 2MiB and 3MiB writes to +-- post-split LHS and RHS ranges respectively. We should see 15MiB extra tokens +-- {deducted,returned}, which comes from (2MiB+3MiB)*3=15MiB. So we stand at +-- 3MiB+15MiB=18MiB now. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 18 MiB + kvflowcontrol.tokens.eval.elastic.returned | 18 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 18 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 18 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe the newly split off replica, with its own three streams. +SELECT range_id, count(*) AS streams + FROM crdb_internal.kv_flow_control_handles_v2 +GROUP BY (range_id) +ORDER BY streams DESC; + + range_id | stream_count +-----------+--------------- + 70 | 3 + 71 | 3 + + +-- (Merging ranges.) + + +-- Flow token metrics from n1 after issuing 4MiB of replicated writes to +-- the post-merged range. We should see 12MiB extra tokens {deducted,returned}, +-- which comes from 4MiB*3=12MiB. So we stand at 18MiB+12MiB=30MiB now. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 30 MiB + kvflowcontrol.tokens.eval.elastic.returned | 30 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 30 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 30 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe only the merged replica with its own three streams. +SELECT range_id, count(*) AS streams + FROM crdb_internal.kv_flow_control_handles_v2 +GROUP BY (range_id) +ORDER BY streams DESC; + + range_id | stream_count +-----------+--------------- + 70 | 3 +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v2_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v2_encoding_apply_to_all similarity index 97% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v2_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v2_encoding_apply_to_all index 2ddb5433834a..9df35bb54ce4 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v2_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v2_encoding_apply_to_all @@ -1,7 +1,7 @@ echo ---- ---- --- Flow token metrics from n1 after issuing + admitting the regular 1MiB 3x +-- Flow token metrics from n1 after issuing + admitting the 1MiB 3x -- replicated write to the pre-split range. There should be 3MiB of -- {regular,elastic} tokens {deducted,returned}. SELECT name, crdb_internal.humanize_bytes(value::INT8) @@ -86,7 +86,7 @@ ORDER BY streams DESC; -- (Merging ranges.) --- Flow token metrics from n1 after issuing 4MiB of regular replicated writes to +-- Flow token metrics from n1 after issuing 4MiB of replicated writes to -- the post-merged range. We should see 12MiB extra tokens {deducted,returned}, -- which comes from 4MiB*3=12MiB. So we stand at 18MiB+12MiB=30MiB now. SELECT name, crdb_internal.humanize_bytes(value::INT8) diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v2_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v2_encoding_apply_to_elastic new file mode 100644 index 000000000000..f705e9eb67ce --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/split_merge_v2_encoding_apply_to_elastic @@ -0,0 +1,134 @@ +echo +---- +---- +-- Flow token metrics from n1 after issuing + admitting the 1MiB 3x +-- replicated write to the pre-split range. There should be 3MiB of +-- {regular,elastic} tokens {deducted,returned}. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Splitting range.) + + +-- Flow token metrics from n1 after further issuing 2MiB and 3MiB writes to +-- post-split LHS and RHS ranges respectively. We should see 15MiB extra tokens +-- {deducted,returned}, which comes from (2MiB+3MiB)*3=15MiB. So we stand at +-- 3MiB+15MiB=18MiB now. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 18 MiB + kvflowcontrol.tokens.eval.elastic.returned | 18 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 18 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 18 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe the newly split off replica, with its own three streams. +SELECT range_id, count(*) AS streams + FROM crdb_internal.kv_flow_control_handles_v2 +GROUP BY (range_id) +ORDER BY streams DESC; + + range_id | stream_count +-----------+--------------- + 70 | 3 + 71 | 3 + + +-- (Merging ranges.) + + +-- Flow token metrics from n1 after issuing 4MiB of replicated writes to +-- the post-merged range. We should see 12MiB extra tokens {deducted,returned}, +-- which comes from 4MiB*3=12MiB. So we stand at 18MiB+12MiB=30MiB now. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 30 MiB + kvflowcontrol.tokens.eval.elastic.returned | 30 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 30 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 30 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- Observe only the merged replica with its own three streams. +SELECT range_id, count(*) AS streams + FROM crdb_internal.kv_flow_control_handles_v2 +GROUP BY (range_id) +ORDER BY streams DESC; + + range_id | stream_count +-----------+--------------- + 70 | 3 +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v1_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v1_encoding_apply_to_all similarity index 96% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v1_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v1_encoding_apply_to_all index 628b6e794b63..b7ee8c1ba1e0 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v1_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v1_encoding_apply_to_all @@ -11,8 +11,8 @@ echo -- (Issuing 1x1MiB, 3x replicated write that's not admitted.) --- Flow token metrics from n1 after issuing 1x1MiB regular 3x replicated write --- that's not admitted. We see 1*1MiB*3=3MiB deductions of regular tokens with +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with -- no corresponding returns. SELECT name, crdb_internal.humanize_bytes(value::INT8) FROM crdb_internal.node_metrics diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v1_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v1_encoding_apply_to_elastic new file mode 100644 index 000000000000..b7ee8c1ba1e0 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v1_encoding_apply_to_elastic @@ -0,0 +1,83 @@ +# -- When using v1 encoding (V2EnabledWhenLeaderV1Encoding), all entries which +# -- are subject to admission control are encoded as `raftpb.LowPri`, +# -- regardless of their original priority; to avoid the overhead of +# -- deserializing the raft admission metadata. Therefore, as the underlying +# -- test is shared between the v1 and v2 encoding testdata files, the reader +# -- should interpret any comments referring to regular tokens as referring to +# -- elastic token. +echo +---- +---- +-- (Issuing 1x1MiB, 3x replicated write that's not admitted.) + + +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with +-- no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 21 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Transferring range lease to n2 and allowing leadership to follow.) + + +-- Flow token metrics from n1 having lost the lease and raft leadership. All +-- deducted tokens are returned. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 3.0 MiB + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v2_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v2_encoding_apply_to_all similarity index 96% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v2_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v2_encoding_apply_to_all index 205257e5adfd..147b69aadafa 100644 --- a/pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v2_encoding +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v2_encoding_apply_to_all @@ -4,8 +4,8 @@ echo -- (Issuing 1x1MiB, 3x replicated write that's not admitted.) --- Flow token metrics from n1 after issuing 1x1MiB regular 3x replicated write --- that's not admitted. We see 1*1MiB*3=3MiB deductions of regular tokens with +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with -- no corresponding returns. SELECT name, crdb_internal.humanize_bytes(value::INT8) FROM crdb_internal.node_metrics diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v2_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v2_encoding_apply_to_elastic new file mode 100644 index 000000000000..75054ba0a60d --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/transfer_lease_v2_encoding_apply_to_elastic @@ -0,0 +1,76 @@ +echo +---- +---- +-- (Issuing 1x1MiB, 3x replicated write that's not admitted.) + + +-- Flow token metrics from n1 after issuing 1x1MiB 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of tokens with +-- no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 21 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Transferring range lease to n2 and allowing leadership to follow.) + + +-- Flow token metrics from n1 having lost the lease and raft leadership. All +-- deducted tokens are returned. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 3.0 MiB + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range_v1_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range_v1_encoding_apply_to_all similarity index 100% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range_v1_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range_v1_encoding_apply_to_all diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range_v1_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range_v1_encoding_apply_to_elastic new file mode 100644 index 000000000000..c8ad83e57d53 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range_v1_encoding_apply_to_elastic @@ -0,0 +1,148 @@ +# -- When using v1 encoding (V2EnabledWhenLeaderV1Encoding), all entries which +# -- are subject to admission control are encoded as `raftpb.LowPri`, +# -- regardless of their original priority; to avoid the overhead of +# -- deserializing the raft admission metadata. Therefore, as the underlying +# -- test is shared between the v1 and v2 encoding testdata files, the reader +# -- should interpret any comments referring to regular tokens as referring to +# -- elastic token. +echo +---- +---- +-- (Issuing 1x1MiB, 3x replicated elastic write that's not admitted.) + + +-- Flow token metrics from n1 after issuing 1x1MiB elastic 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of elastic tokens with +-- no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 21 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Allow below-raft admission to proceed. We've disabled the piggybacked token +-- return mechanism so no tokens are returned via this path. But the tokens will +-- be returned anyway because the range is not quiesced and keeps pinging.) +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Issuing another 1x1MiB 3x elastic write.) +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 21 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Allow below-raft admission to proceed. We've enabled the piggybacked token +-- return mechanism so tokens are returned either via this path, or the normal +-- MsgAppResp flow, depending on which is exercised first.) +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 6.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 6.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Now the range can quiesce. Wait for it.) +---- +---- + +# vim:ft=sql diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range_v2_encoding b/pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range_v2_encoding_apply_to_all similarity index 100% rename from pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range_v2_encoding rename to pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range_v2_encoding_apply_to_all diff --git a/pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range_v2_encoding_apply_to_elastic b/pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range_v2_encoding_apply_to_elastic new file mode 100644 index 000000000000..ece62df0ed35 --- /dev/null +++ b/pkg/kv/kvserver/testdata/flow_control_integration_v2/unquiesced_range_v2_encoding_apply_to_elastic @@ -0,0 +1,141 @@ +echo +---- +---- +-- (Issuing 1x1MiB, 3x replicated elastic write that's not admitted.) + + +-- Flow token metrics from n1 after issuing 1x1MiB elastic 3x replicated write +-- that's not admitted. We see 1*1MiB*3=3MiB deductions of elastic tokens with +-- no corresponding returns. +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 21 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 0 B + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 0 B + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Allow below-raft admission to proceed. We've disabled the piggybacked token +-- return mechanism so no tokens are returned via this path. But the tokens will +-- be returned anyway because the range is not quiesced and keeps pinging.) +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 3.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Issuing another 1x1MiB 3x elastic write.) +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 21 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 21 MiB + kvflowcontrol.tokens.send.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 3.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Allow below-raft admission to proceed. We've enabled the piggybacked token +-- return mechanism so tokens are returned either via this path, or the normal +-- MsgAppResp flow, depending on which is exercised first.) +SELECT name, crdb_internal.humanize_bytes(value::INT8) + FROM crdb_internal.node_metrics + WHERE name LIKE '%kvflowcontrol%tokens%' +ORDER BY name ASC; + + kvflowcontrol.tokens.eval.elastic.available | 24 MiB + kvflowcontrol.tokens.eval.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.eval.elastic.returned | 6.0 MiB + kvflowcontrol.tokens.eval.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.elastic.unaccounted | 0 B + kvflowcontrol.tokens.eval.regular.available | 48 MiB + kvflowcontrol.tokens.eval.regular.deducted | 0 B + kvflowcontrol.tokens.eval.regular.returned | 0 B + kvflowcontrol.tokens.eval.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.eval.regular.unaccounted | 0 B + kvflowcontrol.tokens.send.elastic.available | 24 MiB + kvflowcontrol.tokens.send.elastic.deducted | 6.0 MiB + kvflowcontrol.tokens.send.elastic.deducted.force_flush_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.elastic.returned | 6.0 MiB + kvflowcontrol.tokens.send.elastic.returned.disconnect | 0 B + kvflowcontrol.tokens.send.elastic.unaccounted | 0 B + kvflowcontrol.tokens.send.regular.available | 48 MiB + kvflowcontrol.tokens.send.regular.deducted | 0 B + kvflowcontrol.tokens.send.regular.deducted.prevent_send_queue | 0 B + kvflowcontrol.tokens.send.regular.returned | 0 B + kvflowcontrol.tokens.send.regular.returned.disconnect | 0 B + kvflowcontrol.tokens.send.regular.unaccounted | 0 B + + +-- (Now the range can quiesce. Wait for it.) +---- +---- + +# vim:ft=sql From cdaed19d4bc95ffba656aeb0d697c16201cbeaed Mon Sep 17 00:00:00 2001 From: Austen McClernon Date: Wed, 9 Oct 2024 17:16:16 -0400 Subject: [PATCH 3/8] kvserver: allow waiting for + tokens available in TestFlowControl We already allowed waiting for all tokens to be returned. Also allow waiting for a specific number of tokens to be available, i.e., instead of the limit, any arbitrary value. Part of: #128040 Release note: None --- .../kvserver/flow_control_integration_test.go | 87 +++++++++++++++++-- 1 file changed, 80 insertions(+), 7 deletions(-) diff --git a/pkg/kv/kvserver/flow_control_integration_test.go b/pkg/kv/kvserver/flow_control_integration_test.go index 18c733108b43..bc3a1700b912 100644 --- a/pkg/kv/kvserver/flow_control_integration_test.go +++ b/pkg/kv/kvserver/flow_control_integration_test.go @@ -2885,6 +2885,14 @@ func TestFlowControlRaftSnapshotV2(t *testing.T) { }, incArgs); err != nil { t.Fatal(err) } + // We don't need to assert that the tokens are tracked, but doing so + // will make debugging this test failure easier. + h.waitForTotalTrackedTokens(ctx, repl.RangeID, 5<<20 /* 5 MiB */, 0 /* serverIdx */) + h.waitForAllTokensAvailable(ctx, 5, 0 /* serverIdx */, h.tokensAvailableLimitWithDelta(tokensAvailableDeltaModeEnabled( + mode, + v2EnabledWhenLeaderLevel, + -(1<<20), /* 1 MiB */ + ))) h.comment(` -- Flow token metrics from n1 after issuing 1 1MiB 5x replicated write @@ -4543,6 +4551,73 @@ func (h *flowControlTestHelper) waitForAllTokensReturned( // between v1 and v2 flow control. func (h *flowControlTestHelper) checkAllTokensReturned( ctx context.Context, expStreamCount, serverIdx int, lvl ...kvflowcontrol.V2EnabledWhenLeaderLevel, +) error { + return h.checkTokensAvailable( + ctx, expStreamCount, serverIdx, h.tokensAvailableLimitWithDelta(kvflowinspectpb.Stream{}), lvl...) +} + +func tokensAvailableDeltaModeEnabled( + mode kvflowcontrol.ModeT, enabled kvflowcontrol.V2EnabledWhenLeaderLevel, delta int64, +) kvflowinspectpb.Stream { + streamDelta := kvflowinspectpb.Stream{ + AvailableEvalElasticTokens: delta, + AvailableSendElasticTokens: delta, + } + switch mode { + case kvflowcontrol.ApplyToElastic: + // Handled above, nothing to do. + case kvflowcontrol.ApplyToAll: + if enabled == kvflowcontrol.V2EnabledWhenLeaderV2Encoding { + // NB: We cannot reliably assert on the regular tokens when not using the + // V2 protocol because we will convert all decoded priorities to elastic + // in processor.go: AdmitRaftEntriesRaftMuLocked. + streamDelta.AvailableEvalRegularTokens = delta + streamDelta.AvailableSendRegularTokens = delta + } + default: + panic("unknown flow control mode") + } + + return streamDelta +} + +func (h *flowControlTestHelper) tokensAvailableLimitWithDelta( + delta kvflowinspectpb.Stream, +) kvflowinspectpb.Stream { + elasticTokensPerStream := kvflowcontrol.ElasticTokensPerStream.Get(&h.st.SV) + regularTokensPerStream := kvflowcontrol.RegularTokensPerStream.Get(&h.st.SV) + return kvflowinspectpb.Stream{ + AvailableEvalRegularTokens: regularTokensPerStream + delta.AvailableEvalRegularTokens, + AvailableEvalElasticTokens: elasticTokensPerStream + delta.AvailableEvalElasticTokens, + AvailableSendRegularTokens: regularTokensPerStream + delta.AvailableSendRegularTokens, + AvailableSendElasticTokens: elasticTokensPerStream + delta.AvailableSendElasticTokens, + } +} + +// waitForAllTokensAvaiable waits for all tokens to be equal to the provided +// expTokensStream across all streams. The expected number of streams and +// protocol level is passed in as an argument, in order to allow switching +// between v1 and v2 flow control. +func (h *flowControlTestHelper) waitForAllTokensAvailable( + ctx context.Context, + expStreamCount, serverIdx int, + expTokensStream kvflowinspectpb.Stream, + lvl ...kvflowcontrol.V2EnabledWhenLeaderLevel, +) { + testutils.SucceedsSoon(h.t, func() error { + return h.checkTokensAvailable(ctx, expStreamCount, serverIdx, expTokensStream, lvl...) + }) +} + +// checkTokensAvailableIs checks that the expected number of tokens are available +// across all streams. The expected number of streams and protocol level is +// passed in as an argument, in order to allow switching between v1 and v2 flow +// control. +func (h *flowControlTestHelper) checkTokensAvailable( + ctx context.Context, + expStreamCount, serverIdx int, + expTokensStream kvflowinspectpb.Stream, + lvl ...kvflowcontrol.V2EnabledWhenLeaderLevel, ) error { var streams []kvflowinspectpb.Stream level := h.resolveLevelArgs(lvl...) @@ -4555,8 +4630,6 @@ func (h *flowControlTestHelper) checkAllTokensReturned( h.t.Fatalf("unknown level: %v", level) } - elasticTokensPerStream := kvflowcontrol.ElasticTokensPerStream.Get(&h.st.SV) - regularTokensPerStream := kvflowcontrol.RegularTokensPerStream.Get(&h.st.SV) if len(streams) != expStreamCount { return fmt.Errorf("expected %d replication streams, got %d [%+v]", expStreamCount, len(streams), streams) } @@ -4567,7 +4640,7 @@ func (h *flowControlTestHelper) checkAllTokensReturned( typName string, ) error { if actualTokens != expTokens { - return fmt.Errorf("expected %v of %s flow tokens for %v, got %v [level=%+v %+v]", + return fmt.Errorf("expected %v of %v flow tokens for %v, got %v [level=%+v stream=%v]", humanize.IBytes(uint64(expTokens)), typName, stream, humanize.IBytes(uint64(actualTokens)), level, @@ -4583,24 +4656,24 @@ func (h *flowControlTestHelper) checkAllTokensReturned( StoreID: stream.StoreID, } if err := checkTokens( - regularTokensPerStream, stream.AvailableEvalRegularTokens, s, "regular eval", + expTokensStream.AvailableEvalRegularTokens, stream.AvailableEvalRegularTokens, s, "regular eval", ); err != nil { return err } if err := checkTokens( - elasticTokensPerStream, stream.AvailableEvalElasticTokens, s, "elastic eval", + expTokensStream.AvailableEvalElasticTokens, stream.AvailableEvalElasticTokens, s, "elastic eval", ); err != nil { return err } if level > kvflowcontrol.V2NotEnabledWhenLeader { // V2 flow control also has send tokens. if err := checkTokens( - regularTokensPerStream, stream.AvailableSendRegularTokens, s, "regular send", + expTokensStream.AvailableSendRegularTokens, stream.AvailableSendRegularTokens, s, "regular send", ); err != nil { return err } if err := checkTokens( - elasticTokensPerStream, stream.AvailableSendElasticTokens, s, "elastic send", + expTokensStream.AvailableSendElasticTokens, stream.AvailableSendElasticTokens, s, "elastic send", ); err != nil { return err } From 95e7e96913cf124cd3c2999285ab39edcc346e3d Mon Sep 17 00:00:00 2001 From: Austen McClernon Date: Fri, 11 Oct 2024 16:14:51 -0400 Subject: [PATCH 4/8] kvserver: skip TestFlowControlTransferLeaseV2 under duress We are already going to skip this as part of #132125. Unblock merging by skipping now as well. Informs: #132125 Part of: #128040 Release note: None --- pkg/kv/kvserver/flow_control_integration_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/kv/kvserver/flow_control_integration_test.go b/pkg/kv/kvserver/flow_control_integration_test.go index bc3a1700b912..977fecad1df8 100644 --- a/pkg/kv/kvserver/flow_control_integration_test.go +++ b/pkg/kv/kvserver/flow_control_integration_test.go @@ -3533,6 +3533,7 @@ func TestFlowControlTransferLeaseV2(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + skip.UnderDuressWithIssue(t, 132272) testutils.RunValues(t, "v2_enabled_when_leader_level", []kvflowcontrol.V2EnabledWhenLeaderLevel{ kvflowcontrol.V2EnabledWhenLeaderV1Encoding, kvflowcontrol.V2EnabledWhenLeaderV2Encoding, From dc845cade872ef2a894e7571b9482cc2fc72e2c8 Mon Sep 17 00:00:00 2001 From: Austen McClernon Date: Wed, 9 Oct 2024 02:20:05 -0400 Subject: [PATCH 5/8] kvserver: allow enabling rac pull mode with a send queue By default,`kvadmission.flow_control.mode = "apply_to_elastic"` replication admission control operates in push mode, where Raft has its own flow control machinery in operation and controls sending messages to replicas. In pull mode, which could not be enabled prior to this commit (outside of tests), replication flow control (v2) maintains its own machinery (send queue) and controls when to send a message to a replica. Previously, when `kvadmission.flow_control.mode = "apply_to_all"`, replication admission control would remain in push mode. Allow replication flow control to run in pull mode outside of tests, by setting both: ``` kvadmission.flow_control.enabled = true (default true) kvadmission.flow_control.mode = "apply_to_all" (default "apply_to_elastic") ``` And running a cluster which has finalized its upgrade to `v24.3`. We omit a release note, as this functionality is disabled by default and we wish to retain the ability to selectively suggest enabling pull mode to users, as the feature is (1) newly introduced, (2) on the critical path for all replication and (3) subject to change. Resolves: #132115 Release note: None --- pkg/kv/kvserver/replica_proposal_quota.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/pkg/kv/kvserver/replica_proposal_quota.go b/pkg/kv/kvserver/replica_proposal_quota.go index 81473c94595e..0eb6e08ae0dd 100644 --- a/pkg/kv/kvserver/replica_proposal_quota.go +++ b/pkg/kv/kvserver/replica_proposal_quota.go @@ -109,15 +109,25 @@ func (r *Replica) getQuotaPoolEnabledRLocked(ctx context.Context) bool { // admission/flow control should use pull mode, which allows for a send-queue // and disabled raft's own flow control. func (r *Replica) shouldReplicationAdmissionControlUsePullMode(ctx context.Context) bool { - return r.store.cfg.Settings.Version.IsActive(ctx, clusterversion.V24_3_UseRACV2Full) && + if knobs := r.store.cfg.TestingKnobs.FlowControlTestingKnobs; knobs != nil && + knobs.OverridePullPushMode != nil { + return knobs.OverridePullPushMode() + } + + var versionEnabled bool + if knobs := r.store.TestingKnobs().FlowControlTestingKnobs; knobs != nil && knobs.OverrideV2EnabledWhenLeaderLevel != nil { + versionEnabled = knobs.OverrideV2EnabledWhenLeaderLevel() == kvflowcontrol.V2EnabledWhenLeaderV2Encoding + } else { + versionEnabled = r.store.cfg.Settings.Version.IsActive(ctx, clusterversion.V24_3_UseRACV2Full) + } + + return (versionEnabled && kvflowcontrol.Mode.Get(&r.store.cfg.Settings.SV) == kvflowcontrol.ApplyToAll && - kvflowcontrol.Enabled.Get(&r.store.cfg.Settings.SV) + kvflowcontrol.Enabled.Get(&r.store.cfg.Settings.SV)) } func (r *Replica) replicationAdmissionControlModeToUse(ctx context.Context) rac2.RaftMsgAppMode { - // TODO(sumeer): remove the false. - usePullMode := r.shouldReplicationAdmissionControlUsePullMode(ctx) && false - if usePullMode { + if r.shouldReplicationAdmissionControlUsePullMode(ctx) { return rac2.MsgAppPull } return rac2.MsgAppPush From ad52879078bc574530944d883d4635daf27a34ff Mon Sep 17 00:00:00 2001 From: Austen McClernon Date: Wed, 9 Oct 2024 18:18:20 -0400 Subject: [PATCH 6/8] kvserver: skip v2 flow control integration tests under duress The TestFlowControl.*V2 tests occasionally fail under slow conditions due to a send queue forming, while the test asserts directly on metric counters: - Environment is slow -> - (sending|acking) is slow -> - a send queue forms -> - tokens are deducted for the scheduler -> - deducted/returned token metrics now have an unexpected diff which we can't deterministically assert on Disable these tests under duress and track re-enabling them in #132272. Informs: #132272 Part of: #128040 Release note: None --- pkg/kv/kvserver/flow_control_integration_test.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pkg/kv/kvserver/flow_control_integration_test.go b/pkg/kv/kvserver/flow_control_integration_test.go index 977fecad1df8..a3799388dcd7 100644 --- a/pkg/kv/kvserver/flow_control_integration_test.go +++ b/pkg/kv/kvserver/flow_control_integration_test.go @@ -2210,6 +2210,7 @@ func TestFlowControlGranterAdmitOneByOne(t *testing.T) { func TestFlowControlBasicV2(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + skip.UnderDuressWithIssue(t, 132272, "non-determinism under duress: stress/race/deadlock") testutils.RunValues(t, "v2_enabled_when_leader_level", []kvflowcontrol.V2EnabledWhenLeaderLevel{ kvflowcontrol.V2EnabledWhenLeaderV1Encoding, @@ -2304,6 +2305,7 @@ ORDER BY streams DESC; func TestFlowControlRangeSplitMergeV2(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + skip.UnderDuressWithIssue(t, 132272, "non-determinism under duress: stress/race/deadlock") testutils.RunValues(t, "v2_enabled_when_leader_level", []kvflowcontrol.V2EnabledWhenLeaderLevel{ kvflowcontrol.V2EnabledWhenLeaderV1Encoding, @@ -2418,6 +2420,7 @@ ORDER BY streams DESC; func TestFlowControlBlockedAdmissionV2(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + skip.UnderDuressWithIssue(t, 132272, "non-determinism under duress: stress/race/deadlock") testutils.RunValues(t, "v2_enabled_when_leader_level", []kvflowcontrol.V2EnabledWhenLeaderLevel{ kvflowcontrol.V2EnabledWhenLeaderV1Encoding, @@ -2519,6 +2522,7 @@ func TestFlowControlBlockedAdmissionV2(t *testing.T) { func TestFlowControlAdmissionPostSplitMergeV2(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + skip.UnderDuressWithIssue(t, 132272, "non-determinism under duress: stress/race/deadlock") testutils.RunValues(t, "v2_enabled_when_leader_level", []kvflowcontrol.V2EnabledWhenLeaderLevel{ kvflowcontrol.V2EnabledWhenLeaderV1Encoding, @@ -2669,6 +2673,7 @@ ORDER BY streams DESC; func TestFlowControlCrashedNodeV2(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + skip.UnderDuressWithIssue(t, 132272, "non-determinism under duress: stress/race/deadlock") testutils.RunValues(t, "v2_enabled_when_leader_level", []kvflowcontrol.V2EnabledWhenLeaderLevel{ kvflowcontrol.V2EnabledWhenLeaderV1Encoding, @@ -2773,6 +2778,7 @@ func TestFlowControlCrashedNodeV2(t *testing.T) { func TestFlowControlRaftSnapshotV2(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + skip.UnderDuressWithIssue(t, 132272, "non-determinism under duress: stress/race/deadlock") const numServers int = 5 @@ -3040,6 +3046,7 @@ SELECT store_id, func TestFlowControlRaftMembershipV2(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + skip.UnderDuressWithIssue(t, 132272, "non-determinism under duress: stress/race/deadlock") testutils.RunValues(t, "v2_enabled_when_leader_level", []kvflowcontrol.V2EnabledWhenLeaderLevel{ kvflowcontrol.V2EnabledWhenLeaderV1Encoding, @@ -3172,6 +3179,7 @@ func TestFlowControlRaftMembershipV2(t *testing.T) { func TestFlowControlRaftMembershipRemoveSelfV2(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + skip.UnderDuressWithIssue(t, 132272, "non-determinism under duress: stress/race/deadlock") testutils.RunValues(t, "v2_enabled_when_leader_level", []kvflowcontrol.V2EnabledWhenLeaderLevel{ kvflowcontrol.V2EnabledWhenLeaderV1Encoding, @@ -3308,6 +3316,7 @@ ORDER BY streams DESC; func TestFlowControlClassPrioritizationV2(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + skip.UnderDuressWithIssue(t, 132272, "non-determinism under duress: stress/race/deadlock") testutils.RunValues(t, "v2_enabled_when_leader_level", []kvflowcontrol.V2EnabledWhenLeaderLevel{ kvflowcontrol.V2EnabledWhenLeaderV1Encoding, @@ -3398,6 +3407,7 @@ func TestFlowControlClassPrioritizationV2(t *testing.T) { func TestFlowControlUnquiescedRangeV2(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + skip.UnderDuressWithIssue(t, 132272, "non-determinism under duress: stress/race/deadlock") testutils.RunValues(t, "v2_enabled_when_leader_level", []kvflowcontrol.V2EnabledWhenLeaderLevel{ kvflowcontrol.V2EnabledWhenLeaderV1Encoding, @@ -3532,8 +3542,8 @@ func TestFlowControlUnquiescedRangeV2(t *testing.T) { func TestFlowControlTransferLeaseV2(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + skip.UnderDuressWithIssue(t, 132272, "non-determinism under duress: stress/race/deadlock") - skip.UnderDuressWithIssue(t, 132272) testutils.RunValues(t, "v2_enabled_when_leader_level", []kvflowcontrol.V2EnabledWhenLeaderLevel{ kvflowcontrol.V2EnabledWhenLeaderV1Encoding, kvflowcontrol.V2EnabledWhenLeaderV2Encoding, @@ -3621,6 +3631,7 @@ func TestFlowControlTransferLeaseV2(t *testing.T) { func TestFlowControlLeaderNotLeaseholderV2(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + skip.UnderDuressWithIssue(t, 132272, "non-determinism under duress: stress/race/deadlock") testutils.RunValues(t, "v2_enabled_when_leader_level", []kvflowcontrol.V2EnabledWhenLeaderLevel{ kvflowcontrol.V2EnabledWhenLeaderV1Encoding, @@ -3733,6 +3744,7 @@ func TestFlowControlLeaderNotLeaseholderV2(t *testing.T) { func TestFlowControlGranterAdmitOneByOneV2(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + skip.UnderDuressWithIssue(t, 132272, "non-determinism under duress: stress/race/deadlock") testutils.RunValues(t, "v2_enabled_when_leader_level", []kvflowcontrol.V2EnabledWhenLeaderLevel{ kvflowcontrol.V2EnabledWhenLeaderV1Encoding, @@ -4610,7 +4622,7 @@ func (h *flowControlTestHelper) waitForAllTokensAvailable( }) } -// checkTokensAvailableIs checks that the expected number of tokens are available +// checkTokensAvailable checks that the expected number of tokens are available // across all streams. The expected number of streams and protocol level is // passed in as an argument, in order to allow switching between v1 and v2 flow // control. From c81ec1c431179d4f1e1d7a61b98033910f56a59a Mon Sep 17 00:00:00 2001 From: Austen McClernon Date: Thu, 10 Oct 2024 19:06:59 -0400 Subject: [PATCH 7/8] kvflowcontrol: disable kvadmission.flow_control.mode metamorphism The cluster setting `kvadmission.flow_control.mode` is used to control whether replication admission control applies to only elastic work, or applies to all work which sets an admission priority. The cluster setting was metamorphic in tests, selecting either value with equal probability. When `apply_to_all` is set, the newly introduced RACv2 machinery will operate in pull mode, deciding when to send messages to followers. This newly introduced machinery is naturally less stable, as it is newer. Disable the metamorphic constant in order to prevent undue toil on other areas of the codebase, triaging hairy failures due to `apply_to_all` being set. Re-enabling the metamorphism is tracked in #132364. Informs: #132364 Part of: #128040 Release note: None --- pkg/kv/kvserver/kvflowcontrol/BUILD.bazel | 1 - pkg/kv/kvserver/kvflowcontrol/kvflowcontrol.go | 7 +------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/pkg/kv/kvserver/kvflowcontrol/BUILD.bazel b/pkg/kv/kvserver/kvflowcontrol/BUILD.bazel index 1c49ded700dd..6f358f3ca542 100644 --- a/pkg/kv/kvserver/kvflowcontrol/BUILD.bazel +++ b/pkg/kv/kvserver/kvflowcontrol/BUILD.bazel @@ -18,7 +18,6 @@ go_library( "//pkg/settings", "//pkg/settings/cluster", "//pkg/util/admission/admissionpb", - "//pkg/util/metamorphic", "@com_github_cockroachdb_redact//:redact", "@com_github_dustin_go_humanize//:go-humanize", ], diff --git a/pkg/kv/kvserver/kvflowcontrol/kvflowcontrol.go b/pkg/kv/kvserver/kvflowcontrol/kvflowcontrol.go index 0b5ade521b30..0cf38153a508 100644 --- a/pkg/kv/kvserver/kvflowcontrol/kvflowcontrol.go +++ b/pkg/kv/kvserver/kvflowcontrol/kvflowcontrol.go @@ -19,7 +19,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/settings" "github.com/cockroachdb/cockroach/pkg/settings/cluster" "github.com/cockroachdb/cockroach/pkg/util/admission/admissionpb" - "github.com/cockroachdb/cockroach/pkg/util/metamorphic" "github.com/cockroachdb/redact" "github.com/dustin/go-humanize" ) @@ -38,11 +37,7 @@ var Mode = settings.RegisterEnumSetting( settings.SystemOnly, "kvadmission.flow_control.mode", "determines the 'mode' of flow control we use for replication traffic in KV, if enabled", - metamorphic.ConstantWithTestChoice( - "kvadmission.flow_control.mode", - modeDict[ApplyToElastic], /* default value */ - modeDict[ApplyToAll], /* other value */ - ), + modeDict[ApplyToElastic], /* default value */ modeDict, ) From 6a283920803b2474e34f0d8a639d67a859d4ef55 Mon Sep 17 00:00:00 2001 From: Austen McClernon Date: Fri, 11 Oct 2024 17:59:22 -0400 Subject: [PATCH 8/8] kvserver: skip TestFlowControl.* under duress The TestFlowControl.* tests occasionally fail under slow conditions due to slow raft processing and heartbeats failing which causes streams to unexpectedly disconnect. Disable these tests under duress and track re-enabling them in #132310. Part of: #128040 Informs: #132310 Release note: None --- .../kvserver/flow_control_integration_test.go | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/pkg/kv/kvserver/flow_control_integration_test.go b/pkg/kv/kvserver/flow_control_integration_test.go index a3799388dcd7..4c9346a1d9b7 100644 --- a/pkg/kv/kvserver/flow_control_integration_test.go +++ b/pkg/kv/kvserver/flow_control_integration_test.go @@ -68,6 +68,9 @@ import ( func TestFlowControlBasic(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + // TOOD(kvoli,pav-kv,sumeerbhola): Enable this test under all conditions + // after fixing the flakiness introduced by #132121. + skip.UnderDuressWithIssue(t, 132310) testutils.RunTrueAndFalse(t, "always-enqueue", func(t *testing.T, alwaysEnqueue bool) { ctx := context.Background() @@ -220,6 +223,9 @@ ORDER BY streams DESC; func TestFlowControlRangeSplitMerge(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + // TOOD(kvoli,pav-kv,sumeerbhola): Enable this test under all conditions + // after fixing the flakiness introduced by #132121. + skip.UnderDuressWithIssue(t, 132310) ctx := context.Background() const numNodes = 3 @@ -335,6 +341,9 @@ ORDER BY streams DESC; func TestFlowControlBlockedAdmission(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + // TOOD(kvoli,pav-kv,sumeerbhola): Enable this test under all conditions + // after fixing the flakiness introduced by #132121. + skip.UnderDuressWithIssue(t, 132310) ctx := context.Background() const numNodes = 3 @@ -446,6 +455,9 @@ ORDER BY name ASC; func TestFlowControlAdmissionPostSplitMerge(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + // TOOD(kvoli,pav-kv,sumeerbhola): Enable this test under all conditions + // after fixing the flakiness introduced by #132121. + skip.UnderDuressWithIssue(t, 132310) ctx := context.Background() const numNodes = 3 @@ -582,6 +594,9 @@ ORDER BY streams DESC; func TestFlowControlCrashedNode(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + // TOOD(kvoli,pav-kv,sumeerbhola): Enable this test under all conditions + // after fixing the flakiness introduced by #132121. + skip.UnderDuressWithIssue(t, 132310) ctx := context.Background() const numNodes = 2 @@ -999,6 +1014,9 @@ SELECT store_id, func TestFlowControlRaftTransportBreak(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + // TOOD(kvoli,pav-kv,sumeerbhola): Enable this test under all conditions + // after fixing the flakiness introduced by #132121. + skip.UnderDuressWithIssue(t, 132310) ctx := context.Background() const numNodes = 3 @@ -1111,6 +1129,9 @@ func TestFlowControlRaftTransportBreak(t *testing.T) { func TestFlowControlRaftTransportCulled(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + // TOOD(kvoli,pav-kv,sumeerbhola): Enable this test under all conditions + // after fixing the flakiness introduced by #132121. + skip.UnderDuressWithIssue(t, 132310) ctx := context.Background() const numNodes = 3 @@ -1247,6 +1268,9 @@ func TestFlowControlRaftTransportCulled(t *testing.T) { func TestFlowControlRaftMembership(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + // TOOD(kvoli,pav-kv,sumeerbhola): Enable this test under all conditions + // after fixing the flakiness introduced by #132121. + skip.UnderDuressWithIssue(t, 132310) ctx := context.Background() st := cluster.MakeTestingClusterSettings() @@ -1377,6 +1401,9 @@ ORDER BY name ASC; func TestFlowControlRaftMembershipRemoveSelf(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + // TOOD(kvoli,pav-kv,sumeerbhola): Enable this test under all conditions + // after fixing the flakiness introduced by #132121. + skip.UnderDuressWithIssue(t, 132310) testutils.RunTrueAndFalse(t, "transfer-lease-first", func(t *testing.T, transferLeaseFirst bool) { ctx := context.Background() @@ -1505,6 +1532,9 @@ ORDER BY name ASC; func TestFlowControlClassPrioritization(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + // TOOD(kvoli,pav-kv,sumeerbhola): Enable this test under all conditions + // after fixing the flakiness introduced by #132121. + skip.UnderDuressWithIssue(t, 132310) ctx := context.Background() const numNodes = 5 @@ -1588,6 +1618,9 @@ func TestFlowControlClassPrioritization(t *testing.T) { func TestFlowControlQuiescedRange(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + // TOOD(kvoli,pav-kv,sumeerbhola): Enable this test under all conditions + // after fixing the flakiness introduced by #132121. + skip.UnderDuressWithIssue(t, 132310) ctx := context.Background() const numNodes = 3 @@ -1728,6 +1761,9 @@ ORDER BY name ASC; func TestFlowControlUnquiescedRange(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + // TOOD(kvoli,pav-kv,sumeerbhola): Enable this test under all conditions + // after fixing the flakiness introduced by #132121. + skip.UnderDuressWithIssue(t, 132310) ctx := context.Background() const numNodes = 3 @@ -1881,6 +1917,9 @@ ORDER BY name ASC; func TestFlowControlTransferLease(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + // TOOD(kvoli,pav-kv,sumeerbhola): Enable this test under all conditions + // after fixing the flakiness introduced by #132121. + skip.UnderDuressWithIssue(t, 132310) ctx := context.Background() const numNodes = 5 @@ -1969,6 +2008,9 @@ ORDER BY name ASC; func TestFlowControlLeaderNotLeaseholder(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + // TOOD(kvoli,pav-kv,sumeerbhola): Enable this test under all conditions + // after fixing the flakiness introduced by #132121. + skip.UnderDuressWithIssue(t, 132310) ctx := context.Background() const numNodes = 5 @@ -2102,6 +2144,9 @@ ORDER BY name ASC; func TestFlowControlGranterAdmitOneByOne(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) + // TOOD(kvoli,pav-kv,sumeerbhola): Enable this test under all conditions + // after fixing the flakiness introduced by #132121. + skip.UnderDuressWithIssue(t, 132310) ctx := context.Background() const numNodes = 3