diff --git a/DEPS.bzl b/DEPS.bzl index 0d1970d9b3e6..2d439b1df337 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -1885,10 +1885,10 @@ def go_deps(): patches = [ "@com_github_cockroachdb_cockroach//build/patches:com_github_cockroachdb_pebble.patch", ], - sha256 = "6a36c6ed7f11452d9866fd3e95430a02020a8ed478dac9cde3e60b173904aeb8", - strip_prefix = "github.com/cockroachdb/pebble@v0.0.0-20250111211125-38fb0512c50a", + sha256 = "723fdfb0271c1d50ac9ebb710a9eeb8f249867f918056c0b166c30d13290dc95", + strip_prefix = "github.com/cockroachdb/pebble@v0.0.0-20250113205511-3748221737d4", urls = [ - "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250111211125-38fb0512c50a.zip", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250113205511-3748221737d4.zip", ], ) go_repository( diff --git a/build/bazelutil/distdir_files.bzl b/build/bazelutil/distdir_files.bzl index ba24dc4d7d19..950c49f1691a 100644 --- a/build/bazelutil/distdir_files.bzl +++ b/build/bazelutil/distdir_files.bzl @@ -358,7 +358,7 @@ DISTDIR_FILES = { "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/gostdlib/com_github_cockroachdb_gostdlib-v1.19.0.zip": "c4d516bcfe8c07b6fc09b8a9a07a95065b36c2855627cb3514e40c98f872b69e", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/logtags/com_github_cockroachdb_logtags-v0.0.0-20241215232642-bb51bb14a506.zip": "920068af09e3846d9ebb4e4a7787ff1dd10f3989c5f940ad861b0f6a9f824f6e", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/metamorphic/com_github_cockroachdb_metamorphic-v0.0.0-20231108215700-4ba948b56895.zip": "28c8cf42192951b69378cf537be5a9a43f2aeb35542908cc4fe5f689505853ea", - "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250111211125-38fb0512c50a.zip": "6a36c6ed7f11452d9866fd3e95430a02020a8ed478dac9cde3e60b173904aeb8", + "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/pebble/com_github_cockroachdb_pebble-v0.0.0-20250113205511-3748221737d4.zip": "723fdfb0271c1d50ac9ebb710a9eeb8f249867f918056c0b166c30d13290dc95", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/redact/com_github_cockroachdb_redact-v1.1.5.zip": "11b30528eb0dafc8bc1a5ba39d81277c257cbe6946a7564402f588357c164560", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/returncheck/com_github_cockroachdb_returncheck-v0.0.0-20200612231554-92cdbca611dd.zip": "ce92ba4352deec995b1f2eecf16eba7f5d51f5aa245a1c362dfe24c83d31f82b", "https://storage.googleapis.com/cockroach-godeps/gomod/github.com/cockroachdb/stress/com_github_cockroachdb_stress-v0.0.0-20220803192808-1806698b1b7b.zip": "3fda531795c600daf25532a4f98be2a1335cd1e5e182c72789bca79f5f69fcc1", diff --git a/go.mod b/go.mod index 38d4bf4d4c88..48244f7e52c1 100644 --- a/go.mod +++ b/go.mod @@ -135,7 +135,7 @@ require ( github.com/cockroachdb/go-test-teamcity v0.0.0-20191211140407-cff980ad0a55 github.com/cockroachdb/gostdlib v1.19.0 github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 - github.com/cockroachdb/pebble v0.0.0-20250111211125-38fb0512c50a + github.com/cockroachdb/pebble v0.0.0-20250113205511-3748221737d4 github.com/cockroachdb/redact v1.1.5 github.com/cockroachdb/returncheck v0.0.0-20200612231554-92cdbca611dd github.com/cockroachdb/stress v0.0.0-20220803192808-1806698b1b7b diff --git a/go.sum b/go.sum index 0796cfdab90c..e6130a05ed58 100644 --- a/go.sum +++ b/go.sum @@ -556,8 +556,8 @@ github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506 h1:ASDL+UJcILM github.com/cockroachdb/logtags v0.0.0-20241215232642-bb51bb14a506/go.mod h1:Mw7HqKr2kdtu6aYGn3tPmAftiP3QPX63LdK/zcariIo= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA= github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA= -github.com/cockroachdb/pebble v0.0.0-20250111211125-38fb0512c50a h1:eJJFy22HbaaWZpxgQDZi7sOyy2B8M3kYFfzwVJr2z3M= -github.com/cockroachdb/pebble v0.0.0-20250111211125-38fb0512c50a/go.mod h1:ewJSTQ30qIuX6FeYX+2M37Ghn6r0r2I+g0jDIcTdUXM= +github.com/cockroachdb/pebble v0.0.0-20250113205511-3748221737d4 h1:WnGNMXYEsv7uJbHGkl025sXFbjlXaXck5hQtZ0hCZRA= +github.com/cockroachdb/pebble v0.0.0-20250113205511-3748221737d4/go.mod h1:ewJSTQ30qIuX6FeYX+2M37Ghn6r0r2I+g0jDIcTdUXM= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= diff --git a/pkg/backup/testdata/backup-restore/restore-validation-only b/pkg/backup/testdata/backup-restore/restore-validation-only index eb5c64c3cc5b..8485fe45ce98 100644 --- a/pkg/backup/testdata/backup-restore/restore-validation-only +++ b/pkg/backup/testdata/backup-restore/restore-validation-only @@ -169,7 +169,7 @@ RESTORE DATABASE d FROM LATEST IN 'nodelocal://1/full_database_backup/' with sch ---- # But verify_backup_table_data catches the corrupt backup file -exec-sql expect-error-regex=(pebble/table: invalid table 000000) +exec-sql expect-error-regex=(checksum mismatch) RESTORE DATABASE d FROM LATEST IN 'nodelocal://1/full_database_backup/' with schema_only, verify_backup_table_data, new_db_name='d4'; ---- regex matches error diff --git a/pkg/kv/kvserver/BUILD.bazel b/pkg/kv/kvserver/BUILD.bazel index f2179246e8c6..2817ada46754 100644 --- a/pkg/kv/kvserver/BUILD.bazel +++ b/pkg/kv/kvserver/BUILD.bazel @@ -248,7 +248,7 @@ go_library( "@com_github_cockroachdb_pebble//objstorage", "@com_github_cockroachdb_pebble//objstorage/remote", "@com_github_cockroachdb_pebble//rangekey", - "@com_github_cockroachdb_pebble//sstable", + "@com_github_cockroachdb_pebble//sstable/block", "@com_github_cockroachdb_pebble//vfs", "@com_github_cockroachdb_redact//:redact", "@com_github_gogo_protobuf//proto", diff --git a/pkg/kv/kvserver/metrics.go b/pkg/kv/kvserver/metrics.go index 4611d96e8d7e..06de7ae64f40 100644 --- a/pkg/kv/kvserver/metrics.go +++ b/pkg/kv/kvserver/metrics.go @@ -30,7 +30,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/metric/aggmetric" "github.com/cockroachdb/cockroach/pkg/util/syncutil" "github.com/cockroachdb/pebble" - "github.com/cockroachdb/pebble/sstable" + "github.com/cockroachdb/pebble/sstable/block" "github.com/cockroachdb/pebble/vfs" ) @@ -4161,7 +4161,7 @@ type pebbleCategoryIterMetrics struct { IterBlockReadLatencySum *metric.Counter } -func makePebbleCategorizedIterMetrics(category sstable.Category) pebbleCategoryIterMetrics { +func makePebbleCategorizedIterMetrics(category block.Category) pebbleCategoryIterMetrics { metaBlockBytes := metric.Metadata{ Name: fmt.Sprintf("storage.iterator.category-%s.block-load.bytes", category), Help: "Bytes loaded by storage sstable iterators (possibly cached).", @@ -4190,7 +4190,7 @@ func makePebbleCategorizedIterMetrics(category sstable.Category) pebbleCategoryI // MetricStruct implements the metric.Struct interface. func (m *pebbleCategoryIterMetrics) MetricStruct() {} -func (m *pebbleCategoryIterMetrics) update(stats sstable.CategoryStats) { +func (m *pebbleCategoryIterMetrics) update(stats block.CategoryStats) { m.IterBlockBytes.Update(int64(stats.BlockBytes)) m.IterBlockBytesInCache.Update(int64(stats.BlockBytesInCache)) m.IterBlockReadLatencySum.Update(int64(stats.BlockReadDuration)) @@ -4198,20 +4198,20 @@ func (m *pebbleCategoryIterMetrics) update(stats sstable.CategoryStats) { type pebbleCategoryIterMetricsContainer struct { registry *metric.Registry - // metrics slice for all categories; can be directly indexed by sstable.Category. + // metrics slice for all categories; can be directly indexed by block.Category. metrics []pebbleCategoryIterMetrics } func (m *pebbleCategoryIterMetricsContainer) init(registry *metric.Registry) { m.registry = registry - categories := sstable.Categories() + categories := block.Categories() m.metrics = make([]pebbleCategoryIterMetrics, len(categories)) for _, c := range categories { m.metrics[c] = makePebbleCategorizedIterMetrics(c) } } -func (m *pebbleCategoryIterMetricsContainer) update(stats []sstable.CategoryStatsAggregate) { +func (m *pebbleCategoryIterMetricsContainer) update(stats []block.CategoryStatsAggregate) { for _, s := range stats { m.metrics[s.Category].update(s.CategoryStats) } diff --git a/pkg/storage/fs/BUILD.bazel b/pkg/storage/fs/BUILD.bazel index 6be281be5fb4..9cb59c5705ad 100644 --- a/pkg/storage/fs/BUILD.bazel +++ b/pkg/storage/fs/BUILD.bazel @@ -30,7 +30,7 @@ go_library( "@com_github_cockroachdb_errors//oserror", "@com_github_cockroachdb_pebble//:pebble", "@com_github_cockroachdb_pebble//record", - "@com_github_cockroachdb_pebble//sstable", + "@com_github_cockroachdb_pebble//sstable/block", "@com_github_cockroachdb_pebble//vfs", "@com_github_cockroachdb_pebble//vfs/atomicfs", "@com_github_cockroachdb_pebble//vfs/vfstest", diff --git a/pkg/storage/fs/category.go b/pkg/storage/fs/category.go index 44388a239eaa..2e63b2457dd5 100644 --- a/pkg/storage/fs/category.go +++ b/pkg/storage/fs/category.go @@ -6,7 +6,7 @@ package fs import ( - "github.com/cockroachdb/pebble/sstable" + "github.com/cockroachdb/pebble/sstable/block" "github.com/cockroachdb/pebble/vfs" ) @@ -46,23 +46,23 @@ const ( BackupReadCategory ) -var readCategoryMap = [...]sstable.Category{ - UnknownReadCategory: sstable.RegisterCategory("crdb-unknown", sstable.LatencySensitiveQoSLevel), +var readCategoryMap = [...]block.Category{ + UnknownReadCategory: block.RegisterCategory("crdb-unknown", block.LatencySensitiveQoSLevel), // TODO(sumeer): consider splitting batch-eval into two categories, for // latency sensitive and non latency sensitive. - BatchEvalReadCategory: sstable.RegisterCategory("batch-eval", sstable.LatencySensitiveQoSLevel), - ScanRegularBatchEvalReadCategory: sstable.RegisterCategory("scan-regular", sstable.LatencySensitiveQoSLevel), - ScanBackgroundBatchEvalReadCategory: sstable.RegisterCategory("scan-background", sstable.NonLatencySensitiveQoSLevel), - MVCCGCReadCategory: sstable.RegisterCategory("mvcc-gc", sstable.NonLatencySensitiveQoSLevel), - RangeSnapshotReadCategory: sstable.RegisterCategory("range-snap", sstable.NonLatencySensitiveQoSLevel), - RangefeedReadCategory: sstable.RegisterCategory("rangefeed", sstable.LatencySensitiveQoSLevel), - ReplicationReadCategory: sstable.RegisterCategory("replication", sstable.LatencySensitiveQoSLevel), - IntentResolutionReadCategory: sstable.RegisterCategory("intent-resolution", sstable.LatencySensitiveQoSLevel), - BackupReadCategory: sstable.RegisterCategory("backup", sstable.NonLatencySensitiveQoSLevel), + BatchEvalReadCategory: block.RegisterCategory("batch-eval", block.LatencySensitiveQoSLevel), + ScanRegularBatchEvalReadCategory: block.RegisterCategory("scan-regular", block.LatencySensitiveQoSLevel), + ScanBackgroundBatchEvalReadCategory: block.RegisterCategory("scan-background", block.NonLatencySensitiveQoSLevel), + MVCCGCReadCategory: block.RegisterCategory("mvcc-gc", block.NonLatencySensitiveQoSLevel), + RangeSnapshotReadCategory: block.RegisterCategory("range-snap", block.NonLatencySensitiveQoSLevel), + RangefeedReadCategory: block.RegisterCategory("rangefeed", block.LatencySensitiveQoSLevel), + ReplicationReadCategory: block.RegisterCategory("replication", block.LatencySensitiveQoSLevel), + IntentResolutionReadCategory: block.RegisterCategory("intent-resolution", block.LatencySensitiveQoSLevel), + BackupReadCategory: block.RegisterCategory("backup", block.NonLatencySensitiveQoSLevel), } -// PebbleCategory returns the sstable.Category associated with the given ReadCategory. -func (c ReadCategory) PebbleCategory() sstable.Category { +// PebbleCategory returns the block.Category associated with the given ReadCategory. +func (c ReadCategory) PebbleCategory() block.Category { return readCategoryMap[c] } diff --git a/pkg/storage/pebble.go b/pkg/storage/pebble.go index 11510a73777c..a37bf7f79bcc 100644 --- a/pkg/storage/pebble.go +++ b/pkg/storage/pebble.go @@ -1711,7 +1711,7 @@ func (p *Pebble) ScanInternal( rawLower := EngineKey{Key: lower}.Encode() rawUpper := EngineKey{Key: upper}.Encode() // TODO(sumeer): set category. - return p.db.ScanInternal(ctx, sstable.CategoryUnknown, rawLower, rawUpper, visitPointKey, + return p.db.ScanInternal(ctx, block.CategoryUnknown, rawLower, rawUpper, visitPointKey, visitRangeDel, visitRangeKey, visitSharedFile, visitExternalFile) } @@ -3108,7 +3108,7 @@ func (p *pebbleSnapshot) ScanInternal( rawLower := EngineKey{Key: lower}.Encode() rawUpper := EngineKey{Key: upper}.Encode() // TODO(sumeer): set category. - return p.snapshot.ScanInternal(ctx, sstable.CategoryUnknown, rawLower, rawUpper, visitPointKey, + return p.snapshot.ScanInternal(ctx, block.CategoryUnknown, rawLower, rawUpper, visitPointKey, visitRangeDel, visitRangeKey, visitSharedFile, visitExternalFile) } @@ -3232,7 +3232,7 @@ func (p *pebbleEFOS) ScanInternal( rawLower := EngineKey{Key: lower}.Encode() rawUpper := EngineKey{Key: upper}.Encode() // TODO(sumeer): set category. - return p.efos.ScanInternal(ctx, sstable.CategoryUnknown, rawLower, rawUpper, visitPointKey, + return p.efos.ScanInternal(ctx, block.CategoryUnknown, rawLower, rawUpper, visitPointKey, visitRangeDel, visitRangeKey, visitSharedFile, visitExternalFile) } diff --git a/pkg/storage/pebble_test.go b/pkg/storage/pebble_test.go index 2d50bbbe127b..3b183d704bb4 100644 --- a/pkg/storage/pebble_test.go +++ b/pkg/storage/pebble_test.go @@ -1709,8 +1709,8 @@ func TestPebbleLoggingSlowReads(t *testing.T) { slowCount := testFunc(t, "pebble_logger_and_tracer") require.Equal(t, 0, slowCount) }) - t.Run("reader", func(t *testing.T) { - slowCount := testFunc(t, "reader") + t.Run("block", func(t *testing.T) { + slowCount := testFunc(t, "block") require.Less(t, 0, slowCount) }) }