From dc04bf625864dfc00ec2001a57934c866f477e3a Mon Sep 17 00:00:00 2001 From: Fra Date: Tue, 21 Nov 2023 12:31:28 +0100 Subject: [PATCH 1/9] impr(S3UTILS-150): we found corner cases where the MD answer was: - no a correct JSON object - not empty but Null this should fix these two corner cases, so the scan of the SproxyD keys is not interrupted with an exception but it logs a specific error --- verifyBucketSproxydKeys.js | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/verifyBucketSproxydKeys.js b/verifyBucketSproxydKeys.js index 6a37640f..bed8656e 100644 --- a/verifyBucketSproxydKeys.js +++ b/verifyBucketSproxydKeys.js @@ -196,7 +196,7 @@ function logProgress(message) { haveDupKeys: status.objectsWithDupKeys, haveEmptyMetadata: status.objectsWithEmptyMetadata, haveDupVersionIds: status.objectsWithDupVersionIds, - errors: status.objectErrors, + haveObjectsErrors: status.objectsErrors, url: getObjectURL(status.bucketInProgress, status.KeyMarker), }); } @@ -416,7 +416,28 @@ function listBucketIter(bucket, cb) { const objectUrl = getObjectURL(bucket, item.key); let digestKey = item.key; - const md = JSON.parse(item.value); + let md; + try { + md = JSON.parse(item.value); + } catch (e) { + log.error('Invalid Metadata JSON', { + object: objectUrl, + error: e.message, + mdValue: item.value, + }); + status.objectsScanned += 1; + status.objectsErrors +=1; + return itemDone(); + } + if (md.isNull) { + log.error('Null Metadata JSON', { + object: objectUrl, + mdValue: item.value, + }); + status.objectsScanned += 1; + status.objectsErrors +=1; + return itemDone(); + } if (vidSepPos === -1) { const reVersionIds = /"versionId":"([^"]*)"/g; const versionIds = []; From 8b576a24c9f79fb394460e114639628ce5551d99 Mon Sep 17 00:00:00 2001 From: Fra Date: Tue, 21 Nov 2023 12:46:53 +0100 Subject: [PATCH 2/9] chore linted --- verifyBucketSproxydKeys.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/verifyBucketSproxydKeys.js b/verifyBucketSproxydKeys.js index bed8656e..ffa9227c 100644 --- a/verifyBucketSproxydKeys.js +++ b/verifyBucketSproxydKeys.js @@ -415,27 +415,26 @@ function listBucketIter(bucket, cb) { const vidSepPos = item.key.lastIndexOf('\0'); const objectUrl = getObjectURL(bucket, item.key); let digestKey = item.key; - let md; try { md = JSON.parse(item.value); - } catch (e) { + } catch (e) { log.error('Invalid Metadata JSON', { object: objectUrl, error: e.message, mdValue: item.value, }); status.objectsScanned += 1; - status.objectsErrors +=1; + status.objectsErrors += 1; return itemDone(); - } + } if (md.isNull) { log.error('Null Metadata JSON', { object: objectUrl, mdValue: item.value, }); status.objectsScanned += 1; - status.objectsErrors +=1; + status.objectsErrors += 1; return itemDone(); } if (vidSepPos === -1) { From bc925c3316c9785126e6bc6d9bb24b7034881bea Mon Sep 17 00:00:00 2001 From: Fra Date: Thu, 23 Nov 2023 16:19:00 +0100 Subject: [PATCH 3/9] redefined if for null --- verifyBucketSproxydKeys.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verifyBucketSproxydKeys.js b/verifyBucketSproxydKeys.js index ffa9227c..0ba89186 100644 --- a/verifyBucketSproxydKeys.js +++ b/verifyBucketSproxydKeys.js @@ -428,7 +428,7 @@ function listBucketIter(bucket, cb) { status.objectsErrors += 1; return itemDone(); } - if (md.isNull) { + if (md.isNull && !md.location && !md["md-model-version"] ) { log.error('Null Metadata JSON', { object: objectUrl, mdValue: item.value, From 3e57d5eb0266529d6850f46efd9710cd6df38ffd Mon Sep 17 00:00:00 2001 From: Fra Date: Thu, 23 Nov 2023 16:32:32 +0100 Subject: [PATCH 4/9] Strings must use singlequote --- verifyBucketSproxydKeys.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verifyBucketSproxydKeys.js b/verifyBucketSproxydKeys.js index 0ba89186..2d01c302 100644 --- a/verifyBucketSproxydKeys.js +++ b/verifyBucketSproxydKeys.js @@ -428,7 +428,7 @@ function listBucketIter(bucket, cb) { status.objectsErrors += 1; return itemDone(); } - if (md.isNull && !md.location && !md["md-model-version"] ) { + if (md.isNull && !md.location && !md['md-model-version'] ) { log.error('Null Metadata JSON', { object: objectUrl, mdValue: item.value, From 468471fd76fc58401d4aece019161b9fd51333e0 Mon Sep 17 00:00:00 2001 From: Fra Date: Thu, 23 Nov 2023 16:44:45 +0100 Subject: [PATCH 5/9] There should be no space before this paren --- verifyBucketSproxydKeys.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/verifyBucketSproxydKeys.js b/verifyBucketSproxydKeys.js index 2d01c302..7b72eec2 100644 --- a/verifyBucketSproxydKeys.js +++ b/verifyBucketSproxydKeys.js @@ -428,7 +428,7 @@ function listBucketIter(bucket, cb) { status.objectsErrors += 1; return itemDone(); } - if (md.isNull && !md.location && !md['md-model-version'] ) { + if (md.isNull && !md.location && !md['md-model-version']) { log.error('Null Metadata JSON', { object: objectUrl, mdValue: item.value, From b3fc6ed69da06244848cb4936797da3d55633f65 Mon Sep 17 00:00:00 2001 From: Fra Date: Fri, 24 Nov 2023 08:50:44 +0100 Subject: [PATCH 6/9] Refined check for NotComplete MD --- verifyBucketSproxydKeys.js | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/verifyBucketSproxydKeys.js b/verifyBucketSproxydKeys.js index 7b72eec2..cbaa4e4d 100644 --- a/verifyBucketSproxydKeys.js +++ b/verifyBucketSproxydKeys.js @@ -384,7 +384,18 @@ function fetchAndCheckObject(bucket, itemKey, cb) { return checkSproxydKeys(objectUrl, locations, () => cb(locations)); }); } - +function hasMinimalMetadata(md) { + if (!md) { + return false + } + return md.hasOwnProperty('md-model-version') + && md.hasOwnProperty('owner-display-name') + && md.hasOwnProperty('owner-id') + && md.hasOwnProperty('content-length') + && md.hasOwnProperty('content-type') + && md.hasOwnProperty('last-modified') + && md.hasOwnProperty('content-md5') +} function listBucketIter(bucket, cb) { const url = getBucketdURL(BUCKETD_HOSTPORT, { Bucket: bucket, @@ -428,8 +439,17 @@ function listBucketIter(bucket, cb) { status.objectsErrors += 1; return itemDone(); } - if (md.isNull && !md.location && !md['md-model-version']) { - log.error('Null Metadata JSON', { + if (item.value === '{}') { + log.error('object with empty metadata found', { + objectUrl, + }); + status.objectsScanned += 1; + status.objectsWithEmptyMetadata += 1; + findDuplicateSproxydKeys.skipVersion(); + return itemDone(); + } + if (!hasMinimalMetadata(md)) { + log.error('Not Complete Metadata JSON', { object: objectUrl, mdValue: item.value, }); @@ -478,16 +498,6 @@ function listBucketIter(bucket, cb) { } } - if (item.value === '{}') { - log.error('object with empty metadata found', { - objectUrl, - }); - status.objectsScanned += 1; - status.objectsWithEmptyMetadata += 1; - findDuplicateSproxydKeys.skipVersion(); - return itemDone(); - } - if (md.isPHD) { // object is a Place Holder Delete (PHD) findDuplicateSproxydKeys.skipVersion(); From fd6815890508bfda90e8923689c1a9dfc707313d Mon Sep 17 00:00:00 2001 From: Fra Date: Fri, 24 Nov 2023 08:59:39 +0100 Subject: [PATCH 7/9] linted --- verifyBucketSproxydKeys.js | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/verifyBucketSproxydKeys.js b/verifyBucketSproxydKeys.js index cbaa4e4d..af50de64 100644 --- a/verifyBucketSproxydKeys.js +++ b/verifyBucketSproxydKeys.js @@ -386,15 +386,17 @@ function fetchAndCheckObject(bucket, itemKey, cb) { } function hasMinimalMetadata(md) { if (!md) { - return false + return false; } - return md.hasOwnProperty('md-model-version') - && md.hasOwnProperty('owner-display-name') - && md.hasOwnProperty('owner-id') - && md.hasOwnProperty('content-length') - && md.hasOwnProperty('content-type') - && md.hasOwnProperty('last-modified') - && md.hasOwnProperty('content-md5') + return ( + md.hasOwnProperty("md-model-version") && + md.hasOwnProperty("owner-display-name") && + md.hasOwnProperty("owner-id") && + md.hasOwnProperty("content-length") && + md.hasOwnProperty("content-type") && + md.hasOwnProperty("last-modified") && + md.hasOwnProperty("content-md5") + ); } function listBucketIter(bucket, cb) { const url = getBucketdURL(BUCKETD_HOSTPORT, { From 4040b5c1a287fc20bdde828ec05764b04d74b96a Mon Sep 17 00:00:00 2001 From: Fra Date: Fri, 24 Nov 2023 09:49:29 +0100 Subject: [PATCH 8/9] linted --- verifyBucketSproxydKeys.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/verifyBucketSproxydKeys.js b/verifyBucketSproxydKeys.js index af50de64..beb81f39 100644 --- a/verifyBucketSproxydKeys.js +++ b/verifyBucketSproxydKeys.js @@ -389,13 +389,13 @@ function hasMinimalMetadata(md) { return false; } return ( - md.hasOwnProperty("md-model-version") && - md.hasOwnProperty("owner-display-name") && - md.hasOwnProperty("owner-id") && - md.hasOwnProperty("content-length") && - md.hasOwnProperty("content-type") && - md.hasOwnProperty("last-modified") && - md.hasOwnProperty("content-md5") + md.hasOwnProperty('md-model-version') + && md.hasOwnProperty('owner-display-name') + && md.hasOwnProperty('owner-id') + && md.hasOwnProperty('content-length') + && md.hasOwnProperty('content-type') + && md.hasOwnProperty('last-modified') + && md.hasOwnProperty('content-md5') ); } function listBucketIter(bucket, cb) { From f4b9fe477b18807bdde1b66206c40da0f9b22fe8 Mon Sep 17 00:00:00 2001 From: Fra Date: Fri, 24 Nov 2023 09:57:51 +0100 Subject: [PATCH 9/9] linted again --- verifyBucketSproxydKeys.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/verifyBucketSproxydKeys.js b/verifyBucketSproxydKeys.js index beb81f39..d9d854b6 100644 --- a/verifyBucketSproxydKeys.js +++ b/verifyBucketSproxydKeys.js @@ -389,9 +389,9 @@ function hasMinimalMetadata(md) { return false; } return ( - md.hasOwnProperty('md-model-version') - && md.hasOwnProperty('owner-display-name') - && md.hasOwnProperty('owner-id') + md.hasOwnProperty('md-model-version') + && md.hasOwnProperty('owner-display-name') + && md.hasOwnProperty('owner-id') && md.hasOwnProperty('content-length') && md.hasOwnProperty('content-type') && md.hasOwnProperty('last-modified')