From fbd4d3c1bdfbc2a71d4007db848a95fffde97957 Mon Sep 17 00:00:00 2001 From: Danny Zaken Date: Wed, 22 May 2024 13:30:43 +0300 Subject: [PATCH] fix for list_objects * for postgres queries, there is no need to make an extra DB query, after processing the results of the first query. * in Mongo map-reduce, we could get to a situation where the query returned less than the limit, even though there are still relevant entries. * in Postgres, this is not happening, since the map_common_prefixes, iterates until it finds enough records (or until it reaches the end). * Performing the extra DB query in postgres, can cause a very long query that eventually returns 0 entries. One scenario is where we have a directory that is the last one returned in the previous query, and it has a large number of objects under it (.e.g /folder/file0 .. /folder/file9999999). when returning for the extra query, the marker that is used is 'folder/', so map_common_prefixes starts iterating from 'folder/', looking for all objects that do not start with 'folder/', but ends with a '/' (the delimiters). for postgres there is no smart way to do it, so it just going over all objects under 'folder/' and filetering them out. this can take over several minutes to complete. Signed-off-by: Danny Zaken (cherry picked from commit 219699d7c1e71ae506433c2a1365adaf9c9774e1) --- src/server/object_services/object_server.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server/object_services/object_server.js b/src/server/object_services/object_server.js index 2b47beb746..3fcaeb9899 100644 --- a/src/server/object_services/object_server.js +++ b/src/server/object_services/object_server.js @@ -1212,7 +1212,9 @@ function _list_add_results(state, results) { // this case avoids another last query when we got less results and no common prefixes // with common prefixes we cannot avoid the last query because the results might be // less than the requested limit although there are more results to fetch - if (!has_common_prefixes && count >= state.user_limit) { + // + // for postgres we should not do another query, since the list command returns the required limit + if (config.DB_TYPE === 'postgres' || (!has_common_prefixes && count >= state.user_limit)) { state.done = true; } }