NSFS | Improve list objects performance on top of NS FS (PR 2/3)

Author: naveenpaul1

config.js

@@ -798,6 +798,9 @@ config.NSFS_GLACIER_MIGRATE_INTERVAL = 15 * 60 * 1000;
 // of `manage_nsfs glacier restore`
 config.NSFS_GLACIER_RESTORE_INTERVAL = 15 * 60 * 1000;
 
+// enable/disable unsorted listing application level
+config.NSFS_LIST_OBJECTS_V2_UNSORTED_ENABLED = false; 
+
 // NSFS_GLACIER_EXPIRY_RUN_TIME must be of the format hh:mm which specifies
 // when NooBaa should allow running glacier expiry process
 // NOTE: This will also be in the same timezone as specified in

src/api/object_api.js

@@ -720,6 +720,9 @@ module.exports = {
                     limit: {
                         type: 'integer'
                     },
+                    list_type: {
+                        type: 'string',
+                    },
                 }
             },
             reply: {
@@ -774,6 +777,9 @@ module.exports = {
                     limit: {
                         type: 'integer'
                     },
+                    list_type: {
+                        type: 'string',
+                    },
                 }
             },
             reply: {

src/endpoint/s3/ops/s3_get_bucket.js

@@ -41,6 +41,7 @@ async function get_bucket(req) {
         bucket: req.params.bucket,
         prefix: req.query.prefix,
         delimiter: req.query.delimiter,
+        list_type: list_type,
         limit: Math.min(max_keys_received, 1000),
         key_marker: list_type === '2' ?
             (cont_tok_to_key_marker(cont_tok) || start_after) : req.query.marker,

src/native/fs/fs_napi.cpp

@@ -2042,7 +2042,7 @@ struct TellDir : public FSWrapWorker<DirWrap>
     }
     virtual void OnOK()
     {
-        DBG0("FS::Telldir::OnOK: " << DVAL(_wrap->_path) << DVAL(_tell_res));
+        DBG1("FS::Telldir::OnOK: " << DVAL(_wrap->_path) << DVAL(_tell_res));
         Napi::Env env = Env();
         auto res = Napi::BigInt::New(env, _tell_res);
         _deferred.Resolve(res);

src/sdk/namespace_fs.js

@@ -432,6 +432,22 @@ function get_new_buckets_path_by_test_env(new_buckets_full_path, new_buckets_dir
     return is_nc_coretest ? path.join(new_buckets_full_path, new_buckets_dir) : new_buckets_dir;
 }
 
+function make_dummy_object_sdk() {
+    return {
+        requesting_account: {
+            force_md5_etag: false,
+            nsfs_account_config: {
+                uid: process.getuid(),
+                gid: process.getgid(),
+            }
+        },
+        abort_controller: new AbortController(),
+        throw_if_aborted() {
+            if (this.abort_controller.signal.aborted) throw new Error('request aborted signal');
+        }
+    };
+}
+
 /**
  * write_manual_config_file writes config file directly to the file system without using config FS
  * used for creating backward compatibility tests, invalid config files etc
@@ -525,6 +541,7 @@ exports.delete_fs_user_by_platform = delete_fs_user_by_platform;
 exports.set_path_permissions_and_owner = set_path_permissions_and_owner;
 exports.set_nc_config_dir_in_config = set_nc_config_dir_in_config;
 exports.generate_anon_s3_client = generate_anon_s3_client;
+exports.make_dummy_object_sdk = make_dummy_object_sdk;
 exports.TMP_PATH = TMP_PATH;
 exports.is_nc_coretest = is_nc_coretest;
 exports.generate_nsfs_account = generate_nsfs_account;