NC | Online Upgrade | Integration tests | Add CLI and S3 integration tests

docs/NooBaaNonContainerized/CI&Tests.md

@@ -91,6 +91,7 @@ The following is a list of `NC mocha test` files -
 2. `test_nc_health` - Tests NooBaa Health CLI.  
 3. `test_nsfs_glacier_backend.js` - Tests NooBaa Glacier Backend.  
 4. `test_nc_with_a_couple_of_forks.js` - Tests the `bucket_namespace_cache` when running with a couple of forks. Please notice that it uses `nc_coretest` with setup that includes a couple of forks.
+5. `test_nc_online_upgrade_s3_integrations.js` - Tests S3 operations during mocked config directory upgrade.
 
 #### NC Jest test files
 The following is a list of `NC jest tests` files -   
@@ -112,6 +113,7 @@ The following is a list of `NC jest tests` files -
 16. `test_config_fs_backward_compatibility.test.js` - Tests of the backwards compatibility of configFS functions.
 17. `test_nc_upgrade_manager.test.js` - Tests of the NC upgrade manager.
 18. `test_cli_upgrade.test.js` - Tests of the upgrade CLI commands.
+19. `test_nc_online_upgrade_cli_integrations.test.js` - Tests CLI commands during mocked config directory upgrade.
 
 #### nc_index.js File
 * The `nc_index.js` is a file that runs several NC and NSFS mocha related tests.  

src/test/system_tests/test_utils.js

@@ -687,6 +687,32 @@ async function create_file(fs_context, file_path, file_data) {
     );
 }
 
+/**
+ * create_system_json creates the system.json file
+ * if mock_config_dir_version it sets it before creating the file
+ * @param {import('../../sdk/config_fs').ConfigFS} config_fs
+ * @param {String} [mock_config_dir_version] 
+ * @returns {Promise<Void>}
+ */
+async function create_system_json(config_fs, mock_config_dir_version) {
+    const system_data = await config_fs._get_new_system_json_data();
+    if (mock_config_dir_version) system_data.config_directory.config_dir_version = mock_config_dir_version;
+    await config_fs.create_system_config_file(JSON.stringify(system_data));
+}
+
+/**
+ * update_system_json updates the system.json file
+ * if mock_config_dir_version it sets it before creating the file
+ * @param {import('../../sdk/config_fs').ConfigFS} config_fs
+ * @param {String} [mock_config_dir_version] 
+ * @returns {Promise<Void>}
+ */
+async function update_system_json(config_fs, mock_config_dir_version) {
+    const system_data = await config_fs.get_system_config_file();
+    if (mock_config_dir_version) system_data.config_directory.config_dir_version = mock_config_dir_version;
+    await config_fs.update_system_config_file(JSON.stringify(system_data));
+}
+
 exports.blocks_exist_on_cloud = blocks_exist_on_cloud;
 exports.create_hosts_pool = create_hosts_pool;
 exports.delete_hosts_pool = delete_hosts_pool;
@@ -717,6 +743,8 @@ exports.symlink_account_access_keys = symlink_account_access_keys;
 exports.create_file = create_file;
 exports.create_redirect_file = create_redirect_file;
 exports.delete_redirect_file = delete_redirect_file;
+exports.create_system_json = create_system_json;
+exports.update_system_json = update_system_json;
 exports.fail_test_if_default_config_dir_exists = fail_test_if_default_config_dir_exists;
 exports.create_config_dir = create_config_dir;
 exports.clean_config_dir = clean_config_dir;

src/test/unit_tests/jest_tests/test_nc_online_upgrade_cli_integrations.js

@@ -0,0 +1,212 @@
+/* Copyright (C) 2016 NooBaa */
+'use strict';
+
+// disabling init_rand_seed as it takes longer than the actual test execution
+process.env.DISABLE_INIT_RANDOM_SEED = "true";
+
+const path = require('path');
+const fs_utils = require('../../../util/fs_utils');
+const { exec_manage_cli, TMP_PATH, create_system_json } = require('../../system_tests/test_utils');
+const { folder_delete, create_fresh_path } = require('../../../util/fs_utils');
+const { TYPES, ACTIONS } = require('../../../manage_nsfs/manage_nsfs_constants');
+const { ManageCLIError } = require('../../../manage_nsfs/manage_nsfs_cli_errors');
+const { ManageCLIResponse } = require('../../../manage_nsfs/manage_nsfs_cli_responses');
+
+const config_root = path.join(TMP_PATH, 'test_online_upgrade_cli_integrations');
+const { ConfigFS } = require('../../../sdk/config_fs');
+const config_fs = new ConfigFS(config_root);
+const default_new_buckets_path = path.join(TMP_PATH, 'default_new_buckets_path_online_upgrade_cli_integrations_test');
+const bucket_storage_path = path.join(default_new_buckets_path, 'bucket1');
+const old_config_dir_version = '0.0.0';
+
+const default_account_options = {
+    config_root,
+    name: 'account1',
+    new_buckets_path: default_new_buckets_path,
+    uid: process.getuid(),
+    gid: process.getgid(),
+};
+
+const default_bucket_options = {
+    config_root,
+    name: 'bucket1',
+    path: bucket_storage_path,
+    owner: default_account_options.name
+};
+
+describe('online upgrade CLI bucket operations tests', function() {
+    beforeAll(async () => {
+        await create_fresh_path(config_root);
+        await create_fresh_path(default_new_buckets_path, 770);
+        await fs_utils.file_must_exist(default_new_buckets_path);
+        await create_default_account();
+        await create_fresh_path(bucket_storage_path, 770);
+        await fs_utils.file_must_exist(bucket_storage_path);
+    });
+
+    afterAll(async () => {
+        await folder_delete(config_root);
+        await folder_delete(default_new_buckets_path);
+    });
+
+    afterEach(async () => {
+        await fs_utils.file_delete(config_fs.system_json_path);
+        await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, { config_root, name: default_bucket_options.name }, true);
+    });
+
+    it('create bucket - host is blocked for config dir updates - should fail', async function() {
+        await create_system_json(config_fs, old_config_dir_version);
+        const res = await exec_manage_cli(TYPES.BUCKET, ACTIONS.ADD, default_bucket_options, true);
+        expect(JSON.parse(res.stdout).error.message).toBe(ManageCLIError.ConfigDirUpdateBlocked.message);
+    });
+
+    it('create bucket - host is not blocked for config dir updates', async function() {
+        await create_system_json(config_fs);
+        const res = await exec_manage_cli(TYPES.BUCKET, ACTIONS.ADD, default_bucket_options, true);
+        expect(JSON.parse(res).response.code).toBe(ManageCLIResponse.BucketCreated.code);
+    });
+
+    it('update bucket - host is blocked for config dir updates - should fail', async function() {
+        await create_default_bucket();
+        await create_system_json(config_fs, old_config_dir_version);
+        const update_bucket_options = { ...default_bucket_options, name: default_bucket_options.name, new_name: 'bucket2' };
+        const update_res = await exec_manage_cli(TYPES.BUCKET, ACTIONS.UPDATE, update_bucket_options, true);
+        expect(JSON.parse(update_res.stdout).error.message).toBe(ManageCLIError.ConfigDirUpdateBlocked.message);
+    });
+
+    it('update bucket - host is not blocked for config dir updates', async function() {
+        await create_default_bucket();
+        await create_system_json(config_fs);
+        const update_bucket_options = { ...default_bucket_options, name: default_bucket_options.name, new_name: 'bucket2' };
+        const update_res = await exec_manage_cli(TYPES.BUCKET, ACTIONS.UPDATE, update_bucket_options, true);
+        expect(JSON.parse(update_res).response.code).toBe(ManageCLIResponse.BucketUpdated.code);
+    });
+
+    it('delete bucket - host is blocked for config dir updates - should fail', async function() {
+        await create_default_bucket();
+        await create_system_json(config_fs, old_config_dir_version);
+        const delete_bucket_options = { config_root, name: default_bucket_options.name };
+        const delete_res = await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, delete_bucket_options, true);
+        expect(JSON.parse(delete_res.stdout).error.message).toBe(ManageCLIError.ConfigDirUpdateBlocked.message);
+    });
+
+    it('delete bucket - host is not blocked for config dir updates', async function() {
+        await create_default_bucket();
+        await create_system_json(config_fs);
+        const delete_bucket_options = { config_root, name: default_bucket_options.name };
+        const delete_res = await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, delete_bucket_options, true);
+        expect(JSON.parse(delete_res).response.code).toBe(ManageCLIResponse.BucketDeleted.code);
+    });
+
+    it('list buckets - old config dir version - success', async function() {
+        await create_default_bucket();
+        await create_system_json(config_fs, old_config_dir_version);
+        const list_bucket_options = { config_root };
+        const list_res = await exec_manage_cli(TYPES.BUCKET, ACTIONS.LIST, list_bucket_options, true);
+        expect(JSON.parse(list_res).response.code).toBe(ManageCLIResponse.BucketList.code);
+    });
+
+    it('bucket status - old config dir version - success', async function() {
+        await create_default_bucket();
+        await create_system_json(config_fs, old_config_dir_version);
+        const status_bucket_options = { config_root, name: default_bucket_options.name };
+        const status_res = await exec_manage_cli(TYPES.BUCKET, ACTIONS.STATUS, status_bucket_options, true);
+        expect(JSON.parse(status_res).response.code).toBe(ManageCLIResponse.BucketStatus.code);
+    });
+});
+
+describe('online upgrade CLI account operations tests', function() {
+    beforeAll(async () => {
+        await create_fresh_path(config_root);
+        await create_fresh_path(default_new_buckets_path, 770);
+    });
+
+    afterAll(async () => {
+        await folder_delete(config_root);
+        await folder_delete(default_new_buckets_path);
+    });
+
+    afterEach(async () => {
+        await fs_utils.file_delete(config_fs.system_json_path);
+        await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: default_account_options.name }, true);
+    });
+
+    it('create account - host is blocked for config dir updates - should fail', async function() {
+        await create_system_json(config_fs, old_config_dir_version);
+        const res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, default_account_options, true);
+        expect(JSON.parse(res.stdout).error.message).toBe(ManageCLIError.ConfigDirUpdateBlocked.message);
+    });
+
+    it('create account - host is not blocked for config dir updates', async function() {
+        await create_system_json(config_fs);
+        const res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, default_account_options, true);
+        expect(JSON.parse(res).response.code).toBe(ManageCLIResponse.AccountCreated.code);
+    });
+
+    it('update account - host is blocked for config dir updates - should fail', async function() {
+        await create_default_account();
+        await create_system_json(config_fs, old_config_dir_version);
+        const update_account_options = { ...default_account_options, name: default_account_options.name, new_name: 'account2' };
+        const update_res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.UPDATE, update_account_options, true);
+        expect(JSON.parse(update_res.stdout).error.message).toBe(ManageCLIError.ConfigDirUpdateBlocked.message);
+    });
+
+    it('update account - host is not blocked for config dir updates', async function() {
+        await create_default_account();
+        await create_system_json(config_fs);
+        const update_account_options = { ...default_account_options, name: default_account_options.name, new_name: 'account2' };
+        const update_res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.UPDATE, update_account_options, true);
+        expect(JSON.parse(update_res).response.code).toBe(ManageCLIResponse.AccountUpdated.code);
+    });
+
+    it('delete account - host is blocked for config dir updates - should fail', async function() {
+        await create_default_account();
+        await create_system_json(config_fs, old_config_dir_version);
+        const delete_account_options = { config_root, name: default_account_options.name };
+        const delete_res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, delete_account_options, true);
+        expect(JSON.parse(delete_res.stdout).error.message).toBe(ManageCLIError.ConfigDirUpdateBlocked.message);
+    });
+
+    it('delete account - host is not blocked for config dir updates', async function() {
+        await create_default_account();
+        await create_system_json(config_fs);
+        const delete_account_options = { config_root, name: default_account_options.name };
+        const delete_res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, delete_account_options, true);
+        expect(JSON.parse(delete_res).response.code).toBe(ManageCLIResponse.AccountDeleted.code);
+    });
+
+    it('list accounts - old config dir version - success', async function() {
+        await create_default_account();
+        await create_system_json(config_fs, old_config_dir_version);
+
+        const list_account_options = { config_root };
+        const list_res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.LIST, list_account_options, true);
+        expect(JSON.parse(list_res).response.code).toBe(ManageCLIResponse.AccountList.code);
+    });
+
+    it('account status - old config dir version - success', async function() {
+        await create_default_account();
+        await create_system_json(config_fs, old_config_dir_version);
+        const status_account_options = { config_root, name: default_account_options.name };
+        const status_res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.STATUS, status_account_options, true);
+        expect(JSON.parse(status_res).response.code).toBe(ManageCLIResponse.AccountStatus.code);
+    });
+});
+
+/**
+ * create_default_bucket creates the default bucket for tests that require an existing bucket
+ * @returns {Promise<Void>}
+ */
+async function create_default_bucket() {
+    const res = await exec_manage_cli(TYPES.BUCKET, ACTIONS.ADD, default_bucket_options, true);
+    expect(JSON.parse(res).response.code).toBe(ManageCLIResponse.BucketCreated.code);
+}
+
+/**
+ * create_default_bucket creates the default bucket for tests that require an existing bucket
+ * @returns {Promise<Void>}
+ */
+async function create_default_account() {
+    const res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, default_account_options, true);
+    expect(JSON.parse(res).response.code).toBe(ManageCLIResponse.AccountCreated.code);
+}

src/test/unit_tests/nc_index.js

@@ -19,6 +19,7 @@ require('./test_s3_bucket_policy');
 require('./test_nsfs_versioning');
 require('./test_bucketspace_versioning');
 require('./test_nc_bucket_logging');
+require('./test_nc_online_upgrade_s3_integrations');
 
 // running with a couple of forks - please notice and add only relevant tests here
 require('./test_nc_with_a_couple_of_forks.js'); // please notice that we use a different setup