From 537e06238601b1c0eb412546347f532ec21637fd Mon Sep 17 00:00:00 2001 From: victor5lm Date: Tue, 10 Sep 2024 14:00:19 +0200 Subject: [PATCH 01/10] Added a permissions field in the global section --- bu_isciii/conf/configuration.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) mode change 100755 => 100644 bu_isciii/conf/configuration.json diff --git a/bu_isciii/conf/configuration.json b/bu_isciii/conf/configuration.json old mode 100755 new mode 100644 index d4e4f86ea..b1ce926eb --- a/bu_isciii/conf/configuration.json +++ b/bu_isciii/conf/configuration.json @@ -2,7 +2,11 @@ "global": { "data_path": "/data/bi", "archived_path": "/archived/bi", - "yaml_conf_path": "~/buisciii_config.yml" + "yaml_conf_path": "~/buisciii_config.yml", + "permissions": { + "directory_chmod": "775", + "file_chmod": "664" + } }, "sftp_copy": { "protocol": "rsync", From d7348f655cd250042c8f6e3eeda11c85315c88c3 Mon Sep 17 00:00:00 2001 From: victor5lm Date: Tue, 10 Sep 2024 14:01:45 +0200 Subject: [PATCH 02/10] Added a remake_permissions() function to utils.py to allow permissions correction --- bu_isciii/utils.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/bu_isciii/utils.py b/bu_isciii/utils.py index 6de43fe02..b2832d5c5 100755 --- a/bu_isciii/utils.py +++ b/bu_isciii/utils.py @@ -7,6 +7,7 @@ import os import tarfile import sys +import subprocess import questionary import rich @@ -495,3 +496,30 @@ def process_yaml_file(yaml_file): def validate_date(date_previous, date_posterior): return + + +def remake_permissions(copied_folder_path, permissions_config): + """ + Change permissions of all files and directories in a given absolute path. + + Args: + copied_folder_path: The path to the folder that was copied + permissions_config: Dictionary containing permissions configuration (e.g., {'directory_chmod': '755', 'file_chmod': '664'}) + """ + for root, dirs, files in os.walk(copied_folder_path): + # Full paths for directories + dirpaths = [os.path.join(root, dir) for dir in dirs] + + # Full paths for files + filepaths = [os.path.join(root, file) for file in files] + + # Change permissions for directories + for dir_path in dirpaths: + if "directory_chmod" in permissions_config: + subprocess.run(f"chown -R $(whoami):bi {copied_folder_path}", shell=True, check=True) + subprocess.run(f"find {copied_folder_path} -type d -exec chmod {permissions_config['directory_chmod']} {{}} \;", shell=True, check=True) + + # Change permissions for files + for file_path in filepaths: + if "file_chmod" in permissions_config: + subprocess.run(f"find {copied_folder_path} -type f -exec chmod {permissions_config['file_chmod']} {{}} \;", shell=True, check=True) From d82c380ae2abef1d3c622c07cab433ba8d6c7fdd Mon Sep 17 00:00:00 2001 From: victor5lm Date: Tue, 10 Sep 2024 14:02:43 +0200 Subject: [PATCH 03/10] Updated scratch.py to always make permissions correct after copying files to the service folder --- bu_isciii/scratch.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bu_isciii/scratch.py b/bu_isciii/scratch.py index d165c8e92..3c9cb77aa 100755 --- a/bu_isciii/scratch.py +++ b/bu_isciii/scratch.py @@ -187,6 +187,12 @@ def revert_copy_scratch(self): sync_source_contents=False, ) self.srun_command(self.srun_settings, rsync_command) + + # After successful rsync, apply correct permissions + conf = bu_isciii.config_json.ConfigJson() + permissions_config = conf.get_configuration("global").get("permissions") + bu_isciii.utils.remake_permissions(self.full_path, permissions_config) + stderr.print( "[green]Successfully copied the directory to %s" % dest_folder, From 0d09fac7058ef3f896b6ba88acf066f9c08f7efc Mon Sep 17 00:00:00 2001 From: victor5lm Date: Tue, 10 Sep 2024 14:03:17 +0200 Subject: [PATCH 04/10] Added a new bu-isciii function to fix permissions of a custom folder --- bu_isciii/__main__.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/bu_isciii/__main__.py b/bu_isciii/__main__.py index 352bd6ab5..4e59a91dd 100755 --- a/bu_isciii/__main__.py +++ b/bu_isciii/__main__.py @@ -649,5 +649,30 @@ def autoclean_sftp(ctx, sftp_folder, days): sftp_clean.handle_autoclean_sftp() +# FIX PERMISSIONS +@bu_isciii_cli.command(help_priority=9) +@click.option( + "-d", + "--input_directory", + type=click.Path(), + default=None, + required=True, + help="Input directory to fix permissions (absolute path)", +) +@click.pass_context +def fix_permissions(ctx, input_directory): + """ + Fix permissions + """ + if not os.path.isdir(input_directory): + exit("Invalid input directory") + conf = bu_isciii.config_json.ConfigJson() + permissions = conf.get_configuration("global").get("permissions") + bu_isciii.utils.remake_permissions(input_directory, permissions) + stderr = rich.console.Console( + stderr=True, force_terminal=bu_isciii.utils.rich_force_colors() + ) + stderr.print(f"[green]Correct permissions were applied to {input_directory}") + if __name__ == "__main__": run_bu_isciii() From 6c5a86cb1fee351ab468ba9f2ff8f1cf749dfdff Mon Sep 17 00:00:00 2001 From: victor5lm Date: Tue, 10 Sep 2024 14:12:40 +0200 Subject: [PATCH 05/10] Updated CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bb905aa2..d8356c716 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,7 @@ Code contributions to the new version: - Added full execution support for the MAG template [#321](https://github.com/BU-ISCIII/buisciii-tools/pull/321). - Added labels to services.json and updated bioinfo_doc.py and jinja_template_delivery.j2 so that software versions data is displayed in the delivery pdf [#330](https://github.com/BU-ISCIII/buisciii-tools/pull/330). - Updated several templates (singularity images, outdated paths, improvements, etc) [#331](https://github.com/BU-ISCIII/buisciii-tools/pull/331) +- Added permissions fixing after running scratch_copy, as well as a new fix-permissions module in the tools [#332](https://github.com/BU-ISCIII/buisciii-tools/pull/332). ### Modules From 32e6a73cd2e9a514826d34598dd018b68aa931aa Mon Sep 17 00:00:00 2001 From: victor5lm Date: Tue, 10 Sep 2024 14:15:20 +0200 Subject: [PATCH 06/10] Fixed linting mistake --- bu_isciii/__main__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bu_isciii/__main__.py b/bu_isciii/__main__.py index 4e59a91dd..16e99303d 100755 --- a/bu_isciii/__main__.py +++ b/bu_isciii/__main__.py @@ -674,5 +674,6 @@ def fix_permissions(ctx, input_directory): ) stderr.print(f"[green]Correct permissions were applied to {input_directory}") + if __name__ == "__main__": run_bu_isciii() From f93081379763a793f6e814984d017415fee453c6 Mon Sep 17 00:00:00 2001 From: victor5lm Date: Tue, 10 Sep 2024 14:17:18 +0200 Subject: [PATCH 07/10] Ran black on scratch.py and utils.py --- bu_isciii/scratch.py | 8 ++++++-- bu_isciii/utils.py | 18 +++++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/bu_isciii/scratch.py b/bu_isciii/scratch.py index 3c9cb77aa..529493abc 100755 --- a/bu_isciii/scratch.py +++ b/bu_isciii/scratch.py @@ -190,8 +190,12 @@ def revert_copy_scratch(self): # After successful rsync, apply correct permissions conf = bu_isciii.config_json.ConfigJson() - permissions_config = conf.get_configuration("global").get("permissions") - bu_isciii.utils.remake_permissions(self.full_path, permissions_config) + permissions_config = conf.get_configuration("global").get( + "permissions" + ) + bu_isciii.utils.remake_permissions( + self.full_path, permissions_config + ) stderr.print( "[green]Successfully copied the directory to %s" diff --git a/bu_isciii/utils.py b/bu_isciii/utils.py index b2832d5c5..62248c0bb 100755 --- a/bu_isciii/utils.py +++ b/bu_isciii/utils.py @@ -516,10 +516,22 @@ def remake_permissions(copied_folder_path, permissions_config): # Change permissions for directories for dir_path in dirpaths: if "directory_chmod" in permissions_config: - subprocess.run(f"chown -R $(whoami):bi {copied_folder_path}", shell=True, check=True) - subprocess.run(f"find {copied_folder_path} -type d -exec chmod {permissions_config['directory_chmod']} {{}} \;", shell=True, check=True) + subprocess.run( + f"chown -R $(whoami):bi {copied_folder_path}", + shell=True, + check=True, + ) + subprocess.run( + f"find {copied_folder_path} -type d -exec chmod {permissions_config['directory_chmod']} {{}} \;", + shell=True, + check=True, + ) # Change permissions for files for file_path in filepaths: if "file_chmod" in permissions_config: - subprocess.run(f"find {copied_folder_path} -type f -exec chmod {permissions_config['file_chmod']} {{}} \;", shell=True, check=True) + subprocess.run( + f"find {copied_folder_path} -type f -exec chmod {permissions_config['file_chmod']} {{}} \;", + shell=True, + check=True, + ) From 95e21306d51641c0bf9c5be1e09b08d1c96cd420 Mon Sep 17 00:00:00 2001 From: victor5lm Date: Tue, 10 Sep 2024 16:30:30 +0200 Subject: [PATCH 08/10] Used 2775 for directory chmod in configuration.json --- bu_isciii/conf/configuration.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bu_isciii/conf/configuration.json b/bu_isciii/conf/configuration.json index b1ce926eb..3551c2766 100644 --- a/bu_isciii/conf/configuration.json +++ b/bu_isciii/conf/configuration.json @@ -4,7 +4,7 @@ "archived_path": "/archived/bi", "yaml_conf_path": "~/buisciii_config.yml", "permissions": { - "directory_chmod": "775", + "directory_chmod": "2775", "file_chmod": "664" } }, From c1c42a73967c661dbf0bb71fdb64d085042c1b9b Mon Sep 17 00:00:00 2001 From: victor5lm Date: Tue, 10 Sep 2024 16:31:20 +0200 Subject: [PATCH 09/10] Simplified the remake_permissions() function in utils.py --- bu_isciii/utils.py | 36 ++++++++---------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/bu_isciii/utils.py b/bu_isciii/utils.py index 62248c0bb..f50f77583 100755 --- a/bu_isciii/utils.py +++ b/bu_isciii/utils.py @@ -506,32 +506,12 @@ def remake_permissions(copied_folder_path, permissions_config): copied_folder_path: The path to the folder that was copied permissions_config: Dictionary containing permissions configuration (e.g., {'directory_chmod': '755', 'file_chmod': '664'}) """ - for root, dirs, files in os.walk(copied_folder_path): - # Full paths for directories - dirpaths = [os.path.join(root, dir) for dir in dirs] - - # Full paths for files - filepaths = [os.path.join(root, file) for file in files] - - # Change permissions for directories - for dir_path in dirpaths: - if "directory_chmod" in permissions_config: - subprocess.run( - f"chown -R $(whoami):bi {copied_folder_path}", - shell=True, - check=True, - ) - subprocess.run( - f"find {copied_folder_path} -type d -exec chmod {permissions_config['directory_chmod']} {{}} \;", - shell=True, - check=True, - ) + subprocess.run(f"chown -R $(whoami):bi {copied_folder_path}", shell=True, check=True) - # Change permissions for files - for file_path in filepaths: - if "file_chmod" in permissions_config: - subprocess.run( - f"find {copied_folder_path} -type f -exec chmod {permissions_config['file_chmod']} {{}} \;", - shell=True, - check=True, - ) + # Change permissions for directories + if "directory_chmod" in permissions_config: + subprocess.run(f"find {copied_folder_path} -type d -exec chmod {permissions_config['directory_chmod']} {{}} \;", shell=True, check=True) + + # Change permissions for files + if "file_chmod" in permissions_config: + subprocess.run(f"find {copied_folder_path} -type f -exec chmod {permissions_config['file_chmod']} {{}} \;", shell=True, check=True) From 179b325474c10e3b16d0deaf4aa65c5440662747 Mon Sep 17 00:00:00 2001 From: victor5lm Date: Tue, 10 Sep 2024 16:32:54 +0200 Subject: [PATCH 10/10] Ran black on utils.py --- bu_isciii/utils.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/bu_isciii/utils.py b/bu_isciii/utils.py index f50f77583..eca3c484d 100755 --- a/bu_isciii/utils.py +++ b/bu_isciii/utils.py @@ -506,12 +506,22 @@ def remake_permissions(copied_folder_path, permissions_config): copied_folder_path: The path to the folder that was copied permissions_config: Dictionary containing permissions configuration (e.g., {'directory_chmod': '755', 'file_chmod': '664'}) """ - subprocess.run(f"chown -R $(whoami):bi {copied_folder_path}", shell=True, check=True) + subprocess.run( + f"chown -R $(whoami):bi {copied_folder_path}", shell=True, check=True + ) # Change permissions for directories if "directory_chmod" in permissions_config: - subprocess.run(f"find {copied_folder_path} -type d -exec chmod {permissions_config['directory_chmod']} {{}} \;", shell=True, check=True) + subprocess.run( + f"find {copied_folder_path} -type d -exec chmod {permissions_config['directory_chmod']} {{}} \;", + shell=True, + check=True, + ) # Change permissions for files if "file_chmod" in permissions_config: - subprocess.run(f"find {copied_folder_path} -type f -exec chmod {permissions_config['file_chmod']} {{}} \;", shell=True, check=True) + subprocess.run( + f"find {copied_folder_path} -type f -exec chmod {permissions_config['file_chmod']} {{}} \;", + shell=True, + check=True, + )