From 1f36691385d62eb11a94514ee218b6a5a17c3aa3 Mon Sep 17 00:00:00 2001 From: Steven Soulen Date: Mon, 5 Apr 2021 17:43:56 -0500 Subject: [PATCH 1/4] Code clean up --- contrib/holland-commvault/holland_commvault/commvault.py | 1 - holland/commands/mk_config.py | 6 +++--- holland/core/command/command.py | 5 +++-- .../holland/backup/mysql_lvm/plugin/mysqldump/plugin.py | 4 ++-- .../holland/backup/mysql_lvm/plugin/raw/plugin.py | 3 ++- plugins/holland.lib.common/holland/lib/compression.py | 3 ++- 6 files changed, 12 insertions(+), 10 deletions(-) diff --git a/contrib/holland-commvault/holland_commvault/commvault.py b/contrib/holland-commvault/holland_commvault/commvault.py index f2bb59cb..3f29efa8 100644 --- a/contrib/holland-commvault/holland_commvault/commvault.py +++ b/contrib/holland-commvault/holland_commvault/commvault.py @@ -137,7 +137,6 @@ def main(): status.close() return ret except PidFileAlreadyLockedError: - ret = 1 pid_file = open(pid_location, "r") pid = pid_file.read() pid_file.close() diff --git a/holland/commands/mk_config.py b/holland/commands/mk_config.py index 1462d10b..a8e7f802 100644 --- a/holland/commands/mk_config.py +++ b/holland/commands/mk_config.py @@ -15,7 +15,7 @@ try: from holland.core.config.configobj import ConfigObj, ParseError, flatten_errors -except BaseException: +except ImportError: from configobj import ConfigObj, ParseError, flatten_errors LOG = logging.getLogger(__name__) @@ -215,12 +215,12 @@ def run(self, cmd, opts, *plugin_type): try: cfgspec = sys.modules[plugin_cls.__module__].CONFIGSPEC - except BaseException as ex: + except Exception as ex: print( "Could not load config-spec from plugin %r, %s" % (plugin_type[0], ex), file=sys.stderr, ) - return 1 + raise base_config = """ [holland:backup] diff --git a/holland/core/command/command.py b/holland/core/command/command.py index b0f8ffd2..38c0b86d 100644 --- a/holland/core/command/command.py +++ b/holland/core/command/command.py @@ -110,15 +110,16 @@ def dispatch(self, opts, args): try: return self.run(self.optparser.prog, opts, *args) except KeyboardInterrupt: + LOG.debug("Interrupted by user") raise except TypeError as ex: LOG.error("Failed comamnd %s': %s", self.optparser.prog, ex) return os.EX_SOFTWARE - except BaseException as ex: + except Exception as ex: LOG.error( "Uncaught exception while running command '%s': %r", self.optparser.prog, ex, exc_info=True, ) - return os.EX_SOFTWARE + raise diff --git a/plugins/holland.backup.mysql_lvm/holland/backup/mysql_lvm/plugin/mysqldump/plugin.py b/plugins/holland.backup.mysql_lvm/holland/backup/mysql_lvm/plugin/mysqldump/plugin.py index a34a261a..bfa502c1 100644 --- a/plugins/holland.backup.mysql_lvm/holland/backup/mysql_lvm/plugin/mysqldump/plugin.py +++ b/plugins/holland.backup.mysql_lvm/holland/backup/mysql_lvm/plugin/mysqldump/plugin.py @@ -124,8 +124,8 @@ def backup(self): spooldir=self.target_directory, plugin=self.mysqldump_plugin, ) - except BaseException as ex: - LOG.debug(ex) + except Exception as ex: + raise BackupError(str(ex)) if self.config["mysqldump"]["bin-log-position"]: LOG.warning("bin-log-position is not supported with mysqldump-lvm.") diff --git a/plugins/holland.backup.mysql_lvm/holland/backup/mysql_lvm/plugin/raw/plugin.py b/plugins/holland.backup.mysql_lvm/holland/backup/mysql_lvm/plugin/raw/plugin.py index 39265fb1..563baa0b 100644 --- a/plugins/holland.backup.mysql_lvm/holland/backup/mysql_lvm/plugin/raw/plugin.py +++ b/plugins/holland.backup.mysql_lvm/holland/backup/mysql_lvm/plugin/raw/plugin.py @@ -153,7 +153,8 @@ def backup(self): except LVMCommandError as exc: # Something failed in the snapshot process raise BackupError(str(exc)) - except BaseException as ex: + except Exception as ex: LOG.debug(ex) + raise return None diff --git a/plugins/holland.lib.common/holland/lib/compression.py b/plugins/holland.lib.common/holland/lib/compression.py index 9fe9a30c..c49b7eae 100644 --- a/plugins/holland.lib.common/holland/lib/compression.py +++ b/plugins/holland.lib.common/holland/lib/compression.py @@ -98,7 +98,7 @@ def __iter__(self): """ iter stdout """ - return iter(self.pid.stdout) + return self.pid.stdout def close(self): """ @@ -205,6 +205,7 @@ def close(self): ) pid = subprocess.Popen(argv, stdin=self.fileobj.fileno(), stdout=cmp_f.fileno()) status = pid.wait() + LOG.debug("Return status: %s", status) os.unlink(self.fileobj.name) else: self.pid.stdin.close() From b0af1eef4c30e9524cc92c785833f6172597c1cf Mon Sep 17 00:00:00 2001 From: Steven Soulen Date: Tue, 6 Apr 2021 11:32:43 -0500 Subject: [PATCH 2/4] Changes to support python3.5 --- plugins/holland.lib.common/holland/lib/compression.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/holland.lib.common/holland/lib/compression.py b/plugins/holland.lib.common/holland/lib/compression.py index c49b7eae..b371668e 100644 --- a/plugins/holland.lib.common/holland/lib/compression.py +++ b/plugins/holland.lib.common/holland/lib/compression.py @@ -139,7 +139,7 @@ def __init__(self, path, mode, argv, level, inline, split=False, splitsize=1): split_args = [ which("split"), "-a5", - f"--bytes={splitsize}G", + "--bytes=%sG" % splitsize, "-", path + ".", ] @@ -273,7 +273,7 @@ def open_stream( options=None, split=False, splitsize=1, - **kwargs, + **kwargs ): # pylint: disable=unused-argument """ Opens a compressed data stream, and returns a file descriptor type object From 803ba6c247a0b5e91a30ff3e63b2938581ea3534 Mon Sep 17 00:00:00 2001 From: stev9171 Date: Wed, 21 Apr 2021 15:34:32 -0500 Subject: [PATCH 3/4] fixes #331 --- .../holland/lib/mysql/client/base.py | 46 ++++++++++--------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/plugins/holland.lib.mysql/holland/lib/mysql/client/base.py b/plugins/holland.lib.mysql/holland/lib/mysql/client/base.py index f1ba430e..fe5e199b 100644 --- a/plugins/holland.lib.mysql/holland/lib/mysql/client/base.py +++ b/plugins/holland.lib.mysql/holland/lib/mysql/client/base.py @@ -43,7 +43,10 @@ def flatten_list(a_list): """ # isinstance check to ensure we're not iterating over characters # in a string - return sum([isinstance(item, (list, tuple)) and list(item) or [item] for item in a_list], []) + return sum( + [isinstance(item, (list, tuple)) and list(item) or [item] for item in a_list], + [], + ) class MySQLClient(object): @@ -235,7 +238,10 @@ def show_tables(self, database, full=False): :param full: Optional. include table type n the results :returns: list of table names """ - sql = "SHOW %sTABLES FROM `%s`" % (["", "FULL "][int(full)], database.replace("`", "``")) + sql = "SHOW %sTABLES FROM `%s`" % ( + ["", "FULL "][int(full)], + database.replace("`", "``"), + ) cursor = self.cursor() cursor.execute(sql) try: @@ -502,30 +508,26 @@ def connect(config, client_class=AutoMySQLClient): # map standard my.cnf parameters to # what pymysql.connect expects - # http://mysql-python.sourceforge.net/MySQLdb.html#mysqldb - cnf_to_mysqldb = { - "user": "user", # same - "password": "passwd", # weird - "host": "host", # same - "port": "port", - "socket": "unix_socket", - "ssl": "ssl", - "compress": "compress", - } + # https://pymysql.readthedocs.io/en/latest/modules/connections.html + # "socket": "unix_socket", args = {} for key, value in config.items(): # skip undefined values if config.get(key) is None: continue - try: - if "port" in key: - args[cnf_to_mysqldb[key]] = int(value) - elif isinstance(value, bytes): - args[cnf_to_mysqldb[key]] = value.decode("utf-8") - else: - args[cnf_to_mysqldb[key]] = value - except KeyError: - LOG.warning("Skipping unknown parameter %s", key) - # also, always use utf8 + + if "socket" in key: + key = "unix_socket" + + if "port" in key: + args[key] = int(value) + elif "password" in key: + # Pass password to pymysql as bytes to + # prevent encoding issues + args[key] = value.encode() + elif isinstance(value, bytes): + args[key] = value.decode("utf-8") + else: + args[key] = value return client_class(charset="utf8", **args) From 531c59a2de13885e088689fe615eaa86f7e4351b Mon Sep 17 00:00:00 2001 From: stev9171 Date: Thu, 22 Apr 2021 10:51:09 -0500 Subject: [PATCH 4/4] Remove BaseException, update .gitignore --- .gitignore | 1 + plugins/holland.lib.lvm/holland/lib/lvm/snapshot.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index bee4d7e5..e5b39124 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *egg* build/ .ropeproject/ +.vscode/ diff --git a/plugins/holland.lib.lvm/holland/lib/lvm/snapshot.py b/plugins/holland.lib.lvm/holland/lib/lvm/snapshot.py index 69afe740..436b7603 100644 --- a/plugins/holland.lib.lvm/holland/lib/lvm/snapshot.py +++ b/plugins/holland.lib.lvm/holland/lib/lvm/snapshot.py @@ -59,7 +59,7 @@ def mount_snapshot(self, snapshot): options = None try: filesystem = snapshot.filesystem() - except BaseException: + except (LVMCommandError, ValueError): return self.error(snapshot, "Failed looking up filesystem") if filesystem == "xfs":