From 7d8e9d9c655d61436d97c49ebc7589da05799d48 Mon Sep 17 00:00:00 2001 From: Jaime Ashander Date: Fri, 8 May 2015 10:10:12 -0700 Subject: [PATCH] Migrate CLI parsing to click library Complete overhaul of top-level parsing. Delegation to subcommands using click.group(). Structure of parsing within each of the subcommand is retained, but some of the wrapping code for help functions and version options is no longer needed. Changes to CLI API: - webdav/archive/mirror flags no longer specifiable as mutually exclusive group - argument order in comment is changed to smt comment COMMENT [LABEL] The first of these cannot be fixed without changes to click API (see issue #257 https://github.com/mitsuhiko/click/issues/257). The second is a little more annoying from a user perspective. Though it may be fixable given current api. So far, though, I can't figure out how to pass optional arguments before required. I wouldn't be surprised if it isn't possible either. A few associated changes: * Currently using click to print in simplest way possible: click echo, * and for error-like messages specifying err=True to print to standard error. * Because of change, help files look and print slightly (or in some cases majorly) differently Changes to commands.py API: * Breaking changes to the arguments expected by all functions implementing smt subcommands. This is unavoidable with click. Remaining issues : * many test failures due to API changes -- around 67 (filtering calls to parse_arguments from grep -E 'commands\..+(\(|\,)' test_commands.py) lines of the test suite for commands.py will need to be rewritten * test rewrite should be straightforward change from tuples to named args * migration to click doesn't intend to change downstream behavior so passing tests can serve as a check on rewrite * docs not updated to modified look of help pages --- bin/smt | 34 -- setup.py | 8 +- sumatra/commands.py | 880 +++++++++++++++++++++++--------------------- 3 files changed, 474 insertions(+), 448 deletions(-) delete mode 100755 bin/smt diff --git a/bin/smt b/bin/smt deleted file mode 100755 index dea16dd7..00000000 --- a/bin/smt +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python -""" -Command-line interface to the Sumatra computational experiment management tool. -""" - -import sys -from sumatra import commands, __version__ -from sumatra.versioncontrol.base import VersionControlError -from sumatra.recordstore.base import RecordStoreAccessError - -usage = """Usage: smt [options] [args] - -Simulation/analysis management tool version %s - -Available subcommands: - """ % __version__ -usage += "\n ".join(commands.modes) - -if len(sys.argv) < 2: - print(usage) - sys.exit(1) - -cmd = sys.argv[1] -try: - main = getattr(commands, cmd) -except AttributeError: - print(usage) - sys.exit(1) - -try: - main(sys.argv[2:]) -except (VersionControlError, RecordStoreAccessError) as err: - print("Error: %s" % err.message) - sys.exit(1) diff --git a/setup.py b/setup.py index a8b3e899..f29fe535 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,11 @@ def get_tip_revision(self, path=os.getcwd()): 'web/templates/*.html', 'publishing/latex/sumatra.sty', 'formatting/latex_template.tex', 'external_scripts/script_introspect.R']}, - scripts = ['bin/smt', 'bin/smtweb', 'bin/smt-complete.sh'], + scripts = ['bin/smtweb', 'bin/smt-complete.sh'], + entry_points=""" + [console_scripts] + smt = sumatra.commands:cli + """, author = "Sumatra authors and contributors", author_email = "andrew.davison@unic.cnrs-gif.fr", description = "A tool for automated tracking of computation-based scientific projects", @@ -59,7 +63,7 @@ def get_tip_revision(self, path=os.getcwd()): 'Topic :: Scientific/Engineering'], cmdclass = {'sdist': sdist_hg}, install_requires = ['Django>=1.4, <=1.6.11', 'django-tagging', 'httplib2', - 'docutils', 'jinja2', 'parameters'], + 'docutils', 'jinja2', 'parameters', 'Click'], extras_require = {'svn': 'pysvn', 'hg': 'mercurial', 'git': 'GitPython', diff --git a/sumatra/commands.py b/sumatra/commands.py index 0d8c1ef8..59142eaf 100644 --- a/sumatra/commands.py +++ b/sumatra/commands.py @@ -10,10 +10,9 @@ import os.path import sys -from argparse import ArgumentParser -from textwrap import dedent +import click +from functools import update_wrapper import warnings -import re import logging import sumatra @@ -23,7 +22,10 @@ from sumatra.launch import get_launch_mode from sumatra.parameters import build_parameters from sumatra.recordstore import get_record_store -from sumatra.versioncontrol import get_working_copy, get_repository, UncommittedModificationsError +from sumatra.versioncontrol import ( + get_working_copy, get_repository, + UncommittedModificationsError +) from sumatra.formatting import get_diff_formatter from sumatra.records import MissingInformationError from sumatra.core import TIMESTAMP_FORMAT @@ -36,22 +38,21 @@ logger.debug("STARTING") -modes = ("init", "configure", "info", "run", "list", "delete", "comment", "tag", - "repeat", "diff", "help", "export", "upgrade", "sync", "migrate", "version") +store_arg_help = """The argument can take the following forms: + (1) `/path/to/sqlitedb` - DjangoRecordStore is used with the specified Sqlite database, + (2) `http[s]://location` - remote HTTPRecordStore is used with a remote Sumatra server, + (3) `postgres://username:password@hostname/databasename` - DjangoRecordStore is used with specified Postgres database. + """ -store_arg_help = "The argument can take the following forms: (1) `/path/to/sqlitedb` - DjangoRecordStore is used with the specified Sqlite database, (2) `http[s]://location` - remote HTTPRecordStore is used with a remote Sumatra server, (3) `postgres://username:password@hostname/databasename` - DjangoRecordStore is used with specified Postgres database." -## recommended method for modifying warning formatting -## see https://docs.python.org/2/library/warnings.html#warnings.showwarning -def _warning( - message, - category = UserWarning, - filename = '', - lineno = -1): - print("Warning: ") - print(message) +# recommended method for modifying warning formatting +# see https://docs.python.org/2/library/warnings.html#warnings.showwarning +def _warning(message, category=UserWarning, filename='', lineno=-1): + click.echo("Warning: ") + click.echo(message) warnings.showwarning = _warning + def parse_executable_str(exec_str): """ Split the string describing the executable into a path part and an @@ -62,6 +63,7 @@ def parse_executable_str(exec_str): first_space = len(exec_str) return exec_str[:first_space], exec_str[first_space:] + def parse_arguments(args, input_datastore, stdin=None, stdout=None, allow_command_line_parameters=True): cmdline_parameters = [] @@ -105,104 +107,166 @@ def parse_arguments(args, input_datastore, stdin=None, stdout=None, message, name, value = v.args warnings.warn(message) warnings.warn("'{0}={1}' not defined in the parameter file".format(name, value)) - ps.update({name: value}) ## for now, add the command line param anyway + ps.update({name: value}) # for now, add the command line param anyway else: raise Exception("Command-line parameters supplied but without a parameter file to put them into.") # ought really to have a more specific Exception and to catch it so as to give a helpful error message to user return parameter_sets, input_data, " ".join(script_args) -def init(argv): - """Create a new project in the current directory.""" - usage = "%(prog)s init [options] NAME" - description = "Create a new project called NAME in the current directory." - parser = ArgumentParser(usage=usage, - description=description) - parser.add_argument('project_name', metavar='NAME', help="a short name for the project; should not contain spaces.") - parser.add_argument('-d', '--datapath', metavar='PATH', default='./Data', help="set the path to the directory in which smt will search for output datafiles generated by the simulation/analysis. Defaults to %(default)s.") - parser.add_argument('-i', '--input', metavar='PATH', default='/', help="set the path to the directory relative to which input datafile paths will be given. Defaults to the filesystem root.") - parser.add_argument('-l', '--addlabel', choices=['cmdline', 'parameters', None], metavar='OPTION', - default=None, help="If this option is set, smt will append the record label either to the command line (option 'cmdline') or to the parameter file (option 'parameters'), and will add the label to the datapath when searching for datafiles. It is up to the user to make use of this label inside their program to ensure files are created in the appropriate location.") - parser.add_argument('-e', '--executable', metavar='PATH', help="set the path to the executable. If this is not set, smt will try to infer the executable from the value of the --main option, if supplied, and will try to find the executable from the PATH environment variable, then by searching various likely locations on the filesystem.") - parser.add_argument('-r', '--repository', help="the URL of a Subversion or Mercurial repository containing the code. This will be checked out/cloned into the current directory.") - parser.add_argument('-m', '--main', help="the name of the script that would be supplied on the command line if running the simulation or analysis normally, e.g. init.hoc.") - parser.add_argument('-c', '--on-changed', default='error', help="the action to take if the code in the repository or any of the depdendencies has changed. Defaults to %(default)s") # need to add list of allowed values - parser.add_argument('-s', '--store', help="Specify the path, URL or URI to the record store (must be specified). This can either be an existing record store or one to be created. {0} Not using the `--store` argument defaults to a DjangoRecordStore with Sqlite in `.smt/records`".format(store_arg_help)) - parser.add_argument('-g', '--labelgenerator', choices=['timestamp', 'uuid'], default='timestamp', metavar='OPTION', help="specify which method Sumatra should use to generate labels (options: timestamp, uuid)") - parser.add_argument('-t', '--timestamp_format', help="the timestamp format given to strftime", default=TIMESTAMP_FORMAT) - parser.add_argument('-L', '--launch_mode', choices=['serial', 'distributed', 'slurm-mpi'], default='serial', help="how computations should be launched. Defaults to %(default)s") - parser.add_argument('-o', '--launch_mode_options', help="extra options for the given launch mode") - - datastore = parser.add_mutually_exclusive_group() - datastore.add_argument('-W', '--webdav', metavar='URL', help="specify a webdav URL (with username@password: if needed) as the archiving location for data") - datastore.add_argument('-A', '--archive', metavar='PATH', help="specify a directory in which to archive output datafiles. If not specified, or if 'false', datafiles are not archived.") - datastore.add_argument('-M', '--mirror', metavar='URL', help="specify a URL at which your datafiles will be mirrored.") - - args = parser.parse_args(argv) - +@click.group() +@click.version_option(version=sumatra.__version__, message='%(version)s') +def cli(): + """Simulation/analysis management tool.""" + + +def config_group(fn): + """Decorator for shared arguments between init and config (the "config group" of commands)""" + # see http://click.pocoo.org/4/commands/#decorating-commands + @click.option('-d', '--datapath', metavar='PATH', default='./Data', + show_default=True, + help="set the path to the directory in which smt will search " + "for output datafiles generated by the simulation/analysis.") + @click.option('-i', '--input', metavar='PATH', default='/', show_default=True, + help="set the path to the directory relative to which input " + "datafile paths will be given.") + @click.option('-l', '--addlabel', type=click.Choice(['cmdline', 'parameters']), + help="If this option is set, smt will append the record label " + "either to the command line (option 'cmdline') or to the " + "parameter file (option 'parameters'), and will add the label " + "to the datapath when searching for datafiles. It is up to the " + "user to make use of this label inside their program to ensure " + "files are created in the appropriate location.") + @click.option('-e', '--executable', metavar='PATH', + help="set the path to the executable. If this is not set, smt " + "will try to infer the executable from the value of the --main " + "option, if supplied, and will try to find the executable from " + "the PATH environment variable, then by searching various " + "likely locations on the filesystem.") + @click.option('-r', '--repository', metavar='URL', + help="the URL of a Subversion or Mercurial repository " + "containing the code. This will be checked out/cloned into " + "the current directory.") + @click.option('-m', '--main', metavar='SCRIPTNAME', + help="the name of the script that would be supplied on the " + "command line if running the simulation or analysis " + "normally, e.g. init.hoc.") + @click.option('-c', '--on-changed', default='error', show_default=True, + help="the action to take if the code in the repository or " + "any of the depdendencies has changed.") # need to add list of allowed values + @click.option('-g', '--labelgenerator', + type=click.Choice(['timestamp', 'uuid']), default='timestamp', + help="specify which method Sumatra should use to generate " + "labels (options: timestamp, uuid)") + @click.option('-t', '--timestamp_format', + help="the timestamp format given to strftime", + default=TIMESTAMP_FORMAT) + @click.option('-L', '--launch_mode', + type=click.Choice(['serial', 'distributed', 'slurm-mpi']), + default='serial', show_default=True, + help="how computations should be launched.") + @click.option('-o', '--launch_mode_options', + help="extra options for the given launch mode") + # TODO -- no facility for mutually exclusive group in click + @click.option('-W', '--webdav', metavar='URL', + help="specify a webdav URL (with username@password: if " + "needed) as the archiving location for data") + @click.option('-A', '--archive', metavar='PATH', + help="specify a directory in which to archive output datafiles. " + "If not specified, or if 'false', datafiles are not archived.") + @click.option('-M', '--mirror', metavar='URL', + help="specify a URL at which your datafiles will be mirrored.") + @click.pass_context + def new_func(ctx, *args, **kwargs): + return ctx.invoke(fn, *args, **kwargs) # pass with ctx if extend functionality + return update_wrapper(new_func, fn) + + +@cli.command() +@click.argument('project_name', metavar='NAME') +@click.option('-s', '--store', + help="Specify the path, URL or URI to the record store (must be " + "specified). This can either be an existing record store or one " + "to be created. {0} Not using the `--store` argument defaults to " + "a DjangoRecordStore with Sqlite in `.smt/records`".format(store_arg_help)) +@config_group +def init(project_name, store, datapath, input, addlabel, executable, repository, + main, on_changed, labelgenerator, timestamp_format, launch_mode, + launch_mode_options, webdav, archive, mirror): + """Create a new project current directory. + + Create a new project called NAME in current directory. + """ try: project = load_project() - parser.error("A project already exists in directory '{0}'.".format(project.path)) + click.echo("A project already exists in directory '{0}'.".format(project.path), err=True) except Exception: pass if not os.path.exists(".smt"): os.mkdir(".smt") - if args.repository: - repository = get_repository(args.repository) + if repository: + repository = get_repository(repository) repository.checkout() else: repository = get_working_copy().repository # if no repository is specified, we assume there is a working copy in the current directory. - if args.executable: - executable_path, executable_options = parse_executable_str(args.executable) + if executable: + executable_path, executable_options = parse_executable_str(executable) executable = get_executable(path=executable_path) executable.args = executable_options - elif args.main: + elif main: try: - executable = get_executable(script_file=args.main) + executable = get_executable(script_file=main) except Exception: # assume unrecognized extension - really need more specific exception type # should warn that extension unrecognized executable = None else: executable = None - if args.store: - record_store = get_record_store(args.store) + if store: + record_store = get_record_store(store) else: record_store = 'default' - - if args.webdav: - # should we care about archive migration?? - output_datastore = get_data_store("DavFsDataStore", {"root": args.datapath, "dav_url": args.webdav}) - args.archive = '.smt/archive' - elif args.archive and args.archive.lower() != 'false': - if args.archive.lower() == "true": - args.archive = ".smt/archive" - args.archive = os.path.abspath(args.archive) - output_datastore = get_data_store("ArchivingFileSystemDataStore", {"root": args.datapath, "archive": args.archive}) - elif args.mirror: - output_datastore = get_data_store("MirroredFileSystemDataStore", {"root": args.datapath, "mirror_base_url": args.mirror}) + if (webdav and archive): + click.echo("Warn: using webdav though archive is specified") + if (webdav and mirror): + click.echo("Warn: using webdav though mirror is specified") + if (archive and mirror): + click.echo("Warn: using archive though mirror is specified") + if webdav: + output_datastore = get_data_store("DavFsDataStore", {"root": datapath, "dav_url": webdav}) + archive = '.smt/archive' + elif archive and archive.lower() != 'false': + if archive.lower() == "true": + archive = ".smt/archive" + archive = os.path.abspath(archive) + output_datastore = get_data_store("ArchivingFileSystemDataStore", + {"root": datapath, "archive": archive}) + elif mirror: + output_datastore = get_data_store("MirroredFileSystemDataStore", + {"root": datapath, "mirror_base_url": mirror}) else: - output_datastore = get_data_store("FileSystemDataStore", {"root": args.datapath}) - input_datastore = get_data_store("FileSystemDataStore", {"root": args.input}) + output_datastore = get_data_store("FileSystemDataStore", + {"root": datapath}) + input_datastore = get_data_store("FileSystemDataStore", {"root": input}) - if args.launch_mode_options: - args.launch_mode_options = args.launch_mode_options.strip() - launch_mode = get_launch_mode(args.launch_mode)(options=args.launch_mode_options) + if launch_mode_options: + launch_mode_options = launch_mode_options.strip() + launch_mode = get_launch_mode(launch_mode)(options=launch_mode_options) - project = Project(name=args.project_name, + project = Project(name=project_name, default_executable=executable, default_repository=repository, - default_main_file=args.main, # what if incompatible with executable? + default_main_file=main, # what if incompatible with executable? default_launch_mode=launch_mode, data_store=output_datastore, record_store=record_store, - on_changed=args.on_changed, - data_label=args.addlabel, + on_changed=on_changed, + data_label=addlabel, input_datastore=input_datastore, - label_generator=args.labelgenerator, - timestamp_format=args.timestamp_format) + label_generator=labelgenerator, + timestamp_format=timestamp_format) if os.path.exists('.smt'): f = open('.smt/labels', 'w') f.writelines(project.format_records(tags=None, mode='short', format='text', reverse=False)) @@ -210,316 +274,315 @@ def init(argv): project.save() -def configure(argv): +@cli.command() +@click.option('--plain/--no-plain', default=False, + help="pass arguments to the 'run' command straight through to " + "the program. Otherwise arguments of the form name=value can be " + "used to overwrite default parameter values. If no-plain, " + "arguments to the 'run' command of the form name=value will " + "overwrite default parameter values. This is the opposite of " + "the --plain option.") +@click.option('-s', '--store', + help="Change the record store to the specified path, URL or URI " + "(must be specified). {0}".format(store_arg_help)) +@config_group # decorator only works if up here, changing desired argument ordering +def configure(plain, store, datapath, input, addlabel, executable, repository, + main, on_changed, labelgenerator, timestamp_format, launch_mode, + launch_mode_options, webdav, archive, mirror): """Modify the settings for the current project.""" - usage = "%(prog)s configure [options]" - description = "Modify the settings for the current project." - parser = ArgumentParser(usage=usage, - description=description) - parser.add_argument('-d', '--datapath', metavar='PATH', help="set the path to the directory in which smt will search for datafiles generated by the simulation or analysis.") - parser.add_argument('-i', '--input', metavar='PATH', default=None, help="set the path to the directory in which smt will search for input datafiles.") - parser.add_argument('-l', '--addlabel', choices=['cmdline', 'parameters', None], metavar='OPTION', - default=None, help="If this option is set, smt will append the record label either to the command line (option 'cmdline') or to the parameter file (option 'parameters'), and will add the label to the datapath when searching for datafiles. It is up to the user to make use of this label inside their program to ensure files are created in the appropriate location.") - parser.add_argument('-e', '--executable', metavar='PATH', help="set the path to the executable.") - parser.add_argument('-r', '--repository', help="the URL of a Subversion or Mercurial repository containing the code. This will be checked out/cloned into the current directory.") - parser.add_argument('-m', '--main', help="the name of the script that would be supplied on the command line if running the simulator normally, e.g. init.hoc.") - parser.add_argument('-c', '--on-changed', help="may be 'store-diff' or 'error': the action to take if the code in the repository or any of the dependencies has changed.", choices=['store-diff', 'error']) - parser.add_argument('-g', '--labelgenerator', choices=['timestamp', 'uuid'], metavar='OPTION', help="specify which method Sumatra should use to generate labels (options: timestamp, uuid)") - parser.add_argument('-t', '--timestamp_format', help="the timestamp format given to strftime") - parser.add_argument('-L', '--launch_mode', choices=['serial', 'distributed', 'slurm-mpi'], help="how computations should be launched.") - parser.add_argument('-o', '--launch_mode_options', help="extra options for the given launch mode, to be given in quotes with a leading space, e.g. ' --foo=3'") - parser.add_argument('-p', '--plain', dest='plain', action='store_true', help="pass arguments to the 'run' command straight through to the program. Otherwise arguments of the form name=value can be used to overwrite default parameter values.") - parser.add_argument('--no-plain', dest='plain', action='store_false', help="arguments to the 'run' command of the form name=value will overwrite default parameter values. This is the opposite of the --plain option.") - parser.add_argument('-s', '--store', help="Change the record store to the specified path, URL or URI (must be specified). {0}".format(store_arg_help)) - - datastore = parser.add_mutually_exclusive_group() - datastore.add_argument('-W', '--webdav', metavar='URL', help="specify a webdav URL (with username@password: if needed) as the archiving location for data") - datastore.add_argument('-A', '--archive', metavar='PATH', help="specify a directory in which to archive output datafiles. If not specified, or if 'false', datafiles are not archived.") - datastore.add_argument('-M', '--mirror', metavar='URL', help="specify a URL at which your datafiles will be mirrored.") - - args = parser.parse_args(argv) - project = load_project() - if args.store: - new_store = get_record_store(args.store) + if store: + new_store = get_record_store(store) project.change_record_store(new_store) - if args.archive: - if args.archive.lower() == "true": - args.archive = ".smt/archive" + if archive: + if archive.lower() == "true": + archive = ".smt/archive" if hasattr(project.data_store, 'archive_store'): # current data store is archiving - if args.archive.lower() == 'false': + if archive.lower() == 'false': project.data_store = get_data_store("FileSystemDataStore", {"root": project.data_store.root}) else: - project.data_store.archive_store = args.archive + project.data_store.archive_store = archive else: # current data store is not archiving - if args.archive.lower() != 'false': - project.data_store = get_data_store("ArchivingFileSystemDataStore", {"root": args.datapath, "archive": args.archive}) - if args.webdav: + if archive.lower() != 'false': + project.data_store = get_data_store("ArchivingFileSystemDataStore", {"root": datapath, "archive": archive}) + if webdav: # should we care about archive migration?? - project.data_store = get_data_store("DavFsDataStore", {"root": args.datapath, "dav_url": args.webdav}) + project.data_store = get_data_store("DavFsDataStore", {"root": datapath, "dav_url": webdav}) project.data_store.archive_store = '.smt/archive' - if args.datapath: - project.data_store.root = args.datapath - if args.input: - project.input_datastore.root = args.input - if args.repository: - repository = get_repository(args.repository) + if datapath: + project.data_store.root = datapath + if input: + project.input_datastore.root = input + if repository: + repository = get_repository(repository) repository.checkout() project.default_repository = repository - if args.main: - project.default_main_file = args.main - if args.executable: - executable_path, executable_options = parse_executable_str(args.executable) + if main: + try: + executable = get_executable(script_file=main) + except Exception: # assume unrecognized extension - really need more specific exception type + # should warn that extension unrecognized + executable = None + elif executable: + executable_path, executable_options = parse_executable_str(executable) project.default_executable = get_executable(executable_path, - script_file=args.main or project.default_main_file) + script_file=main or project.default_main_file) project.default_executable.options = executable_options - if args.on_changed: - project.on_changed = args.on_changed - if args.addlabel: - project.data_label = args.addlabel - if args.labelgenerator: - project.label_generator = args.labelgenerator - if args.timestamp_format: - project.timestamp_format = args.timestamp_format - if args.launch_mode: - project.default_launch_mode = get_launch_mode(args.launch_mode)() - if args.launch_mode_options: - project.default_launch_mode.options = args.launch_mode_options.strip() - if args.plain is not None: - project.allow_command_line_parameters = not args.plain + if on_changed: + project.on_changed = on_changed + if addlabel: + project.data_label = addlabel + if labelgenerator: + project.label_generator = labelgenerator + if timestamp_format: + project.timestamp_format = timestamp_format + if launch_mode: + project.default_launch_mode = get_launch_mode(launch_mode)() + if launch_mode_options: + project.default_launch_mode.options = launch_mode_options.strip() + if plain is not None: + project.allow_command_line_parameters = not plain project.save() -def info(argv): +@cli.command() +def info(): """Print information about the current project.""" - usage = "%(prog)s info" - description = "Print information about the current project." - parser = ArgumentParser(usage=usage, - description=description) - args = parser.parse_args(argv) try: - project = load_project() + project = load_project() # should config pass cwd? except IOError as err: - print(err) + click.echo(err, err=True) sys.exit(1) - print(project.info()) - - -def run(argv): - """Run a simulation or analysis.""" - usage = "%(prog)s run [options] [arg1, ...] [param=value, ...]" - description = dedent("""\ - The list of arguments will be passed on to the simulation/analysis script. - It should normally contain at least the name of a parameter file, but - can also contain input files, flags, etc. - - If the parameter file should be in a format that Sumatra understands (see - documentation), then the parameters will be stored to allow future - searching, comparison, etc. of records. - - For convenience, it is possible to specify a file with default parameters - and then specify those parameters that are different from the default values - on the command line with any number of param=value pairs (note no space - around the equals sign).""") - parser = ArgumentParser(usage=usage, - description=description) - parser.add_argument('-v', '--version', metavar='REV', - help="use version REV of the code (if this is not the same as the working copy, it will be checked out of the repository). If this option is not specified, the most recent version in the repository will be used. If there are changes in the working copy, the user will be prompted to commit them first") - parser.add_argument('-l', '--label', help="specify a label for the experiment. If no label is specified, one will be generated automatically.") - parser.add_argument('-r', '--reason', help="explain the reason for running this simulation/analysis.") - parser.add_argument('-e', '--executable', metavar='PATH', help="Use this executable for this run. If not specified, the project's default executable will be used.") - parser.add_argument('-m', '--main', help="the name of the script that would be supplied on the command line if running the simulation/analysis normally, e.g. init.hoc. If not specified, the project's default will be used.") - parser.add_argument('-n', '--num_processes', metavar='N', type=int, - help="run a distributed computation on N processes using MPI. If this option is not used, or if N=0, a normal, serial simulation/analysis is run.") - parser.add_argument('-t', '--tag', help="tag you want to add to the project") - parser.add_argument('-D', '--debug', action='store_true', help="print debugging information.") - parser.add_argument('-i', '--stdin', help="specify the name of a file that should be connected to standard input.") - parser.add_argument('-o', '--stdout', help="specify the name of a file that should be connected to standard output.") - - args, user_args = parser.parse_known_args(argv) - - if args.debug: + click.echo(project.info()) + + +@cli.command(context_settings=dict(ignore_unknown_options=True, )) +@click.option('-v', '--version', metavar='REV', + help="use version REV of the code (if this is not the same as the " + "working copy, it will be checked out of the repository). If this " + "option is not specified, the most recent version in the " + "repository will be used. If there are changes in the working " + "copy, the user will be prompted to commit them first") +@click.option('-l', '--label', metavar='LABEL', + help="specify a label for the experiment. If no label is " + "specified, one will be generated automatically.") +@click.option('-r', '--reason', + help="explain the reason for running this simulation/analysis.") +@click.option('-e', '--executable', metavar='PATH', + help="Use this executable for this run. If not specified, the " + "project's default executable will be used.") +@click.option('-m', '--main', metavar='SCRIPTNAME', + help="the name of the script that would be supplied on the " + "command line if running the simulation/analysis normally, e.g. " + "init.hoc. If not specified, the project's default will be used.") +@click.option('-n', '--num_processes', metavar='N', type=int, + help="run a distributed computation on N processes using MPI. " + "If this option is not used, or if N=0, a normal, serial " + "simulation/analysis is run.") +@click.option('-t', '--tag', help="tag you want to add to the project") +@click.option('-D', '--debug', is_flag=True, + help="print debugging information.") +@click.option('-i', '--stdin', + help="specify the name of a file that should be connected to standard input.") +@click.option('-o', '--stdout', + help="specify the name of a file that should be connected to standard output.") +@click.argument('user_args', metavar='[arg1, ...] [param=value, ...]', + nargs=-1, type=click.UNPROCESSED) +def run(version, label, reason, executable, main, num_processes, tag, debug, stdin, stdout, user_args): + """Run a simulation or analysis. + + The list of arguments will be passed on to the simulation/analysis script. + It should normally contain at least the name of a parameter file, but + can also contain input files, flags, etc. + + If the parameter file should be in a format that Sumatra understands (see + documentation), then the parameters will be stored to allow future + searching, comparison, etc. of records. + + For convenience, it is possible to specify a file with default parameters + and then specify those parameters that are different from the default values + on the command line with any number of param=value pairs (note no space + around the equals sign).""" + if debug: logger.setLevel(logging.DEBUG) project = load_project() parameters, input_data, script_args = parse_arguments(user_args, project.input_datastore, - args.stdin, - args.stdout, + stdin, + stdout, project.allow_command_line_parameters) if len(parameters) == 0: parameters = {} elif len(parameters) == 1: parameters = parameters[0] else: - parser.error("Only a single parameter file allowed.") # for now + click.echo("Only a single parameter file allowed.", err=True) # for now - if args.executable: - executable_path, executable_options = parse_executable_str(args.executable) + if executable: + executable_path, executable_options = parse_executable_str(executable) executable = get_executable(path=executable_path) executable.options = executable_options - elif args.main: - executable = get_executable(script_file=args.main) # should we take the options from project.default_executable, if they match? + elif main: + executable = get_executable(script_file=main) # should we take the options from project.default_executable, if they match? else: executable = 'default' - if args.num_processes: + if num_processes: if hasattr(project.default_launch_mode, 'n'): - project.default_launch_mode.n = args.num_processes + project.default_launch_mode.n = num_processes else: - parser.error("Your current launch mode does not support using multiple processes.") - reason = args.reason or '' + click.echo("Your current launch mode does not support using multiple processes.", err=True) + reason = reason or '' if reason: reason = reason.strip('\'"') - label = args.label + label = label try: run_label = project.launch(parameters, input_data, script_args, label=label, reason=reason, executable=executable, - main_file=args.main or 'default', - version=args.version or 'current') + main_file=main or 'default', + version=version or 'current') except (UncommittedModificationsError, MissingInformationError) as err: - print(err) + click.echo(err, err=True) sys.exit(1) - if args.tag: - project.add_tag(run_label, args.tag) - - -def list(argv): # add 'report' and 'log' as aliases - """List records belonging to the current project.""" - usage = "%(prog)s list [options] [TAGS]" - description = dedent("""\ - If TAGS (optional) is specified, then only records with a tag in TAGS - will be listed.""") - parser = ArgumentParser(usage=usage, - description=description) - parser.add_argument('tags', metavar='TAGS', nargs='*') - parser.add_argument('-l', '--long', action="store_const", const="long", - dest="mode", default="short", - help="prints full information for each record"), - parser.add_argument('-T', '--table', action="store_const", const="table", - dest="mode", help="prints information in tab-separated columns") - parser.add_argument('-f', '--format', metavar='FMT', choices=['text', 'html', 'latex', 'shell'], default='text', - help="FMT can be 'text' (default), 'html', 'latex' or 'shell'.") - parser.add_argument('-r', '--reverse', action="store_true", dest="reverse", default=False, - help="list records in reverse order (default: newest first)"), - args = parser.parse_args(argv) + if tag: + project.add_tag(run_label, tag) + + +@cli.command() +@click.argument('tags', nargs=-1, required=False) +@click.option('-l', '--long', 'mode', flag_value='long', + help="prints full information for each record") +@click.option('-T', '--table', 'mode', flag_value='table', + help="prints information in tab-separated columns") +@click.option('-f', '--format', metavar='FMT', show_default = True, + type=click.Choice(['text','html', 'latex', 'shell']), + default='text') +@click.option('-r', '--reverse', is_flag=True, + help="list records in reverse order (default: newest first)") +def list(tags, mode, format, reverse): # add 'report' and 'log' as aliases + """List records in current project. + + If TAGS (optional) is specified, then only records with a tag in TAGS + will be listed.""" + if mode is None: + mode = 'short' project = load_project() + project_list = project.format_records(tags=tags, mode=mode, format=format, reverse=reverse) + # below for bash completion if os.path.exists('.smt'): - f = open('.smt/labels', 'w') - f.writelines(project.format_records(tags=None, mode='short', format='text', reverse=False)) - f.close() - print(project.format_records(tags=args.tags, mode=args.mode, format=args.format, reverse=args.reverse)) - -def delete(argv): - """Delete records or records with a particular tag from a project.""" - usage = "%(prog)s delete [options] LIST" - description = dedent("""\ - LIST should be a space-separated list of labels for individual records or - of tags. If it contains tags, you must set the --tag/-t option (see below). - The special value "last" allows you to delete the most recent simulation/analysis. - If you want to delete all records, just delete the .smt directory and use - smt init to create a new, empty project.""") - parser = ArgumentParser(usage=usage, - description=description) - parser.add_argument('labels', metavar='LIST', nargs="+", help="a space-separated list of labels for individual records or of tags") - parser.add_argument('-t', '--tag', action='store_true', - help="interpret LIST as containing tags. Records with any of these tags will be deleted.") - parser.add_argument('-d', '--data', action='store_true', - help="also delete any data associated with the record(s).") - args = parser.parse_args(argv) - + f = open('.smt/labels', 'w') + f.writelines(project.format_records(tags=None, mode='short', format='text', reverse=False)) + f.close() + click.echo(project_list) # TODO -- for table output weird wrapping occurrs + + +@cli.command() +@click.argument('labels', metavar='LIST', nargs=-1, required=True) +@click.option('-t', '--tag', is_flag=True, + help="interpret LIST as containing tags. Records with any of " + "these tags will be deleted.") +@click.option('-d', '--data', is_flag=True, + help="also delete any data associated with the record(s).") +def delete(labels, tag, data): + """Delete records from a project. + + LIST should be a space-separated list of labels for individual records or + of tags. If it contains tags, you must set the --tag/-t option (see below). + The special value "last" allows you to delete the most recent simulation/analysis. + If you want to delete all records, just delete the .smt directory and use + smt init to create a new, empty project. + """ project = load_project() - if args.tag: - for tag in args.labels: - n = project.delete_by_tag(tag, delete_data=args.data) - print("%s records deleted." % n) + if tag: + for tag in labels: + n = project.delete_by_tag(tag, delete_data=data) + click.echo("%s records deleted." % n) else: - for label in args.labels: + for label in labels: if label == 'last': label = project.most_recent().label try: - project.delete_record(label, delete_data=args.data) - except Exception: # could be KeyError or DoesNotExist: should create standard NoSuchRecord or RecordDoesNotExist exception + project.delete_record(label, delete_data=data) + except Exception: + # could be KeyError or DoesNotExist + #: should create standard NoSuchRecord or RecordDoesNotExist exception warnings.warn("Could not delete record '%s' because it does not exist" % label) -def comment(argv): - """Add a comment to an existing record.""" - usage = "%(prog)s comment [options] [LABEL] COMMENT" - description = dedent("""\ - This command is used to describe the outcome of the simulation/analysis. - If LABEL is omitted, the comment will be added to the most recent experiment. - If the '-f/--file' option is set, COMMENT should be the name of a file - containing the comment, otherwise it should be a string of text. - By default, comments will be appended to any existing comments. - To overwrite existing comments, use the '-r/--replace flag.""") - parser = ArgumentParser(usage=usage, - description=description) - parser.add_argument('label', nargs='?', metavar='LABEL', help="the record to which the comment will be added") - parser.add_argument('comment', help="a string of text, or the name of a file containing the comment.") - parser.add_argument('-r', '--replace', action='store_true', - help="if this flag is set, any existing comment will be overwritten, otherwise, the new comment will be appended to the end, starting on a new line") - parser.add_argument('-f', '--file', action='store_true', - help="interpret COMMENT as the path to a file containing the comment") - args = parser.parse_args(argv) - - if args.file: - f = open(args.comment, 'r') +@cli.command() +@click.argument('comment', required=True) # TODO -- had to switch order figure out how to swithc back +@click.argument('label', required=False) +@click.option('-r', '--replace', is_flag=True, + help="""if this flag is set, any existing comment will be " + "overwritten, otherwise, the new comment will be appended to the " + "end, starting on a new line""") +@click.option('-f', '--file', is_flag=True, + help="interpret COMMENT as the path to a file containing the comment") +def comment(label, comment, replace, file): + """Add a comment to an existing record. + + Add COMMENT, a description of the outcome of the simulation or + analysis, to record LABEL. If LABEL is omitted, the comment will + be added to the most recent experiment. If the '-f/--file' option + is set, COMMENT should be the name of a file containing the + comment, otherwise it should be a string of text. By default, + comments will be appended to any existing comments. To overwrite + existing comments, use the '-r/--replace flag. + """ + + if file: + f = open(comment, 'r') comment = f.read() f.close() else: - comment = args.comment - + comment = comment project = load_project() - label = args.label or project.most_recent().label - project.add_comment(label, comment, replace=args.replace) - - -def tag(argv): - """Tag, or remove a tag, from a record or records.""" - usage = "%(prog)s tag [options] TAG [LIST]" - description = dedent("""\ - If TAG contains spaces, it must be enclosed in quotes. LIST should be a - space-separated list of labels for individual records. If it is omitted, - only the most recent record will be tagged. If the '-d/--delete' option - is set, the tag will be removed from the records.""") - parser = ArgumentParser(usage=usage, - description=description) - parser.add_argument('tag', metavar='TAG', help="tag to add") - parser.add_argument('labels', metavar='LIST', nargs='*', help="a space-separated list of records to be tagged") - parser.add_argument('-r', '--remove', action='store_true', - help="remove the tag from the record(s), rather than adding it.") - args = parser.parse_args(argv) + label = label or project.most_recent().label + project.add_comment(label, comment, replace=replace) + + +@cli.command() +@click.argument('tag', metavar='TAG', nargs=1, required=True) +@click.argument('labels', nargs=-1, metavar="[LIST]", required=False) +@click.option('-r', '--remove', is_flag=True, + help="remove the tag from the record(s), rather than adding it.") +def tag(tag, labels, remove): + """Tag or un-tag records. + + TAG the space-separate LIST of labels for individual records. If + TAG contains spaces, it must be enclosed in quotes. If LIST is + omitted, only the most recent record will be tagged. If the + '-d/--delete' option is set, the tag will be removed from the + records. + """ project = load_project() - if args.remove: + if remove: op = project.remove_tag else: op = project.add_tag - labels = args.labels or [project.most_recent().label] + labels = labels or [project.most_recent().label] for label in labels: - op(label, args.tag) - - -def repeat(argv): - """Re-run a previous simulation or analysis.""" - usage = "%(prog)s repeat LABEL" - description = dedent("""\ - Re-run a previous simulation/analysis under (in theory) identical - conditions, and check that the results are unchanged.""") - parser = ArgumentParser(usage=usage, - description=description) - parser.add_argument('original_label', metavar='LABEL', help='label of record to be repeated') - parser.add_argument('-l', '--label', metavar='NEW_LABEL', help="specify a label for the new experiment. If no label is specified, one will be generated automatically.") - - args = parser.parse_args(argv) - original_label = args.original_label + op(label, tag) + + +@cli.command() +@click.argument('original_label', metavar='LABEL') +@click.option('-l', '--label', metavar='NEW_LABEL', + help="specify a label for the new experiment. If no label is " + "specified, one will be generated automatically.") +def repeat(original_label, label): + """Re-run a previous simulation or analysis. + + Re-run the simulation LABEL under (in theory) identical + conditions, and check that the results are unchanged.""" project = load_project() - new_label, original_label = project.repeat(original_label, args.label) + new_label, original_label = project.repeat(original_label, label) diff = project.compare(original_label, new_label) if diff: formatter = get_diff_formatter()(diff) @@ -529,66 +592,61 @@ def repeat(argv): msg = "\n".join(msg) else: msg = "The new record exactly matches the original." - print(msg) + click.echo(msg) project.add_comment(new_label, msg) -def diff(argv): - """Show the differences, if any, between two records.""" - usage = "%(prog)s diff [options] LABEL1 LABEL2" - description = dedent("Show the differences, if any, between two records.") - parser = ArgumentParser(usage=usage, - description=description) - parser.add_argument('label1') - parser.add_argument('label2') - parser.add_argument('-i', '--ignore', action="append", - help="a regular expression pattern for filenames to ignore when evaluating differences in output data. To supply multiple patterns, use the -i option multiple times.") - parser.add_argument('-l', '--long', action="store_const", const="long", - dest="mode", default="short", - help="prints full information for each record"), - args = parser.parse_args(argv) - if args.ignore is None: - args.ignore = [] +@cli.command() +@click.argument('label1', required=True) +@click.argument('label2', required=True) +@click.option('-i', '--ignore', multiple=True, + help="a regular expression pattern for filenames to ignore when " + "evaluating differences in output data. To supply multiple " + "patterns, use the -i option multiple times.") +@click.option('-l', '--long', 'mode', flag_value='long', + help="prints full information for each record") +def diff(label1, label2, ignore, mode): + """Show the differences between two records. + + Records identified by LABEL1 and LABEL2 are differenced and the + output printed. + """ + if mode is None: + mode = 'short' project = load_project() - print(project.show_diff(args.label1, args.label2, mode=args.mode, - ignore_filenames=args.ignore)) - - -def help(argv): - usage = "%(prog)s help CMD" - description = dedent("""Get help on an %(prog)s command.""") - parser = ArgumentParser(usage=usage, - description=description) - parser.add_argument('cmd', nargs='?') - args = parser.parse_args(argv) - if args.cmd is None: - parser.error('Please specify a command on which you would like help.\n\nAvailable commands:\n ' + "\n ".join(modes)) + click.echo(project.show_diff(label1, label2, mode=mode, + ignore_filenames=ignore)) + + +@cli.command() +@click.argument('cmd', nargs=1, required=False) +def help(cmd): + """Get help on an smt command.""" + + if cmd is None: + cli(['--help']) else: try: - func = globals()[args.cmd] - func(['--help']) + cli([cmd, '--help']) except KeyError: - parser.error('"%s" is not an smt command.' % args.cmd) + click.echo('"%s" is not an smt command.' % cmd, err=True) -def upgrade(argv): - usage = "%(prog)s upgrade" - description = dedent("""\ - Upgrade an existing Sumatra project. You must have previously run - "smt export" or the standalone 'export.py' script.""") - parser = ArgumentParser(usage=usage, - description=description) - args = parser.parse_args(argv) +@cli.command() +def upgrade(): + """Upgrade an existing Sumatra project. + You must have previously run "smt export" or the standalone 'export.py' script. + """ project = load_project() if hasattr(project, 'sumatra_version') and project.sumatra_version == sumatra.__version__: - print("No upgrade needed (project was created with an up-to-date version of Sumatra).") + click.echo("No upgrade needed (project was created with an up-to-date version of Sumatra).") sys.exit(1) if not os.path.exists(".smt/project_export.json"): - print("Error: project must have been exported (with the original " - "version of Sumatra) before upgrading.") + click.echo("Error: project must have been exported (with the original " + "version of Sumatra) before upgrading.", err=True) sys.exit(1) # backup and remove .smt @@ -608,40 +666,37 @@ def upgrade(argv): project.record_store.import_(project.name, f.read()) f.close() else: - print("Record file not found") + click.echo("Record file not found") sys.exit(1) - print("Project successfully upgraded to Sumatra version {}.".format(project.sumatra_version)) + click.echo("Project successfully upgraded to Sumatra " + "version {}.".format(project.sumatra_version)) + +@cli.command() +def export(): + """Export Sumatra project records to JSON. -def export(argv): - usage = "%(prog)s export" - description = dedent("""\ - Export a Sumatra project and its records to JSON. This is needed before running upgrade.""") - parser = ArgumentParser(usage=usage, - description=description) - args = parser.parse_args(argv) + This is needed before running upgrade.""" project = load_project() project.export() -def sync(argv): - usage = "%(prog)s sync PATH1 [PATH2]" - description = dedent("""\ - Synchronize two record stores. If both PATH1 and PATH2 are given, the - record stores at those locations will be synchronized. If only PATH1 is - given, and the command is run in a directory containing a Sumatra - project, only that project's records be synchronized with the store at - PATH1. Note that PATH1 and PATH2 may be either filesystem paths or URLs. - """) # need to say what happens if the sync is incomplete due to label collisions - parser = ArgumentParser(usage=usage, - description=description) - parser.add_argument('path1') - parser.add_argument('path2', nargs='?') - args = parser.parse_args(argv) - - store1 = get_record_store(args.path1) - if args.path2: - store2 = get_record_store(args.path2) +@cli.command() +@click.argument('path1', required=True) +@click.argument('path2', required=False) +def sync(path1, path2): + """Synchronize two record stores. + + If both PATH1 and PATH2 are given, the + record stores at those locations will be synchronized. If only PATH1 is + given, and the command is run in a directory containing a Sumatra + project, only that project's records be synchronized with the store at + PATH1. Note that PATH1 and PATH2 may be either filesystem paths or URLs. + """ + # need to say what happens if the sync is incomplete due to label collisions + store1 = get_record_store(path1) + if path2: + store2 = get_record_store(path2) collisions = store1.sync_all(store2) else: project = load_project() @@ -649,25 +704,30 @@ def sync(argv): collisions = store1.sync(store2, project.name) if collisions: - print("Synchronization incomplete: there are two records with the same name for the following: %s" % ", ".join(collisions)) + click.echo("Synchronization incomplete: there are two records with the " + "same name for the following: %s" % ", ".join(collisions)) sys.exit(1) -def migrate(argv): - usage = "%(prog)s migrate [options]" - description = dedent("""\ - If you have moved your data files to a new location, update the record - store to reflect the new paths. - """) +@cli.command() +@click.option('-d', '--datapath', metavar='PATH', + help="modify the path to the directory in which your results " + "are stored.") +@click.option('-i', '--input', metavar='PATH', + help="modify the path to the directory in which your input " + "data files are stored.") +@click.option('-A', '--archive', metavar='PATH', + help="modify the directory in which your results are archived.") +@click.option('-M', '--mirror', metavar='URL', + help="modify the URL at which your data files are mirrored.") +def migrate(**kwargs): + """Update location of data files. + + If you have moved your data files to a new location, update the record + store to reflect the new paths. + """ # might also want to update the repository upstream # should we keep a history of such changes? - parser = ArgumentParser(usage=usage, - description=description) - parser.add_argument('-d', '--datapath', metavar='PATH', help="modify the path to the directory in which your results are stored.") - parser.add_argument('-i', '--input', metavar='PATH', help="modify the path to the directory in which your input data files are stored.") - parser.add_argument('-A', '--archive', metavar='PATH', help="modify the directory in which your results are archived.") - parser.add_argument('-M', '--mirror', metavar='URL', help="modify the URL at which your data files are mirrored.") - args = parser.parse_args(argv) project = load_project() field_map = { "datapath": "datastore.root", @@ -675,22 +735,18 @@ def migrate(argv): "archive": "datastore.archive", "mirror": "datastore.mirror_base_url" } - - if not any(vars(args).values()): + if not any(kwargs.values()): warnings.warn( "Command 'smt migrate' had no effect. Please provide at least one " "argument. (Run 'smt help migrate' for help.)") else: for option_name, field in field_map.items(): - value = getattr(args, option_name) - if value: + value = kwargs.pop(option_name) + if value is not None: project.record_store.update(project.name, field, value) -def version(argv): - usage = "%(prog)s version" - description = "Print the Sumatra version." - parser = ArgumentParser(usage=usage, - description=description) - args = parser.parse_args(argv) - print(sumatra.__version__) +@cli.command() +def version(): + """Print the Sumatra version.""" + cli(['--version'])