From ffc08dff0be6f0e773242a09da268052d760c42b Mon Sep 17 00:00:00 2001 From: Sergey Yakubov Date: Wed, 27 Apr 2016 12:18:29 +0200 Subject: [PATCH] create test/tutorial via cli, always install tests --- CMakeLists.txt | 8 +-- Modules/Others/s2e/CMakeLists.txt | 7 --- Sources/python/SimEx/CLI/commands/modules.py | 3 +- .../commands/modules_commands/set_param.py | 12 +++- Sources/python/SimEx/CLI/commands/project.py | 4 +- .../project_commands/create_test_project.py | 62 +++++++++++++++++++ .../SimEx/CLI/utilities/module_files.py | 2 +- .../SimEx/CLI/utilities/project_files.py | 10 +-- .../S2EReconstruction_ParamTemplate.py | 6 +- Sources/python/SimEx/Templates/main.py | 20 +++--- simex_vars.sh.in | 1 + simex_vars_dev.sh.in | 2 + 12 files changed, 100 insertions(+), 37 deletions(-) create mode 100644 Sources/python/SimEx/CLI/commands/project_commands/create_test_project.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 3198ca83..47f907ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,9 +17,6 @@ set (CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${SIMEX_CMAKE_DIR}) option (DEVELOPER_INSTALL "Install modules only, do not install SimEx files" OFF) -CMAKE_DEPENDENT_OPTION(INSTALL_TESTS "Install test files" ON - "NOT DEVELOPER_INSTALL" OFF) - set (LIBDIR lib) set (INCLUDEDIR include) set (BINDIR bin) @@ -35,15 +32,12 @@ endif() if (DEVELOPER_INSTALL) set (FRAMEWORK_DIR ${SIMEX_SOURCE_DIR}/Sources) set (UNITTESTS_DIR ${FRAMEWORK_DIR}/unittest/) - set (TESTFILES_DIR ${UNITTESTS_DIR}/TestFiles/) set (UTILITIES_DIR ${FRAMEWORK_DIR}/python/SimEx/Utilities/) set (CALCULATORS_DIR ${FRAMEWORK_DIR}/python/SimEx/Calculators/) else() set (UTILITIES_DIR ${PYPATH}/SimEx/Utilities/) set (CALCULATORS_DIR ${PYPATH}/SimEx/Calculators/) - if (INSTALL_TESTS) - add_subdirectory (Tests) - endif() + add_subdirectory (Tests) add_subdirectory (Sources) endif() diff --git a/Modules/Others/s2e/CMakeLists.txt b/Modules/Others/s2e/CMakeLists.txt index 381f5509..49c739e5 100644 --- a/Modules/Others/s2e/CMakeLists.txt +++ b/Modules/Others/s2e/CMakeLists.txt @@ -12,13 +12,6 @@ INSTALL_COMMAND "" ExternalProject_Get_Property(s2e SOURCE_DIR) -#if (DEVELOPER_INSTALL) -#install(FILES ${SOURCE_DIR}/data/sim_example/diffr/s2e.beam -# ${SOURCE_DIR}/data/sim_example/diffr/s2e.geom -# ${SOURCE_DIR}/data/sim_example/sample/sample.h5 -# DESTINATION ${TESTFILES_DIR}) -#endif() - install(FILES ${SOURCE_DIR}/modules/diffr/prepHDF5.py DESTINATION ${PYPATH}) diff --git a/Sources/python/SimEx/CLI/commands/modules.py b/Sources/python/SimEx/CLI/commands/modules.py index 1d01c873..68d3d375 100644 --- a/Sources/python/SimEx/CLI/commands/modules.py +++ b/Sources/python/SimEx/CLI/commands/modules.py @@ -19,8 +19,7 @@ def set_arguments(parser): #enable module parser_enable = subparsers.add_parser('enable', help='enable module(s)') parser_enable.set_defaults(func=enable_module.process_args_enable) - - + #disable module parser_disable = subparsers.add_parser('disable', help='disable module(s)') parser_disable.set_defaults(func=enable_module.process_args_disable) diff --git a/Sources/python/SimEx/CLI/commands/modules_commands/set_param.py b/Sources/python/SimEx/CLI/commands/modules_commands/set_param.py index f943d138..b820ccb7 100644 --- a/Sources/python/SimEx/CLI/commands/modules_commands/set_param.py +++ b/Sources/python/SimEx/CLI/commands/modules_commands/set_param.py @@ -1,5 +1,5 @@ from SimEx.CLI.utilities import parse_modules,parse_classes,parse_settings,module_files - +import os def process_args(args): set_parameters(args.name,args.parameters) @@ -11,9 +11,10 @@ def getFromDict(dataDict, mapList): raise Exception return vals + def setInDict(dataDict, mapList, value): - getFromDict(dataDict, mapList) # to check if parameter exists - getFromDict(dataDict, mapList[:-1])[mapList[-1]] = value + old_value = getFromDict(dataDict, mapList) # to check if parameter exists + getFromDict(dataDict, mapList[:-1])[mapList[-1]] = type(old_value)(value) def set_param(param,module): try: @@ -39,7 +40,12 @@ def set_parameters(name,params): return paramfile = name+'_params' + module = __import__(paramfile) + + picfile = name+'_params.pyc' + os.remove(picfile) + to_write = False for param in params: diff --git a/Sources/python/SimEx/CLI/commands/project.py b/Sources/python/SimEx/CLI/commands/project.py index c5242122..6f657e75 100644 --- a/Sources/python/SimEx/CLI/commands/project.py +++ b/Sources/python/SimEx/CLI/commands/project.py @@ -1,4 +1,4 @@ -from project_commands import create_project +from project_commands import create_project,create_test_project def set_arguments(parser): @@ -6,5 +6,7 @@ def set_arguments(parser): parser = subparsers.add_parser('create', help='create project') parser.add_argument('name',metavar='',nargs=1,help='Project name') parser.set_defaults(func=create_project.process_args) + parser = subparsers.add_parser('create-test', help='create test project') + parser.set_defaults(func=create_test_project.process_args) diff --git a/Sources/python/SimEx/CLI/commands/project_commands/create_test_project.py b/Sources/python/SimEx/CLI/commands/project_commands/create_test_project.py new file mode 100644 index 00000000..648502dd --- /dev/null +++ b/Sources/python/SimEx/CLI/commands/project_commands/create_test_project.py @@ -0,0 +1,62 @@ + +import json +import os + +import create_project + + +def process_args(args): + create_test_project() + +def prGreen(prt,newline=True): + if (newline): + print("\033[92m{}\033[00m" .format(prt)) + else: + print("\033[92m{}\033[00m" .format(prt)), + +def prCyan(prt,newline=True): + if (newline): + print("\033[96m{}\033[00m" .format(prt)) + else: + print("\033[96m{}\033[00m" .format(prt)), + + +def runCommented(cmd,comment): + prCyan('*'*80) + prCyan(comment+':') + prGreen(cmd) + prCyan('*'*80) + os.system(cmd) + + +def create_test_project(): + if os.listdir('.'): + print('Cannot create test project, directory is not empty') + return + + modules_to_test = ["XFELPhotonSource","XFELPhotonPropagator","XMDYNDemoPhotonMatterInteractor", + "SingFELPhotonDiffractor","PerfectPhotonDetector","S2EReconstruction"] + + filestocopy = ["$SIMEX_TESTS/python/unittest/TestFiles/s2e.*","$SIMEX_TESTS/python/unittest/TestFiles/sample.h5", + "$SIMEX_TESTS/python/unittest/TestFiles/FELsource_out.h5"] + + commands = [ + ["create project","simex project create test"], + ["print available modules","simex module avail"], + ["add modules to the project","simex module add "+' '.join(modules_to_test)], + ["print project modules","simex module list"], + ["make input dir and copy necessary files (not simex command)", + "mkdir -p input/FELsource && cp "+ ' '.join(filestocopy)+" input" + \ + " && mv input/FELsource_out.h5 input/FELsource"], + ["set module parameter","simex module set XFELPhotonSource input_path=input/FELsource"], + ["set another module parameter","simex module set S2EReconstruction EMC_Parameters:max_number_of_iterations=2"], + ] + + for cmd in commands: + runCommented(cmd[1],cmd[0]) + + prCyan("Type",False),prGreen("simex run",False),prCyan("to start simulations") + +if __name__ == "__main__": + import sys + create_test_project() \ No newline at end of file diff --git a/Sources/python/SimEx/CLI/utilities/module_files.py b/Sources/python/SimEx/CLI/utilities/module_files.py index d89b85b3..55f6db71 100644 --- a/Sources/python/SimEx/CLI/utilities/module_files.py +++ b/Sources/python/SimEx/CLI/utilities/module_files.py @@ -23,6 +23,6 @@ def write_module_parameters(moduleName,module): s=pprint.pformat(module.parameters) print>>modfile,'parameters = ',s - modfile.close + modfile.close() diff --git a/Sources/python/SimEx/CLI/utilities/project_files.py b/Sources/python/SimEx/CLI/utilities/project_files.py index c2111f8e..c1028aa5 100644 --- a/Sources/python/SimEx/CLI/utilities/project_files.py +++ b/Sources/python/SimEx/CLI/utilities/project_files.py @@ -33,11 +33,11 @@ def create_modulecall_code(moduleName,prevModule,nextModule): input_path=${ModuleName}_params.input_path, output_path=${ModuleName}_params.output_path ) -print "-"*80 -print ("Running ${ModuleName} ...") +prCyan("-"*80) +prCyan ("Running ${ModuleName} ...") module_time=time.time() ${ModuleName}_inst.backengine() -print "Done in "+str(datetime.timedelta(seconds=time.time()-start_time)) +prCyan ("Done in "+str(datetime.timedelta(seconds=time.time()-module_time))) """ prevpath = """ if (${ModuleName}_params.input_path == 'default'): @@ -64,7 +64,7 @@ def update_main_file(): shutil.copy(src,dest) for line in fileinput.FileInput(dest, inplace=1): line=line.replace('${PROJECT_NAME}',fname) - print line.strip() + print line.rstrip() except shutil.Error as e: print('Error: %s' % e) # eg. source or destination doesn't exist @@ -86,4 +86,4 @@ def update_main_file(): for line in fileinput.FileInput(dest, inplace=1): line=line.replace('# modules will be added here',string) - print line.strip() + print line.rstrip() diff --git a/Sources/python/SimEx/Calculators/RegisteredCalculators/S2EReconstruction_ParamTemplate.py b/Sources/python/SimEx/Calculators/RegisteredCalculators/S2EReconstruction_ParamTemplate.py index ec9fdcc4..fa2c49f3 100644 --- a/Sources/python/SimEx/Calculators/RegisteredCalculators/S2EReconstruction_ParamTemplate.py +++ b/Sources/python/SimEx/Calculators/RegisteredCalculators/S2EReconstruction_ParamTemplate.py @@ -24,7 +24,7 @@ output_path = 'default' -emc_parameters = {'initial_number_of_quaternions' : 1, +EMC_Parameters = {'initial_number_of_quaternions' : 1, 'max_number_of_quaternions' : 9, 'max_number_of_iterations' : 3, 'min_error' : 1.0e-8, @@ -32,13 +32,13 @@ 'detailed_output' : False } -dm_parameters = {'number_of_trials' : 5, +DM_Parameters = {'number_of_trials' : 5, 'number_of_iterations' : 2, 'averaging_start' : 15, 'leash' : 0.2, 'number_of_shrink_cycles' : 2, } -parameters = {'EMC_Parameters' : emc_parameters, 'DM_Parameters' : dm_parameters} +parameters = {'EMC_Parameters' : EMC_Parameters, 'DM_Parameters' : DM_Parameters} diff --git a/Sources/python/SimEx/Templates/main.py b/Sources/python/SimEx/Templates/main.py index 2283db6a..7286db68 100644 --- a/Sources/python/SimEx/Templates/main.py +++ b/Sources/python/SimEx/Templates/main.py @@ -22,18 +22,22 @@ import time,datetime,os - - +def prGreen(prt,newline=True): + print ("\033[92m{}\033[00m" .format(prt)) + +def prCyan(prt,newline=True): + print ("\033[96m{}\033[00m" .format(prt)) + start_time=time.time() -print "="*80 -print "Simex platform. Copyright (C) 2016." -print "Running project ${PROJECT_NAME}" -print "-"*80 +prCyan("="*80) +prCyan("Simex platform. Copyright (C) 2016.") +prCyan("Running project ${PROJECT_NAME}") +prCyan("-"*80) # modules will be added here -print "-"*80 +prCyan("-"*80) -print "Finished in "+str(datetime.timedelta(seconds=time.time()-start_time)) +prCyan("Simex finished in "+str(datetime.timedelta(seconds=time.time()-start_time))) diff --git a/simex_vars.sh.in b/simex_vars.sh.in index e59f1b7b..2b0d8152 100644 --- a/simex_vars.sh.in +++ b/simex_vars.sh.in @@ -2,3 +2,4 @@ export PATH=${CMAKE_INSTALL_PREFIX}/${BINDIR}:$PATH export PYTHONPATH=${CMAKE_INSTALL_PREFIX}/${PYPATH}:$PYTHONPATH +export SIMEX_TESTS=${CMAKE_INSTALL_PREFIX}/${TESTSDIR} diff --git a/simex_vars_dev.sh.in b/simex_vars_dev.sh.in index 293ef3dc..f8ee736c 100644 --- a/simex_vars_dev.sh.in +++ b/simex_vars_dev.sh.in @@ -2,3 +2,5 @@ export PATH=${CMAKE_INSTALL_PREFIX}/${BINDIR}:$PATH export PYTHONPATH=${FRAMEWORK_DIR}/python:${CMAKE_INSTALL_PREFIX}/${PYPATH}:$PYTHONPATH +export SIMEX_TESTS=${SIMEX_SOURCE_DIR}/${TESTSDIR} +