221.Add force checkout and compile functionality

fre/make/create_checkout_script.py

@@ -1,21 +1,48 @@
 '''
-checks out a makefile for a given model from the yamls
-i think!
+Checks out source code
 '''
 
 import os
 import subprocess
 import logging
 import sys
+import shutil
 import click
 import fre.yamltools.combine_yamls as cy
 from .gfdlfremake import varsfre, yamlfre, checkout, targetfre
 
-def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose):
+def baremetal_checkout_write_steps(model_yaml,src_dir,jobs,pc):
+    """
+    Go through steps to write the checkout script for bare-metal build
+    """
+    fre_checkout = checkout.checkout("checkout.sh",src_dir)
+    fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc)
+    fre_checkout.finish(pc)
+
+    # Make checkout script executable
+    os.chmod(src_dir+"/checkout.sh", 0o744)
+    print("    Checkout script created in "+ src_dir + "/checkout.sh \n")
+
+    return fre_checkout
+
+def container_checkout_write_steps(model_yaml,src_dir,tmp_dir,jobs,pc):
+    """
+    Go through steps to write the checkout script for container
+    """
+    fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir)
+    fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc)
+    fre_checkout.finish(pc)
+    print("    Checkout script created at " + tmp_dir + "/checkout.sh" + "\n")
+
+    return fre_checkout
+
+def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose,force_checkout):
+    """
+    Call gfdlfremake/checkout.py to create the checkout script
+    """
     # Define variables
     yml = yamlfile
     name = yamlfile.split(".")[0]
-    run = execute
     jobs = str(jobs)
     pcheck = no_parallel_checkout
 
@@ -65,29 +92,35 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v
 
         platform = model_yaml.platforms.getPlatformFromName(platform_name)
 
-        # ceate the source directory for the platform
+        # Create the source directory for the platform
         if not platform["container"]:
             src_dir = platform["modelRoot"] + "/" + fremake_yaml["experiment"] + "/src"
             # if the source directory does not exist, it is created
             if not os.path.exists(src_dir):
                 os.system("mkdir -p " + src_dir)
             # if the checkout script does not exist, it is created
             if not os.path.exists(src_dir+"/checkout.sh"):
-                fre_checkout = checkout.checkout("checkout.sh",src_dir)
-                fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc)
-                fre_checkout.finish(pc)
-                # Make checkout script executable
-                os.chmod(src_dir+"/checkout.sh", 0o744)
-                print("\nCheckout script created in "+ src_dir + "/checkout.sh \n")
+                print("\nCreating checkout script...")
+                fre_checkout = baremetal_checkout_write_steps(model_yaml,src_dir,jobs,pc)
 
                 # Run the checkout script
-                if run:
+                if execute:
                     fre_checkout.run()
                 else:
                     sys.exit()
             else:
-                print("\nCheckout script PREVIOUSLY created in "+ src_dir + "/checkout.sh \n")
-                if run:
+                if force_checkout:
+                    # Remove previous checkout
+                    print("\nRemoving previously checkout script and checked out source code")
+                    shutil.rmtree(src_dir)
+
+                    # Create checkout script
+                    print("Re-creating the checkout script...")
+                    fre_checkout = baremetal_checkout_write_steps(model_yaml,src_dir,jobs,pc)
+                else:
+                    print("\nCheckout script PREVIOUSLY created in "+ src_dir + "/checkout.sh \n")
+
+                if execute:
                     try:
                         subprocess.run(args=[src_dir+"/checkout.sh"], check=True)
                     except:
@@ -101,18 +134,29 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v
             src_dir = platform["modelRoot"] + "/" + fremake_yaml["experiment"] + "/src"
             bld_dir = platform["modelRoot"] + "/" + fremake_yaml["experiment"] + "/exec"
             tmp_dir = "tmp/"+platform_name
-            fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir)
-            fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(),jobs,pc)
-            fre_checkout.finish(pc)
-            print("\nCheckout script created at " + tmp_dir + "/checkout.sh" + "\n")
+            if not os.path.exists(tmp_dir+"/checkout.sh"):
+                # Create the checkout script
+                print("Creating checkout script...")
+                container_checkout_write_steps(model_yaml,src_dir,tmp_dir,jobs,pc)
+            else:
+                if force_checkout:
+                    # Remove the checkout script
+                    print("\nRemoving previously made checkout script")
+                    os.remove(tmp_dir+"/checkout.sh")
+
+                    # Create the checkout script
+                    print("Re-creating checkout script...")
+                    container_checkout_write_steps(model_yaml,src_dir,tmp_dir,jobs,pc)
+                else:
+                    print("\nCheckout script PREVIOUSLY created in "+ tmp_dir + "/checkout.sh" + "\n")
 
 @click.command()
-def _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose):
+def _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose,force_checkout):
     '''
     Decorator for calling checkout_create - allows the decorated version
     of the function to be separate from the undecorated version
     '''
-    return checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose)
+    return checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose,force_checkout)
 
 if __name__ == "__main__":
     checkout_create()

fre/make/create_compile_script.py

@@ -6,24 +6,44 @@
 from pathlib import Path
 from multiprocessing.dummy import Pool
 import click
+import subprocess
 from .gfdlfremake import varsfre, yamlfre, targetfre, buildBaremetal
 import fre.yamltools.combine_yamls as cy
 
-def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose):
+def compile_script_write_steps(yaml_obj,mkTemplate,src_dir,bld_dir,target,modules,modulesInit,jobs):
+    """
+    Go through steps to create the compile script
+    """
+    ## Create a list of compile scripts to run in parallel
+    fremakeBuild = buildBaremetal.buildBaremetal(exp = yaml_obj["experiment"],
+                                                 mkTemplatePath = mkTemplate,
+                                                 srcDir = src_dir,
+                                                 bldDir = bld_dir,
+                                                 target = target,
+                                                 modules = modules,
+                                                 modulesInit = modulesInit,
+                                                 jobs = jobs)
+    for c in yaml_obj['src']:
+        fremakeBuild.writeBuildComponents(c)
+    fremakeBuild.writeScript()
+
+    print("    Compile script created in " + bld_dir + "/compile.sh" + "\n")
+    return fremakeBuild
+
+def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_compile):
     # Define variables
     yml = yamlfile
     name = yamlfile.split(".")[0]
     nparallel = parallel
     jobs = str(jobs)
-    run = execute
 
     if verbose:
       logging.basicCOnfig(level=logging.INFO)
     else:
       logging.basicConfig(level=logging.ERROR)
 
-    srcDir="src"
-    checkoutScriptName = "checkout.sh"
+    src_dir="src"
+    checkout_script_name = "checkout.sh"
     baremetalRun = False # This is needed if there are no bare metal runs
 
     ## Split and store the platforms and targets in a list
@@ -38,10 +58,10 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose):
     full_combined = cy.combined_compile_existcheck(combined,yml,platform,target)
 
     ## Get the variables in the model yaml
-    freVars = varsfre.frevars(full_combined)
+    fre_vars = varsfre.frevars(full_combined)
 
     ## Open the yaml file and parse as fremakeYaml
-    modelYaml = yamlfre.freyaml(full_combined,freVars)
+    modelYaml = yamlfre.freyaml(full_combined,fre_vars)
     fremakeYaml = modelYaml.getCompileYaml()
 
     ## Error checking the targets
@@ -64,36 +84,59 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose):
          ## Check for type of build
          if platform["container"] is False:
               baremetalRun = True
-              bldDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/" + platformName + "-" + target.gettargetName() + "/exec"
-              os.system("mkdir -p " + bldDir)
-              ## Create a list of compile scripts to run in parallel
-              fremakeBuild = buildBaremetal.buildBaremetal(exp = fremakeYaml["experiment"],
-                                                       mkTemplatePath = platform["mkTemplate"],
-                                                       srcDir = srcDir,
-                                                       bldDir = bldDir,
-                                                       target = target,
-                                                       modules = platform["modules"],
-                                                       modulesInit = platform["modulesInit"],
-                                                       jobs = jobs)
-              for c in fremakeYaml['src']:
-                   fremakeBuild.writeBuildComponents(c)
-              fremakeBuild.writeScript()
-              fremakeBuildList.append(fremakeBuild)
-              click.echo("\nCompile script created at " + bldDir + "/compile.sh" + "\n")
-    if run:
+              bld_dir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/" + platformName + "-" + target.gettargetName() + "/exec"
+              os.system("mkdir -p " + bld_dir)
+              if not os.path.exists(bld_dir+"/compile.sh"):
+                  print("\nCreating the compile script...")
+                  fremakeBuild = compile_script_write_steps(yaml_obj = fremakeYaml,
+                                                            mkTemplate = platform["mkTemplate"],
+                                                            src_dir = src_dir,
+                                                            bld_dir = bld_dir,
+                                                            target = target,
+                                                            modules = platform["modules"],
+                                                            modulesInit = platform["modulesInit"],
+                                                            jobs = jobs)
+                  fremakeBuildList.append(fremakeBuild)
+                  if execute:
+                      print("Running the compile script\n")
+                      fremakeBuild.run()
+              else:
+                  if force_compile:
+                      # Remove compile script
+                      os.remove(bld_dir + "/compile.sh")
+                      # Re-create compile script
+                      print("\nRe-creating the compile script...")
+                      fremakeBuild = compile_script_write_steps(yaml_obj = fremakeYaml,
+                                                                mkTemplate = platform["mkTemplate"],
+                                                                src_dir = src_dir,
+                                                                bld_dir = bld_dir,
+                                                                target = target,
+                                                                modules = platform["modules"],
+                                                                modulesInit = platform["modulesInit"],
+                                                                jobs = jobs)
+                      fremakeBuildList.append(fremakeBuild)
+                      if execute:
+                          print("Running the compile script\n")
+                          fremakeBuild.run()
+                  else:
+                        print("Compile script PREVIOUSLY created here: " + bld_dir + "/compile.sh" + "\n")
+                        if execute:
+                            subprocess.run(args=[bld_dir+"/compile.sh"], check=True)
+                        ##TO-DO --> THIS COULD CAUSE PROBLEMS IF USER FORGOT TO DO FORCE-COMPILE AFTER A CHANGE --> IT'LL JUST RUN PREVIOUS ONE. I have the message about running previous compile script, but is it better to just do --force-compile (even after no change?)
+    if execute:
         if baremetalRun:
-            pool = Pool(processes=nparallel)                         # Create a multiprocessing Pool
+            pool = Pool(processes=nparallel)                            # Create a multiprocessing Pool
             pool.map(buildBaremetal.fremake_parallel,fremakeBuildList)  # process data_inputs iterable with pool
     else:
         sys.exit()
 
 @click.command()
-def _compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose):
+def _compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_compile):
     '''
     Decorator for calling compile_create - allows the decorated version
     of the function to be separate from the undecorated version
     '''
-    return compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose)
+    return compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose,force_compile)
 
 if __name__ == "__main__":
     compile_create()

fre/make/create_docker_script.py

@@ -5,11 +5,33 @@
 import subprocess
 from pathlib import Path
 import click
-#from .gfdlfremake import varsfre, targetfre, makefilefre, platformfre, yamlfre, buildDocker
 from .gfdlfremake import varsfre, targetfre, yamlfre, buildDocker
 import fre.yamltools.combine_yamls as cy
 
-def dockerfile_create(yamlfile,platform,target,execute):
+def dockerfile_write_steps(yaml_obj,img,run_env,target,td,cr,cb,cd):
+    """
+    Go through steps to write the Dockerfile
+    """
+    dockerBuild = buildDocker.container(base = img,
+                                        exp = yaml_obj["experiment"],
+                                        libs = yaml_obj["container_addlibs"],
+                                        RUNenv = run_env,
+                                        target = target)
+
+    dockerBuild.writeDockerfileCheckout("checkout.sh", td+"/checkout.sh")
+    dockerBuild.writeDockerfileMakefile(td+"/Makefile", td+"/linkline.sh")
+
+    for c in yaml_obj['src']:
+        dockerBuild.writeDockerfileMkmf(c)
+
+    dockerBuild.writeRunscript(run_env,cr,td+"/execrunscript.sh")
+    print(f"    Dockerfile created here: {cd}")
+
+    # Create build script for container
+    dockerBuild.createBuildScript(cb, cr)
+    print(f"    Container build script created here: {dockerBuild.userScriptPath}\n")
+
+def dockerfile_create(yamlfile,platform,target,execute,force_dockerfile):
     srcDir="src"
     checkoutScriptName = "checkout.sh"
     baremetalRun = False # This is needed if there are no bare metal runs
@@ -53,38 +75,53 @@ def dockerfile_create(yamlfile,platform,target,execute):
                 image=modelYaml.platforms.getContainerImage(platformName)
                 bldDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/exec"
                 tmpDir = "tmp/"+platformName
-                dockerBuild = buildDocker.container(base = image,
-                                              exp = fremakeYaml["experiment"],
-                                              libs = fremakeYaml["container_addlibs"],
-                                              RUNenv = platform["RUNenv"],
-                                              target = targetObject,
-                                              mkTemplate = platform["mkTemplate"])
-                dockerBuild.writeDockerfileCheckout("checkout.sh", tmpDir+"/checkout.sh")
-                dockerBuild.writeDockerfileMakefile(tmpDir+"/Makefile", tmpDir+"/linkline.sh")
-
-                for c in fremakeYaml['src']:
-                    dockerBuild.writeDockerfileMkmf(c)
-
-                dockerBuild.writeRunscript(platform["RUNenv"],platform["containerRun"],tmpDir+"/execrunscript.sh")
-                currDir = os.getcwd()
-                click.echo("\ntmpDir created in " + currDir + "/tmp")
-                click.echo("Dockerfile created in " + currDir +"\n")
-
-                # create build script for container
-                dockerBuild.createBuildScript(platform["containerBuild"], platform["containerRun"])
-                print("Container build script created at "+dockerBuild.userScriptPath+"\n\n")
-
-                # run the script if option is given
-                if run:
-                    subprocess.run(args=[dockerBuild.userScriptPath], check=True)
+                curr_dir = os.getcwd()
+                if not os.path.exists(f"{curr_dir}/Dockerfile"):
+                    dockerfile_write_steps(yaml_obj = fremakeYaml,
+                                           #makefile_obj = freMakefile,
+                                           img = image,
+                                           run_env = platform["RUNenv"],
+                                           target = targetObject,
+                                           td = tmpDir,
+                                           cr = platform["containerRun"],
+                                           cb = platform["containerBuild"],
+                                           cd = curr_dir)
+                else:
+                    if force_dockerfile:
+                        # Remove the dockerfile
+                        print("\nRemoving previously made dockerfile")
+                        os.remove(curr_dir+"/Dockerfile")
+
+                        # Create the checkout script
+                        print("Re-creating Dockerfile...")
+                        dockerfile_write_steps(yaml_obj = fremakeYaml,
+#                                               makefile_obj = freMakefile,
+                                               img = image,
+                                               run_env = RUNenv,
+                                               target = targetObject,
+                                               td = tmpDir,
+                                               cr = containerRun,
+                                               cb = containerBuild,
+                                               cd = curr_dir)
+                    else:
+                        print(f"Dockerfile PREVIOUSLY created here: {curr_dir}/Dockerfile")
+                        print(f"Container build script created here: {curr_dir}/createContainer.sh\n")
+
+                # Execute if flag is given
+                if execute:
+                    try:
+                        subprocess.run(args=[f"{curr_dir}/createContainer.sh"], check=True)
+                    except:
+                        print(f"There was an error in runnning the container build script: {curr_dir}/createContainer.sh")
+                        raise
 
 @click.command()
-def _dockerfile_create(yamlfile,platform,target,execute):
+def _dockerfile_create(yamlfile,platform,target,execute,force_dockerfile):
     '''
     Decorator for calling dockerfile_create - allows the decorated version
     of the function to be separate from the undecorated version
     '''
-    return dockerfile_create(yamlfile,platform,target,execute)
+    return dockerfile_create(yamlfile,platform,target,execute,force_dockerfile)
 
 if __name__ == "__main__":
     dockerfile_create()