feature/morph-serve-cache

.pre-commit-config.yaml

@@ -54,4 +54,4 @@ repos:
       - id: mypy
         types: [ python ]
         exclude: ^.*example.*$
-        additional_dependencies: [ pandas-stubs, types-requests, types-PyYAML, pydantic ]
+        additional_dependencies: [ pandas-stubs, types-requests, types-PyYAML, pydantic, types-tabulate ]

core/morph/api/service.py

@@ -1,4 +1,3 @@
-import ast
 import asyncio
 import io
 import json
@@ -25,13 +24,14 @@
 )
 from morph.api.utils import convert_file_output, convert_variables_values
 from morph.cli.flags import Flags
-from morph.config.project import load_project
+from morph.config.project import default_output_paths, load_project
 from morph.task.resource import PrintResourceTask
 from morph.task.run import RunTask
 from morph.task.utils.morph import find_project_root_dir
 from morph.task.utils.run_backend.errors import MorphFunctionLoadError
+from morph.task.utils.run_backend.execution import execution_cache
 from morph.task.utils.run_backend.state import MorphGlobalContext
-from morph.task.utils.sqlite import SqliteDBManager
+from morph.task.utils.run_backend.types import RunStatus
 
 logger = logging.getLogger("uvicorn")
 
@@ -61,9 +61,6 @@ def run_file_with_type_service(
             f"Alias not found {input.name}. Check the console for more detailed error information.",
         )
 
-    db_manager = SqliteDBManager(project_root)
-    db_manager.initialize_database()
-
     with click.Context(click.Command(name="")) as ctx:
         ctx.params["FILENAME"] = input.name
         ctx.params["RUN_ID"] = f"{int(time.time() * 1000)}"
@@ -80,20 +77,10 @@ def run_file_with_type_service(
             str(e),
         )
 
-    run_results = db_manager.get_run_records_by_run_id(task.run_id)
-    run_result = next(
-        (result for result in run_results if result["cell_alias"] == input.name), None
-    )
-    if run_result is None:
-        raise WarningError(
-            ErrorCode.ExecutionError,
-            ErrorMessage.ExecutionErrorMessage["executionFailed"],
-            "run result not found",
-        )
-    elif run_result["status"] == "failed":
-        if run_result["error"] is not None:
+    if task.final_state != RunStatus.DONE.value:
+        if task.error is not None:
             try:
-                error = json.loads(run_result["error"])["details"]
+                error = json.loads(task.error)
             except Exception:  # noqa
                 raise WarningError(
                     ErrorCode.ExecutionError,
@@ -110,14 +97,19 @@ def run_file_with_type_service(
             ErrorCode.ExecutionError,
             ErrorMessage.ExecutionErrorMessage["executionFailed"],
         )
-    try:
-        output_paths = ast.literal_eval(run_result["outputs"])
-    except Exception:  # noqa
+    output_paths = task.output_paths
+    if not output_paths or len(output_paths) == 0:
         raise WarningError(
             ErrorCode.ExecutionError,
             ErrorMessage.ExecutionErrorMessage["executionFailed"],
             "output not found",
         )
+
+    # ------------------------------------------------------------------
+    # After execution, update the global cache
+    # ------------------------------------------------------------------
+    execution_cache.update_cache(input.name, output_paths)
+
     output_path = output_paths[0]
     if input.type == "image" or input.type == "html":
         if len(output_paths) == 2:
@@ -185,9 +177,6 @@ def run_file_service(input: RunFileService) -> SuccessResponse:
             f"Alias not found {input.name}. Check the console for more detailed error information.",
         )
 
-    db_manager = SqliteDBManager(project_root)
-    db_manager.initialize_database()
-
     with click.Context(click.Command(name="")) as ctx:
         run_id = input.run_id if input.run_id else f"{int(time.time() * 1000)}"
         ctx.params["FILENAME"] = input.name
@@ -205,20 +194,10 @@ def run_file_service(input: RunFileService) -> SuccessResponse:
             str(e),
         )
 
-    run_results = db_manager.get_run_records_by_run_id(task.run_id)
-    run_result = next(
-        (result for result in run_results if result["cell_alias"] == input.name), None
-    )
-    if run_result is None:
-        raise WarningError(
-            ErrorCode.ExecutionError,
-            ErrorMessage.ExecutionErrorMessage["executionFailed"],
-            "run result not found",
-        )
-    elif run_result["status"] == "failed":
-        if run_result["error"] is not None:
+    if task.final_state != RunStatus.DONE.value:
+        if task.error is not None:
             try:
-                error = json.loads(run_result["error"])["details"]
+                error = json.loads(task.error)
             except Exception:  # noqa
                 raise WarningError(
                     ErrorCode.ExecutionError,
@@ -354,16 +333,7 @@ async def file_upload_service(input: UploadFileService) -> Any:
         )
 
         # Retrieve the result of the file_upload function
-        project_root = find_project_root_dir()
-        db_manager = SqliteDBManager(project_root)
-
-        run_results = db_manager.get_run_records_by_run_id(run_id)
-        run_result = next(
-            (result for result in run_results if result["run_id"] == run_id), None
-        )
-
-        # Retrieve the saved file path from the output
-        output_file = ast.literal_eval(run_result["outputs"])[0] if run_result else None
+        output_file = default_output_paths()[0]
         with open(output_file, "r") as f:
             saved_filepath = f.read()
 

core/morph/config/project.py

@@ -4,12 +4,12 @@
 import yaml
 from pydantic import BaseModel, Field
 
-from morph.constants import MorphConstant
 from morph.task.utils.connection import (
     CONNECTION_TYPE,
     MORPH_DUCKDB_CONNECTION_SLUG,
     MorphConnection,
 )
+from morph.task.utils.morph import find_project_root_dir
 
 
 class Schedule(BaseModel):
@@ -27,11 +27,6 @@ class MorphProject(BaseModel):
     profile: Optional[str] = "default"
     source_paths: List[str] = Field(default_factory=lambda: ["src"])
     default_connection: Optional[str] = MORPH_DUCKDB_CONNECTION_SLUG
-    output_paths: List[str] = Field(
-        default_factory=lambda: [
-            f"{MorphConstant.TMP_MORPH_DIR}/{{name}}/{{run_id}}{{ext()}}"
-        ]
-    )
     scheduled_jobs: Optional[Dict[str, ScheduledJob]] = Field(default=None)
     result_cache_ttl: Optional[int] = Field(default=0)
     project_id: Optional[str] = Field(default=None)
@@ -40,6 +35,11 @@ class Config:
         arbitrary_types_allowed = True
 
 
+def default_output_paths() -> List[str]:
+    project_root = find_project_root_dir()
+    return [f"{project_root}/.morph/cache/{{name}}{{ext()}}"]
+
+
 def default_initial_project() -> MorphProject:
     return MorphProject()
 

core/morph/constants.py

@@ -7,7 +7,6 @@ class MorphConstant:
     TMP_MORPH_DIR = "/tmp/morph"
     # Files
     MORPH_CRED_PATH = os.path.expanduser("~/.morph/credentials")
-    MORPH_PROJECT_DB = "morph_project.sqlite3"
     MORPH_CONNECTION_PATH = os.path.expanduser("~/.morph/connections.yml")
     # Others
     EXECUTABLE_EXTENSIONS = [".sql", ".py"]

core/morph/include/starter_template/.gitignore

@@ -1,5 +1,4 @@
 # Morph config files and directories
 .gitconifg
 .morph
-morph_project.sqlite3
 node_modules

core/morph/include/starter_template/morph_project.yml

@@ -1,6 +1,4 @@
 default_connection: DUCKDB
-output_paths:
-- /tmp/morph/{name}/{run_id}{ext()}
 result_cache_ttl: 0
 scheduled_jobs: null
 source_paths:

core/morph/task/clean.py

@@ -23,32 +23,24 @@ def run(self):
             raise e
 
         clean_dir = Path(project_root).joinpath(".morph")
-        clean_files = ["meta.json", "knowledge.json", "template.json"]
-        clean_directories = ["frontend"]
-
-        for _f in clean_files:
-            clean_file = clean_dir.joinpath(_f)
-            if clean_file.exists():
-                if verbose:
-                    click.echo(click.style(f"Removing {clean_file}", fg="yellow"))
-                clean_file.unlink()
-            else:
-                if verbose:
-                    click.echo(click.style(f"File {clean_file} not found", fg="yellow"))
-
-        for _d in clean_directories:
-            clean_directory = clean_dir.joinpath(_d)
-            if clean_directory.exists():
-                if verbose:
-                    click.echo(click.style(f"Removing {clean_directory}", fg="yellow"))
-                shutil.rmtree(clean_directory)
-            else:
-                if verbose:
-                    click.echo(
-                        click.style(
-                            f"Directory {clean_directory} not found", fg="yellow"
-                        )
-                    )
+
+        if clean_dir.exists():
+            # Delete the entire .morph directory
+            if verbose:
+                click.echo(
+                    click.style(f"Removing directory {clean_dir}...", fg="yellow")
+                )
+            shutil.rmtree(clean_dir)
+
+            # Recreate the empty .morph directory
+            clean_dir.mkdir(parents=True, exist_ok=True)
+            if verbose:
+                click.echo(
+                    click.style(f"Recreated empty directory {clean_dir}", fg="yellow")
+                )
+        else:
+            if verbose:
+                click.echo(click.style(f"Directory {clean_dir} not found", fg="yellow"))
 
         click.echo(
             click.style(

core/morph/task/new.py

@@ -11,7 +11,6 @@
 from morph.task.base import BaseTask
 from morph.task.utils.connection import ConnectionYaml
 from morph.task.utils.run_backend.state import MorphGlobalContext
-from morph.task.utils.sqlite import SqliteDBManager
 
 
 class NewTask(BaseTask):
@@ -51,15 +50,6 @@ def run(self):
                 dest_file = os.path.join(target_path, template_file)
                 shutil.copy2(src_file, dest_file)
 
-        db_path = f"{self.project_root}/morph_project.sqlite3"
-        if not os.path.exists(db_path):
-            with open(db_path, "w") as f:
-                f.write("")
-
-        # Initialize the project database
-        db_manager = SqliteDBManager(self.project_root)
-        db_manager.initialize_database()
-
         # Execute the post-setup tasks
         original_working_dir = os.getcwd()
         os.chdir(self.project_root)

core/morph/task/resource.py

@@ -13,7 +13,6 @@
 from morph.task.utils.morph import Resource, find_project_root_dir
 from morph.task.utils.run_backend.inspection import get_checksum
 from morph.task.utils.run_backend.state import MorphGlobalContext, load_cache
-from morph.task.utils.sqlite import SqliteDBManager
 
 
 class PrintResourceTask(BaseTask):
@@ -45,10 +44,6 @@ def __init__(self, args: Flags):
             click.echo(click.style(str(e), fg="red"))
             raise e
 
-        # Initialize SQLite database
-        self.db_manager = SqliteDBManager(self.project_root)
-        self.db_manager.initialize_database()
-
     def run(self):
         try:
             cache = load_cache(self.project_root)