Miroslavsimek/be 184 better builder import handling

.github/workflows/main.yml → .github/workflows/build.yaml

@@ -1,6 +1,16 @@
-name: Test python
+name: Build and test
 
-on: [ push ]
+on:
+  workflow_call:
+    inputs:
+      oarepo:
+        description: OARepo version (11, 12, ...)
+        required: true
+        default: 11
+        type: string
+
+env:
+  OAREPO_VERSION: ${{ github.event.inputs.oarepo }}
 
 jobs:
   build:
@@ -32,6 +42,11 @@ jobs:
           sudo sysctl -w fs.file-max=262144
           sudo sysctl -w vm.max_map_count=262144
 
+      - name: Runs Opensearch
+        uses: ankane/setup-opensearch@v1
+        with:
+          plugins: analysis-icu
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip wheel setuptools
@@ -40,52 +55,27 @@ jobs:
 
       - name: Test with pytest
         run: |
-          pytest tests
-      #
-      #    - name: Coveralls
-      #      uses: AndreMiras/coveralls-python-action@develop
-      #      with:
-      #        parallel: true
-      #        flag-name: Test with pytest
-
-      #  coveralls_finish:
-      #    needs: build
-      #    runs-on: ubuntu-latest
-      #    steps:
-      #      - name: Coveralls Finished
-      #        uses: AndreMiras/coveralls-python-action@develop
-      #        with:
-      #          parallel-finished: true
+          ./run-tests.sh
+
 
       - name: Build package to publish
         run: |
           python setup.py sdist bdist_wheel
 
+      - name: Freeze packages
+        run: |
+          .venv/bin/pip freeze > requirements.txt
+          .venv-server/bin/pip freeze >>requirements.txt
+          .venv-tests/bin/pip freeze >>requirements.txt
+
       - name: Archive production artifacts
         uses: actions/upload-artifact@v3
         with:
           name: dist
           path: dist
 
-  deploy:
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - name: Use built artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: dist
-          path: dist
-
-      - name: List files
-        run: |
-          ls -la
-          ls -la dist
-
-      - name: Publish package
-        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
-        uses: pypa/gh-action-pypi-publish@release/v1
+      - name: Archive production artifacts
+        uses: actions/upload-artifact@v3
         with:
-          skip_existing: true
-          user: __token__
-          password: ${{ secrets.PYPI_PASSWORD }}
+          name: requirements.txt
+          path: requirements.txt

.github/workflows/manual.yaml

@@ -0,0 +1,15 @@
+name: Dispatch
+
+on:
+  workflow_dispatch:
+    inputs:
+      oarepo:
+        description: OARepo version (11, 12, ...)
+        required: true
+        default: 11
+
+jobs:
+  build:
+    uses: ./.github/workflows/build.yaml
+    with:
+      oarepo: ${{ github.event.inputs.oarepo }}

.github/workflows/push.yaml

@@ -0,0 +1,36 @@
+name: Build, test and publish
+
+on: push
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  build:
+    uses: ./.github/workflows/build.yaml
+    with:
+      oarepo: 11
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Use built artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: dist
+          path: dist
+
+      - name: List files
+        run: |
+          ls -la
+          ls -la dist
+
+      - name: Publish package
+        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          skip_existing: true
+          user: __token__
+          password: ${{ secrets.PYPI_PASSWORD }}

.gitignore

@@ -90,3 +90,8 @@ schema-dump.json5
 schema-dump.json5
 vocabulary-postprocessed-schema.json
 .model_venv
+
+complex-model
+.venv*
+
+.token

oarepo_model_builder/builders/setup_cfg.py

@@ -34,11 +34,7 @@ def finish(self):
         output.setdefault("metadata", "authors", "")
 
         output.setdefault("options", "python", ">=3.9")
-        try:
-            ov = int(self.settings["oarepo-version"])
-            output.add_dependency("oarepo", f">={ov},<{ov+1}")
-        except ValueError:
-            output.add_dependency("oarepo", self.settings["oarepo-version"])
+
         output.add_dependency("oarepo-runtime", ">=1.0.0")
 
         output.setdefault("options", "packages", "find:")

oarepo_model_builder/builtin_models/invenio.json

@@ -1,20 +1,12 @@
 {
   "marshmallow": {
-    "base-classes": ["BaseRecordSchema"],
-      "imports": [
-        {
-          "import": "oarepo_runtime.marshmallow.BaseRecordSchema"
-        }
-      ]
+    "base-classes": ["oarepo_runtime.marshmallow.BaseRecordSchema"],
+      "imports": []
   },
   "ui": {
     "marshmallow": {
-      "base-classes": ["InvenioUISchema"],
-      "imports": [
-        {
-          "import": "oarepo_runtime.ui.marshmallow.InvenioUISchema"
-        }
-      ]
+      "base-classes": ["oarepo_runtime.services.schema.ui.InvenioUISchema"],
+      "imports": []
     }
   },
   "properties": {

oarepo_model_builder/datatypes/__init__.py

@@ -27,7 +27,7 @@
     NestedDataType,
     ObjectDataType,
 )
-from .datatypes import DataType, DataTypeComponent, Import, Section, datatypes  # noqa
+from .datatypes import DataType, DataTypeComponent, Section, datatypes  # noqa
 from .dates import (  # noqa
     DateDataType,
     DateTimeDataType,

oarepo_model_builder/datatypes/components/__init__.py

@@ -17,6 +17,7 @@
     AppModelComponent,
     BlueprintsModelComponent,
     DefaultsModelComponent,
+    EDTFIntervalDumperModelComponent,
     ExtResourceModelComponent,
     FacetsModelComponent,
     JSONSchemaModelComponent,
@@ -81,4 +82,5 @@
     FieldSortComponent,
     StringMarshmallowComponent,
     NumberMarshmallowComponent,
+    EDTFIntervalDumperModelComponent,
 ]

oarepo_model_builder/datatypes/components/enum.py

@@ -13,12 +13,12 @@ def process_marshmallow(self, datatype, section, **kwargs):
         enum = datatype.definition.get("enum")
         if enum:
             section.config.setdefault("validators", []).append(
-                f"ma_validate.OneOf({repr(enum)})"
+                f"{{{{marshmallow.validate.OneOf}}}}({repr(enum)})"
             )
 
     def process_ui(self, datatype, section, **kwargs):
         enum = datatype.definition.get("enum")
         if enum:
             section.config.setdefault("marshmallow", {}).setdefault(
                 "validators", []
-            ).append(f"ma_validate.OneOf({repr(enum)})")
+            ).append(f"{{{{marshmallow.validate.OneOf}}}}({repr(enum)})")

oarepo_model_builder/datatypes/components/facets/__init__.py

@@ -1,6 +1,12 @@
 import dataclasses
 from typing import Dict, List, Optional
 
+from oarepo_model_builder.utils.jinja import (
+    extract_extra_code_imports,
+    generate_extra_code,
+)
+from oarepo_model_builder.utils.python_name import PythonQualifiedName
+
 
 @dataclasses.dataclass
 class FacetDefinition:
@@ -20,10 +26,17 @@ def update(self, facet_section):
 
     def set_field(self, facet_section, arguments, field_class=None):
         field = facet_section.get("field")
+        extra_imports = None
         if field:
-            self.field = field
+            extra_imports = extract_extra_code_imports(field)
+            self.field = generate_extra_code(field)
         else:
             if not field_class:
                 field_class = facet_section.get("facet-class")
             if field_class:
-                self.field = f"{field_class}({', '.join(arguments)})"
+                field_class = PythonQualifiedName(field_class)
+                self.field = f"{field_class.local_name}({', '.join(arguments)})"
+                extra_imports = field_class.imports
+
+        if extra_imports:
+            self.imports = [*self.imports, *extra_imports]

oarepo_model_builder/datatypes/components/marshmallow/field.py

@@ -5,10 +5,18 @@
 from marshmallow import fields
 
 from oarepo_model_builder.datatypes import DataTypeComponent
-from oarepo_model_builder.utils.python_name import convert_name_to_python
+from oarepo_model_builder.utils.jinja import (
+    extract_extra_code_imports,
+    generate_extra_code,
+)
+from oarepo_model_builder.utils.python_name import (
+    Import,
+    PythonQualifiedName,
+    convert_name_to_python,
+)
 from oarepo_model_builder.validation.utils import ImportSchema, StrictSchema
 
-from ...datatypes import DataType, Import
+from ...datatypes import DataType
 from .graph import MarshmallowField
 
 
@@ -52,8 +60,11 @@ def _create_marshmallow_field(
             field_class = marshmallow.get("field-class")
             if not field_class:
                 return
+
+            field_class = PythonQualifiedName(field_class)
+            imports.extend(field_class.imports)
             field_decl = [
-                field_class,
+                field_class.local_name,
                 "(",
                 ", ".join(
                     self._marshmallow_field_arguments(
@@ -64,6 +75,11 @@ def _create_marshmallow_field(
             ]
             field = "".join(field_decl)
 
+            validators = marshmallow.get("validators", [])
+            if validators:
+                for validator in validators:
+                    imports.extend(extract_extra_code_imports(validator))
+
         fields.append(MarshmallowField(field_name, field, imports))
 
     def _marshmallow_field_arguments(
@@ -88,6 +104,7 @@ def _marshmallow_field_arguments(
 
         validators = marshmallow.get("validators", [])
         if validators:
+            validators = [generate_extra_code(x) for x in validators]
             arguments.append(f"validate=[{', '.join(validators)}]")
 
         return arguments

oarepo_model_builder/datatypes/components/marshmallow/graph.py

@@ -3,8 +3,7 @@
 
 from oarepo_model_builder.validation import InvalidModelException
 
-from ....utils.python_name import base_name, package_name
-from ...datatypes import Import
+from ....utils.python_name import Import, base_name, package_name
 
 
 @dataclasses.dataclass

oarepo_model_builder/datatypes/components/marshmallow/object.py

@@ -197,7 +197,8 @@ def _build_class(
         classes.append(
             MarshmallowClass(
                 class_name=marshmallow["class"],
-                base_classes=marshmallow.get("base-classes", []) or ["ma.Schema"],
+                base_classes=marshmallow.get("base-classes", [])
+                or ["marshmallow.Schema"],
                 imports=Import.from_config(marshmallow.get("imports", [])),
                 fields=fields,
                 unknown=marshmallow.get("unknown", "RAISE"),

oarepo_model_builder/datatypes/components/model/__init__.py

@@ -1,6 +1,7 @@
 from .app import AppModelComponent
 from .blueprints import BlueprintsModelComponent
 from .defaults import DefaultsModelComponent
+from .edtf_interval import EDTFIntervalDumperModelComponent
 from .ext_resource import ExtResourceModelComponent
 from .facets import FacetsModelComponent
 from .jsonschema import JSONSchemaModelComponent
@@ -44,4 +45,5 @@
     "ProxyModelComponent",
     "ExtResourceModelComponent",
     "SortModelComponent",
+    "EDTFIntervalDumperModelComponent",
 ]