Merge pull request #103 from ecmwf/develop

Release 0.4.4

.github/workflows/ci.yml

@@ -33,11 +33,11 @@ jobs:
       codecov_upload: true
     secrets: inherit
 
-  # Build downstream packages on HPC
-  downstream-ci-hpc:
-    name: downstream-ci-hpc
-    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
-    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
-    with:
-      anemoi-inference: ecmwf/anemoi-inference@${{ github.event.pull_request.head.sha || github.sha }}
-    secrets: inherit
+  # # Build downstream packages on HPC
+  # downstream-ci-hpc:
+  #   name: downstream-ci-hpc
+  #   if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+  #   uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
+  #   with:
+  #     anemoi-inference: ecmwf/anemoi-inference@${{ github.event.pull_request.head.sha || github.sha }}
+  #   secrets: inherit

.github/workflows/python-pull-request.yml

@@ -17,7 +17,7 @@ jobs:
   checks:
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
     uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2
     with:
       python-version: ${{ matrix.python-version }}

.gitignore

@@ -194,3 +194,4 @@ _version.py
 *.gif
 *.zarr/
 /*-plots/
+/definitions*/

.pre-commit-config.yaml

@@ -40,7 +40,7 @@ repos:
     - --force-single-line-imports
     - --profile black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.8.1
+  rev: v0.8.6
   hooks:
   - id: ruff
     args:
@@ -64,7 +64,7 @@ repos:
   hooks:
   - id: pyproject-fmt
 -   repo: https://github.com/jshwi/docsig # Check docstrings against function sig
-    rev: v0.65.0
+    rev: v0.66.1
     hooks:
     -   id: docsig
         args:

CHANGELOG.md

@@ -11,6 +11,9 @@ Keep it human-readable, your future self will thank you!
 ## [Unreleased]
 
 ### Added
+- Add support for models with unconnected nodes dropped from input [#95](https://github.com/ecmwf/anemoi-inference/pull/95).
+- Change trigger for boundary forcings [#95](https://github.com/ecmwf/anemoi-inference/pull/95).
+- Add support for automatic loading of anemoi-datasets of more general type [#95](https://github.com/ecmwf/anemoi-inference/pull/95).
 - Add initial state output in netcdf format
 - Fix: Enable inference when no constant forcings are used
 - Add anemoi-transform link to documentation
@@ -35,6 +38,7 @@ Keep it human-readable, your future self will thank you!
 - Fix SimpleRunner
 
 ### Removed
+- ci: turn off hpc workflow
 
 
 ## [0.2.0](https://github.com/ecmwf/anemoi-inference/compare/0.1.9...0.2.0) - Use earthkit-data

docs/configs/inputs.rst

@@ -95,3 +95,39 @@ You can change that to use ERA5 reanalysis data (``class=ea``).
 
 The ``mars`` input also accepts the ``namer`` parameter of the GRIB
 input.
+
+*****
+ cds
+*****
+
+You can also specify the input as ``cds`` to read the data from the
+`Climate Data Store <https://cds.climate.copernicus.eu/>`_. This
+requires the `cdsapi` package to be installed, and the user to have a
+CDS account.
+
+.. literalinclude:: inputs_8.yaml
+   :language: yaml
+
+As the CDS contains a plethora of `datasets
+<https://cds.climate.copernicus.eu/datasets>`_, you can specify the
+dataset you want to use with the key `dataset`.
+
+This can be a str in which case the dataset is used for all requests, or
+a dict of any number of levels which will be descended based on the
+key/values for each request.
+
+You can use `*` to represent any not given value for a key, i.e. set a
+dataset for `param: 2t`. and `param: *` to represent any other param.
+
+.. literalinclude:: inputs_9.yaml
+   :language: yaml
+
+In the above example, the dataset `reanalysis-era5-pressure-levels` is
+used for all with `levtype: pl` and `reanalysis-era5-single-levels` used
+for all with `levtype: sfc`.
+
+Additionally, any kwarg can be passed to be added to all requests, i.e.
+for ERA5 data, `product_type: 'reanalysis'` is needed.
+
+.. literalinclude:: inputs_10.yaml
+   :language: yaml

docs/configs/inputs_10.yaml

@@ -0,0 +1,7 @@
+input:
+  cds:
+    dataset:
+      levtype:
+        pl: reanalysis-era5-pressure-levels
+        sfc: reanalysis-era5-single-levels
+    product_type: 'reanalysis'

docs/configs/inputs_8.yaml

@@ -0,0 +1,3 @@
+input:
+  cds:
+    dataset: ???

docs/configs/inputs_9.yaml

@@ -0,0 +1,24 @@
+input:
+  cds:
+    # Dataset examples
+    ## As a string
+    dataset:
+      'reanalysis-era5-pressure-levels'
+
+    ## As a simple dictionary
+    dataset:
+      levtype:
+        pl: reanalysis-era5-pressure-levels
+        sfc: reanalysis-era5-single-levels
+
+    ## As a complex dictionary
+    dataset:
+      stream:
+        oper:
+          levtype:
+            pl: reanalysis-era5-pressure-levels
+            sfc: reanalysis-era5-single-levels
+        an:
+          # ... Other datasets
+        '*': # Any other stream
+          # ... Other datasets

pyproject.toml

@@ -54,7 +54,7 @@ dependencies = [
   "torch",
 ]
 
-optional-dependencies.all = [ "anemoi-inference[plugin]", "anemoi-utils[all]>=0.4.9" ]
+optional-dependencies.all = [ "anemoi-inference[plugin,huggingface]", "anemoi-utils[all]>=0.4.9" ]
 
 optional-dependencies.dev = [ "anemoi-inference[all,docs,plugin,tests]" ]
 
@@ -71,7 +71,7 @@ optional-dependencies.docs = [
 optional-dependencies.huggingface = [ "huggingface-hub" ]
 
 optional-dependencies.plugin = [ "ai-models>=0.7", "tqdm" ]
-optional-dependencies.tests = [ "anemoi-datasets[all]", "hypothesis", "pytest" ]
+optional-dependencies.tests = [ "anemoi-datasets[all]", "anemoi-inference[all]", "hypothesis", "pytest" ]
 
 urls.Documentation = "https://anemoi-inference.readthedocs.io/"
 urls.Homepage = "https://github.com/ecmwf/anemoi-inference/"

src/anemoi/inference/checkpoint.py

@@ -22,7 +22,7 @@
 LOG = logging.getLogger(__name__)
 
 
-def _download_huggingfacehub(huggingface_config):
+def _download_huggingfacehub(huggingface_config) -> str:
     """Download model from huggingface"""
     try:
         from huggingface_hub import hf_hub_download
@@ -34,17 +34,17 @@ def _download_huggingfacehub(huggingface_config):
         huggingface_config = {"repo_id": huggingface_config}
 
     if "filename" in huggingface_config:
-        config_path = hf_hub_download(**huggingface_config)
+        return str(hf_hub_download(**huggingface_config))
+
+    repo_path = Path(snapshot_download(**huggingface_config))
+    ckpt_files = list(repo_path.glob("*.ckpt"))
+
+    if len(ckpt_files) == 1:
+        return str(ckpt_files[0])
     else:
-        repo_path = Path(snapshot_download(**huggingface_config))
-        ckpt_files = list(repo_path.glob("*.ckpt"))
-        if len(ckpt_files) == 1:
-            return str(ckpt_files[0])
-        else:
-            ValueError(
-                f"Multiple ckpt files found in repo, {ckpt_files}.\nCannot pick one to load, please specify `filename`."
-            )
-    return config_path
+        raise ValueError(
+            f"None or Multiple ckpt files found in repo, {ckpt_files}.\nCannot pick one to load, please specify `filename`."
+        )
 
 
 class Checkpoint:
@@ -58,7 +58,7 @@ def __repr__(self):
         return f"Checkpoint({self.path})"
 
     @cached_property
-    def path(self):
+    def path(self) -> str:
         import json
 
         try:
@@ -67,7 +67,7 @@ def path(self):
             path = self._path
 
         if isinstance(path, (Path, str)):
-            return path
+            return str(path)
         elif isinstance(path, dict):
             if "huggingface" in path:
                 return _download_huggingfacehub(path["huggingface"])
@@ -312,7 +312,7 @@ def mars_requests(self, *, variables, dates, use_grib_paramid=False, **kwargs):
 
     @cached_property
     def _supporting_arrays(self):
-        return self._metadata.supporting_arrays
+        return self._metadata._supporting_arrays
 
     @property
     def name(self):