Echometrics improvements early 2023

.dockerignore

@@ -4,5 +4,4 @@
 .gitignore
 .travis.yml
 example/
-gutils/tests/
 .envrc

.github/workflows/publish.yml

@@ -85,7 +85,7 @@ jobs:
         outputs: type=docker
 
     - name: Run tests from the Docker image
-      run: docker run axiom/gutils:${{ github.event.release.tag_name }} pytest -m "not long and not watch"
+      run: docker run axiom/gutils:${{ github.event.release.tag_name }} pytest -m "not long"
 
   docker-push:
     name: Push tagged image to Docker Hub

.github/workflows/push.yml

@@ -123,7 +123,7 @@ jobs:
         outputs: type=docker
 
     - name: Run tests from the Docker image
-      run: docker run axiom/gutils:latest pytest -m "not long and not watch"
+      run: docker run axiom/gutils:latest pytest -m "not long"
 
   docker-push:
     name: Push latest image to Docker Hub

Dockerfile

@@ -32,19 +32,14 @@ RUN apt-get update && apt-get install -y \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Setup CONDA (https://hub.docker.com/r/continuumio/miniconda3/~/dockerfile/)
-ENV MINICONDA_VERSION py39_4.11.0
-ENV MINICONDA_SHA256 4ee9c3aa53329cd7a63b49877c0babb49b19b7e5af29807b793a76bdb1d362b4
-RUN curl -k -o /miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-$MINICONDA_VERSION-Linux-x86_64.sh && \
-    echo $MINICONDA_SHA256 /miniconda.sh | sha256sum --check && \
-    /bin/bash /miniconda.sh -b -p /opt/conda && \
-    rm /miniconda.sh && \
-    /opt/conda/bin/conda update -c conda-forge -n base conda && \
-    /opt/conda/bin/conda clean -afy && \
-    /opt/conda/bin/conda init && \
-    find /opt/conda/ -follow -type f -name '*.a' -delete && \
-    find /opt/conda/ -follow -type f -name '*.js.map' -delete && \
-    /opt/conda/bin/conda install -y -c conda-forge -n base mamba && \
-    /opt/conda/bin/conda clean -afy
+ENV MAMBAFORGE_VERSION 23.1.0-1
+ENV MAMBAFORGE_DOWNLOAD Mambaforge-${MAMBAFORGE_VERSION}-Linux-x86_64.sh
+RUN curl -k -L -O "https://github.com/conda-forge/miniforge/releases/download/${MAMBAFORGE_VERSION}/${MAMBAFORGE_DOWNLOAD}" && \
+    curl -k -L -O "https://github.com/conda-forge/miniforge/releases/download/${MAMBAFORGE_VERSION}/${MAMBAFORGE_DOWNLOAD}.sha256" && \
+    sha256sum --check "${MAMBAFORGE_DOWNLOAD}.sha256" && \
+    /bin/bash ${MAMBAFORGE_DOWNLOAD} -b -p /opt/conda && \
+    rm ${MAMBAFORGE_DOWNLOAD} && \
+    /opt/conda/bin/mamba clean -afy
 
 ENV PATH /opt/conda/bin:$PATH
 

environment.yml

@@ -17,37 +17,38 @@ dependencies:
   - brotlipy=0.7.0
   - bzip2=1.0.8
   - c-ares=1.18.1
-  - ca-certificates=2022.12.7
+  - ca-certificates=2023.5.7
   - cairo=1.16.0
   - cc-plugin-glider=1.0.6
-  - certifi=2022.12.7
+  - certifi=2023.5.7
   - cf-units=3.1.1
   - cffi=1.15.1
   - cfgv=3.3.1
   - cftime=1.6.1
   - charset-normalizer=2.1.1
   - colorama=0.4.6
   - compliance-checker=5.0.2
-  - cryptography=39.0.0
-  - curl=7.87.0
+  - contourpy=1.0.7
+  - cryptography=41.0.1
+  - curl=8.1.2
   - cycler=0.11.0
   - dataclasses=0.8
   - dbus=1.13.6
   - decorator=5.1.1
   - distlib=0.3.6
-  - exceptiongroup=1.1.1
+  - exceptiongroup=1.1.0
   - executing=1.2.0
   - expat=2.5.0
   - fftw=3.3.10
-  - filelock=3.10.0
+  - filelock=3.9.0
   - font-ttf-dejavu-sans-mono=2.37
   - font-ttf-inconsolata=3.000
   - font-ttf-source-code-pro=2.038
   - font-ttf-ubuntu=0.83
   - fontconfig=2.14.2
   - fonts-conda-ecosystem=1
   - fonts-conda-forge=1
-  - fonttools=4.39.2
+  - fonttools=4.39.0
   - freetype=2.12.1
   - geos=3.11.0
   - gettext=0.21.1
@@ -62,9 +63,11 @@ dependencies:
   - hdf4=4.2.15
   - hdf5=1.12.2
   - icu=70.1
-  - identify=2.5.21
+  - identify=2.5.19
   - idna=3.4
-  - importlib-metadata=6.1.0
+  - importlib-metadata=6.0.0
+  - importlib-resources=5.12.0
+  - importlib_resources=5.12.0
   - iniconfig=2.0.0
   - ipdb=0.13.11
   - ipython=8.11.0
@@ -90,7 +93,7 @@ dependencies:
   - libclang=15.0.7
   - libclang13=15.0.7
   - libcups=2.3.3
-  - libcurl=7.87.0
+  - libcurl=8.1.2
   - libdb=6.2.32
   - libdeflate=1.17
   - libedit=3.1.20191231
@@ -109,24 +112,24 @@ dependencies:
   - liblapack=3.9.0
   - libllvm15=15.0.7
   - libnetcdf=4.8.1
-  - libnghttp2=1.51.0
+  - libnghttp2=1.52.0
   - libnsl=2.0.0
   - libogg=1.3.4
   - libopenblas=0.3.21
   - libopus=1.3.1
   - libpng=1.6.39
-  - libpq=15.1
+  - libpq=15.3
   - libsndfile=1.2.0
   - libsqlite=3.40.0
-  - libssh2=1.10.0
+  - libssh2=1.11.0
   - libstdcxx-ng=12.2.0
   - libsystemd0=252
   - libtiff=4.5.0
   - libtool=2.4.7
   - libudev1=253
   - libuuid=2.32.1
   - libvorbis=1.3.7
-  - libwebp-base=1.3.0
+  - libwebp-base=1.2.4
   - libxcb=1.13
   - libxkbcommon=1.5.0
   - libxml2=2.10.3
@@ -139,18 +142,18 @@ dependencies:
   - matplotlib=3.5.3
   - matplotlib-base=3.5.3
   - matplotlib-inline=0.1.6
-  - mpg123=1.31.3
+  - mpg123=1.31.2
   - munkres=1.1.4
   - mysql-common=8.0.32
   - mysql-libs=8.0.32
   - ncurses=6.3
   - netcdf4=1.6.0
   - nodeenv=1.7.0
   - nspr=4.35
-  - nss=3.89
+  - nss=3.88
   - numpy=1.19.5
   - openjpeg=2.5.0
-  - openssl=1.1.1t
+  - openssl=3.1.1
   - owslib=0.28.1
   - packaging=23.0
   - pandas=1.4.4
@@ -162,12 +165,13 @@ dependencies:
   - pillow=9.4.0
   - pip=23.0.1
   - pixman=0.40.0
-  - platformdirs=3.1.1
+  - platformdirs=3.1.0
   - pluggy=1.0.0
   - ply=3.11
   - pocean-core=3.0.0
-  - pre-commit=3.2.0
-  - proj=9.1.1
+  - pooch=1.7.0
+  - pre-commit=3.1.1
+  - proj=9.2.0
   - prompt-toolkit=3.0.38
   - prompt_toolkit=3.0.38
   - pthread-stubs=0.4
@@ -178,14 +182,15 @@ dependencies:
   - pygeoif=1.0.0
   - pygments=2.14.0
   - pyinotify=0.9.6
-  - pyopenssl=23.0.0
+  - pyopenssl=23.2.0
   - pyparsing=3.0.9
-  - pyproj=3.4.1
+  - pyproj=3.6.0
   - pyqt=5.15.7
   - pyqt5-sip=12.11.0
   - pysocks=1.7.1
   - pytest=7.2.2
-  - python=3.9.15
+  - pytest-sugar=0.9.7
+  - python=3.9.16
   - python-dateutil=2.8.2
   - python_abi=3.9
   - pytz=2022.7.1
@@ -198,27 +203,28 @@ dependencies:
   - scipy=1.9.1
   - setuptools=67.6.0
   - shapely=1.8.4
-  - simplejson=3.18.4
+  - simplejson=3.18.3
   - sip=6.7.7
   - six=1.16.0
   - sqlite=3.40.0
   - stack_data=0.6.2
+  - termcolor=2.3.0
   - tk=8.6.12
   - toml=0.10.2
   - tomli=2.0.1
   - tornado=6.2
   - traitlets=5.9.0
-  - typing-extensions=4.5.0
-  - typing_extensions=4.5.0
+  - typing-extensions=4.4.0
+  - typing_extensions=4.4.0
   - tzdata=2022g
   - udunits2=2.2.28
   - ukkonen=1.0.1
   - unicodedata2=15.0.0
-  - urllib3=1.26.15
+  - urllib3=1.26.14
   - validators=0.20.0
-  - virtualenv=20.21.0
+  - virtualenv=20.20.0
   - wcwidth=0.2.6
-  - wheel=0.40.0
+  - wheel=0.38.4
   - whichcraft=0.6.1
   - xarray=2022.9.0
   - xcb-util=0.4.0

gutils/__init__.py

@@ -12,7 +12,12 @@
 
 import numpy as np
 import pandas as pd
-from scipy.signal import boxcar, convolve
+try:
+    # SciPy 1.13.0 DepricationWarning
+    from scipy.signal.windows import boxcar
+    from scipy.signal import convolve
+except ImportError:
+    from scipy.signal import boxcar, convolve
 
 from pocean.meta import MetaInterface
 from pocean.utils import (
@@ -21,6 +26,7 @@
 
 import logging
 L = logging.getLogger(__name__)
+L.setLevel(logging.INFO)
 
 __version__ = "3.2.0"
 

gutils/filters.py

@@ -113,13 +113,13 @@ def process_dataset(file,
         raise ValueError('Must specify path to combined ASCII file')
 
     try:
-        reader = reader_class(file)
+        reader = reader_class(file, kwargs=extra_kwargs)
         data = reader.standardize(z_axis_method=z_axis_method)
 
         # The data section is also passed to extras processing.
         # In certain circumstances, the dimension for certain
         # variables are reassigned with the extras dimension.
-        extras, data = reader.extras(data, **extra_kwargs)
+        extras, data = reader.extras(data)
 
         if 'z' not in data.columns:
             L.warning("No Z axis found - Skipping {}".format(file))

gutils/nc.py

@@ -14,7 +14,6 @@
 from datetime import datetime
 from collections import OrderedDict
 
-import numpy as np
 import pandas as pd
 import netCDF4 as nc4
 from compliance_checker.runner import ComplianceChecker, CheckSuite
@@ -371,10 +370,11 @@ def create_netcdf(attrs, data, output_path, mode, profile_id_type=ProfileIdTypes
 
             removable_columns = all_columns - set(reserved_columns)
             orphans = removable_columns - set(attrs.get('variables', {}).keys())
-            L.debug(
-                "Excluded from output (absent from JSON config):\n  * {}".format('\n  * '.join(orphans))
-            )
-            df.drop(orphans, axis=1, inplace=True)
+            if orphans != set():
+                L.debug(
+                    "Excluded from output (absent from JSON config):\n  * {}".format('\n  * '.join(orphans))
+                )
+                df.drop(orphans, axis=1, inplace=True)
 
         # Change to the datatype defined in the JSON. This is so
         # all netCDF files have the same dtypes for the variables in the end
@@ -386,17 +386,23 @@ def create_netcdf(attrs, data, output_path, mode, profile_id_type=ProfileIdTypes
     for pi, profile in data.groupby('profile'):
 
         # Fill in regular profile with empty data
+        # Q: Is cross filling required by the DAC?
+        """
         for c in extras_df:
             if c not in profile:
                 profile.loc[:, c] = np.nan
                 profile.loc[:, c] = profile[c].astype(extras_df[c].dtype)
+        """
 
         if not extras_df.empty:
 
+            # Skip extra flag
+            skip_extra = False
+
             # Write the extras dimension to a new profile file
             profile_extras = extras_df.loc[extras_df.profile == pi].copy()
             if profile_extras.empty:
-                continue
+                skip_extra = True
 
             # Standardize the columns of the "extras" from the matched profile
             profile_extras.loc[:, 't'] = profile_extras.index
@@ -409,14 +415,26 @@ def create_netcdf(attrs, data, output_path, mode, profile_id_type=ProfileIdTypes
                 profile_extras.loc[:, 'y'] = profile.y.dropna().iloc[0]
 
             # Fill in extras with empty data
+            # Q: Is cross filling required by the DAC?
+            """
             for c in profile:
                 if c not in profile_extras:
                     profile_extras.loc[:, c] = np.nan
                     profile_extras.loc[:, c] = profile_extras[c].astype(profile[c].dtype)
+            """
+
+            # If there is less than 3 time coordinates with an echogram, do not
+            # write out the profile.
+            if 'echogram_sv' in profile_extras.columns:
+                if 't' not in profile_extras.columns:
+                    skip_extra = True
+                if len(pd.unique(profile_extras['t'])) < 3:
+                    skip_extra = True
 
             try:
-                cr = create_profile_netcdf(attrs, profile_extras, output_path, mode + '_extra', profile_id_type)
-                written_files.append(cr)
+                if not skip_extra:
+                    cr = create_profile_netcdf(attrs, profile_extras, output_path, mode + '_extra', profile_id_type)
+                    written_files.append(cr)
             except BaseException:
                 L.exception('Error creating extra netCDF profile {}. Skipping.'.format(pi))
                 continue