Merge pull request #38 from AgPipeline/develop

Merging develop to main branch - no review

.github/workflows/testing_checks.yaml

@@ -32,10 +32,10 @@ jobs:
     steps:
       - name: Current python version
         run: python3 --version || echo python3 not installed
-      - name: Install Python 3.7
+      - name: Install Python 3.8
         uses: actions/setup-python@v2
         with:
-          python-version: '3.7'
+          python-version: '3.8'
       - name: Updated python version
         run: python3 --version
       - name: PYTHONPATH environment variable
@@ -46,7 +46,7 @@ jobs:
         run: python3 -m pip install --upgrade --no-cache-dir setuptools
       - name: Install python-opencv
         run: |
-          sudo apt-get install -y --no-install-recommends gcc g++ python3.7-dev
+          sudo apt-get install -y --no-install-recommends gcc g++ python3.8-dev
           python3 -m pip install --upgrade --no-cache-dir wheel
           python3 -m pip install --upgrade --no-cache-dir opencv-python
       - name: Fetch/update testing pip installations
@@ -57,7 +57,7 @@ jobs:
         run: find . -type f -name "*.py" > action_test_files.txt
       - name: Install system requirements
         shell: bash
-        run: 'sudo apt-get install -y python3-gdal gdal-bin libgdal-dev gcc g++ python3.7-dev'
+        run: 'sudo apt-get install -y python3-gdal gdal-bin libgdal-dev gcc g++ python3.8-dev'
       - name: Install Python numpy
         shell: bash
         run: 'python3 -m pip install --upgrade --no-cache-dir numpy wheel'
@@ -79,7 +79,7 @@ jobs:
       - name: Fetching testing files
         if: ${{ matrix.app == 'pytest' }}
         run: |
-          curl -X GET https://de.cyverse.org/dl/d/3CBFD03C-C82E-4EDE-A8E5-DD4DBD45C696/orthomosaic.tif -o "${PWD}/test_data/orthomosaic.tif"
+          curl -X GET "https://de.cyverse.org/dl/d/3CBFD03C-C82E-4EDE-A8E5-DD4DBD45C696/orthomosaic.tif" -o "${PWD}/test_data/orthomosaic.tif"
           curl -X GET "https://de.cyverse.org/dl/d/FD32B0CE-DBAB-4A44-B8A6-0E9AA4555A31/experiment.yaml" -o "${PWD}/test_data/experiment.yaml"
       - name: Set execution permission for testing
         run: chmod +x soilmask.py

.github/workflows/testing_docker.yaml

@@ -22,6 +22,18 @@ jobs:
     steps:
       - name: Fetch source code
         uses: actions/checkout@v2
+      - name: Current python version
+        run: python3 --version || echo python3 not installed
+      - name: Install Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+      - name: Updated python version
+        run: python3 --version
+      - name: Install python dependencies
+        run: |
+             python3 -m pip install -U pip
+             python3 -m pip install -U numpy Pillow
       - name: Create folders
         run: |
              mkdir ./inputs && chmod 777 ./inputs
@@ -55,7 +67,7 @@ jobs:
              chmod +x "./.github/workflows/docker_test_check.sh"
              "./.github/workflows/docker_test_check.sh"
       - name: Data quality check
-        run: if [[ "$(diff './test_data/orthomosaic_mask.tif' './outputs/orthomosaic_mask.tif')" == "" ]]; then echo "Test soilmask content matches expected"; exit 0; else echo "Error - soilmask contents mismatch from test_data/orthomosaicmask.tif"; diff './test_data/orthomosaic_mask.tif' './outputs/orthomosaic_mask.tif'; exit 1; fi;
+        run: python3 tests/compare_image_pixels.py './test_data/orthomosaic_mask.tif' './outputs/orthomosaic_mask.tif'
 
   artifact_cleanup:
     runs-on: ubuntu-latest

Dockerfile

@@ -1,5 +1,6 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 LABEL maintainer="Chris Schnaufer <[email protected]>"
+ENV DEBIAN_FRONTEND noninteractive
 
 # Add user
 RUN useradd -u 49044 extractor \
@@ -11,11 +12,11 @@ RUN chown -R extractor /home/extractor \
 RUN apt-get update && \
     apt-get upgrade -y && \
     apt-get install -y --no-install-recommends \
-        python3.7 \
+        python3.8 \
         python3-pip && \
-    ln -sfn /usr/bin/python3.7 /usr/bin/python && \
-    ln -sfn /usr/bin/python3.7 /usr/bin/python3 && \
-    ln -sfn /usr/bin/python3.7m /usr/bin/python3m && \
+    ln -sfn /usr/bin/python3.8 /usr/bin/python && \
+    ln -sfn /usr/bin/python3.8 /usr/bin/python3 && \
+    ln -sfn /usr/bin/python3.8m /usr/bin/python3m && \
     apt-get autoremove -y && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
@@ -32,13 +33,13 @@ RUN apt-get update && \
         libgdal-dev  \
         gcc \
         g++ \
-        python3.7-dev && \
+        python3.8-dev && \
     python3 -m pip install --upgrade --no-cache-dir \
         wheel && \
     python3 -m pip install --upgrade --no-cache-dir \
         numpy && \
     python3 -m pip install --upgrade --no-cache-dir \
-        pygdal==2.2.3.* && \
+        pygdal==3.0.4.* && \
     python3 -m pip install --upgrade --no-cache-dir \
         opencv-python && \
     apt-get remove -y \
@@ -83,4 +84,3 @@ RUN chmod a+x /home/extractor/soilmask.py
 
 USER extractor
 ENTRYPOINT ["/home/extractor/soilmask.py"]
-

soilmask.py

@@ -8,6 +8,7 @@
 import numpy as np
 from agpypeline import entrypoint, algorithm, geoimage
 from agpypeline.environment import Environment
+from agpypeline.checkmd import CheckMD
 import cv2
 
 from osgeo import gdal
@@ -37,7 +38,6 @@ def prepare_metadata_for_geotiff(transformer_info: dict = None) -> dict:
             A dict containing information to save with an image
         """
         extra_metadata = {}
-
         if transformer_info:
             extra_metadata["transformer_name"] = str(transformer_info.get("name", ""))
             extra_metadata["transformer_version"] = str(transformer_info.get("version", ""))
@@ -339,7 +339,7 @@ def add_parameters(self, parser: argparse.ArgumentParser) -> None:
         parser.epilog = 'Mask files are saved with the .msk filename extension added when it\'s not specified. ' + \
                         parser.epilog
 
-    def check_continue(self, environment: Environment, check_md: dict, transformer_md: list,
+    def check_continue(self, environment: Environment, check_md: CheckMD, transformer_md: list,
                        full_md: list) -> tuple:
         """Checks if conditions are right for continuing processing
         Arguments:
@@ -355,8 +355,8 @@ def check_continue(self, environment: Environment, check_md: dict, transformer_m
         result = {'code': -1002, 'message': "No TIFF files were specified for processing"}
 
         # Ensure we have a TIFF file
-        if check_md and 'list_files' in check_md:
-            files = check_md['list_files']()
+        if check_md:
+            files = check_md.get_list_files()
             try:
                 for one_file in files:
                     ext = os.path.splitext(one_file)[1].lower()
@@ -391,15 +391,14 @@ def perform_process(self, environment: Environment, check_md: dict, transformer_
 
         # Loop through the files
         try:
-            for one_file in check_md['list_files']():
+            for one_file in check_md.get_list_files():
                 # Check file by type
                 ext = os.path.splitext(one_file)[1].lower()
                 if ext not in self.supported_file_ext:
                     continue
                 if not os.path.exists(one_file):
                     logging.warning("Unable to access file '%s'", one_file)
                     continue
-
                 # Get the image's EPSG code
                 epsg = geoimage.get_epsg(one_file)
                 if epsg is None:
@@ -413,16 +412,14 @@ def perform_process(self, environment: Environment, check_md: dict, transformer_
                     logging.warning("Unable to get bounds of georeferenced image: '%s'",
                                     os.path.basename(one_file))
                     continue
-
                 # Get the mask name
                 if environment.args.out_file:
                     rgb_mask_tif = environment.args.out_file
                     if not os.path.dirname(rgb_mask_tif):
-                        rgb_mask_tif = os.path.join(check_md['working_folder'], rgb_mask_tif)
+                        rgb_mask_tif = os.path.join(check_md.working_folder, rgb_mask_tif)
                 else:
                     # Use the original name
-                    rgb_mask_tif = os.path.join(check_md['working_folder'], __internal__.get_maskfilename(one_file))
-
+                    rgb_mask_tif = os.path.join(check_md.working_folder, __internal__.get_maskfilename(one_file))
                 # Create the mask file
                 logging.debug("Creating mask file '%s'", rgb_mask_tif)
                 mask_ratio, mask_rgb = gen_cc_enhanced(one_file)

test_data/experiment.yaml

@@ -0,0 +1,9 @@
+%YAML 1.1
+---
+pipeline:
+    studyName: 'S7_20181011'
+    season: 'S7_20181011'
+    germplasmName: Sorghum bicolor
+    collectingSite: Maricopa
+    observationTimeStamp: '2018-10-11T13:01:02-08:00'
+

tests/compare_image_pixels.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""Python script for comparing two images they way we want them to be
+"""
+
+import argparse
+import PIL.Image
+import numpy as np
+
+
+def _get_params() -> tuple:
+    """Get the paths to the files
+    Returns:
+        A tuple containing the two paths
+    """
+    parser = argparse.ArgumentParser(description='Compares two image files by size and pixel value')
+
+    parser.add_argument('first_file', type=argparse.FileType('r'), help='The first image file to compare')
+    parser.add_argument('second_file', type=argparse.FileType('r'), help='The second image file to compare')
+
+    args = parser.parse_args()
+
+    return args.first_file.name, args.second_file.name
+
+
+def check_images(first_path: str, second_path: str) -> None:
+    """Compares the two image files and throws an exception if they don't match
+    Arguments:
+        first_path: the path to the first file to compare
+        second_path: the path to the second file to compare
+    """
+    first_pixels = PIL.Image.open(first_path)
+    first_array = np.array(first_pixels)
+    del first_pixels
+
+    second_pixels = PIL.Image.open(second_path)
+    second_array = np.array(second_pixels)
+    del second_pixels
+
+    if first_array.shape != second_array.shape:
+        raise RuntimeError("Image dimensions are different: %s vs %s" % (str(first_array.shape), str(second_array.shape)))
+
+    for i in range(0, first_array.shape[0]):
+        for j in range(0, first_array.shape[1]):
+            for k in range(0, first_array.shape[2]):
+                if first_array[i][j][k] - second_array[i][j][k] != 0:
+                    raise RuntimeError("Image pixels are different. First difference at %s %s %s" % (str(i), str(j), str(k)))
+
+
+if __name__ == '__main__':
+    path1, path2 = _get_params()
+    check_images(path1, path2)