Merge pull request #26 from AgPipeline/develop

Merge develop to master - no review

README.md

@@ -105,7 +105,7 @@ The command line parameters after the image name are passed to the software insi
 Note that the paths provided are relative to the running image (see the --mount option specified above).
 
 - `--working_space "/mnt"` specifies the folder to use as a workspace
-- `--metadata "/mnt/experiment.yaml"` is the name of the source metadata to be cleaned
+- `--metadata "/mnt/experiment.yaml"` is the name of the source metadata
 - `"/mnt/orthomosaic.tif"` is the name of the image to mask
 
 ## Acceptance Testing

configuration.py

@@ -9,13 +9,13 @@ class ConfigurationSoilmask(Configuration):
     # pylint: disable=too-few-public-methods
 
     # The version number of the transformer
-    transformer_version = '2.0'
+    transformer_version = '2.2'
 
     # The transformer description
-    transformer_description = 'Stereo RGB Image Enhancement & Masking'
+    transformer_description = 'RGB Image Soil Masking'
 
     # Short name of the transformer
-    transformer_name = 'terra.stereo-rgb.rgbmask'
+    transformer_name = 'soilmask'
 
     # The name of the author of the extractor
     author_name = 'Chris Schnaufer'
@@ -24,7 +24,7 @@ class ConfigurationSoilmask(Configuration):
     author_email = '[email protected]'
 
     # Repository URI of where the source code lives
-    repository = 'https://github.com/Chris-Schnaufer/rgbmask.git'
+    repository = 'https://github.com/AgPipeline/transformer-soilmask'
 
     # Contributors to this transformer
     contributors = ['Max Burnette', 'Zongyang Li', 'Todd Nicholson']
@@ -33,4 +33,4 @@ class ConfigurationSoilmask(Configuration):
     transformer_sensor = 'stereoTop'
 
     # The transformer type (eg: 'rgbmask', 'plotclipper')
-    transformer_type = 'rgb_mask'
+    transformer_type = 'rgbmask'

soilmask.py

@@ -2,6 +2,7 @@
 """Soil masking Transformer
 """
 
+import argparse
 import logging
 import os
 import numpy as np
@@ -219,7 +220,10 @@ def gen_rgb_mask(img: np.ndarray, bin_mask: np.ndarray) -> np.ndarray:
         Return:
             A new image that had the mask applied
         """
-        rgb_mask = cv2.bitwise_and(img, img, mask=bin_mask)
+        rgb_mask = cv2.bitwise_and(img[:, :, 0:3], img[:, :, 0:3], mask=bin_mask)
+
+        if img.shape[2] > 3:
+            rgb_mask = np.concatenate((rgb_mask, img[:, :, 3:]), axis=2)
 
         return rgb_mask
 
@@ -236,9 +240,10 @@ def check_saturation(img: np.ndarray) -> list:
 
         over_threshold = gray_img > SATURATE_THRESHOLD
         under_threshold = gray_img < 20  # 20 is a threshold to classify low pixel value
+        masked_count = 0 if img.shape[2] < 4 else np.sum(img[:, :, 3] == 0)
 
         over_rate = float(np.sum(over_threshold)) / float(gray_img.size)
-        low_rate = float(np.sum(under_threshold)) / float(gray_img.size)
+        low_rate = float(np.sum(under_threshold) - masked_count) / float(gray_img.size)
 
         return [over_rate, low_rate]
 
@@ -255,6 +260,22 @@ def get_maskfilename(filename: str) -> str:
 
         return base + "_mask" + ext
 
+    @staticmethod
+    def check_brightness(img: np.ndarray) -> float:
+        """Generate average pixel value from a BGR (blue, green, red) image array
+        Arguments:
+            img: the ndarray of image pixels to evaluate
+        Returns:
+            The average pixel value of the image
+        Notes:
+            This method computes the grayscale image used in the evaluation
+        """
+        gray_img = cv2.cvtColor(img, cv2.COLOR_BGRA2GRAY)
+
+        ave_value = np.average(gray_img)
+
+        return ave_value
+
 
 def gen_cc_enhanced(input_path: str, kernel_size: int = 3) -> tuple:
     """Generates an image mask keeping plants
@@ -266,7 +287,7 @@ def gen_cc_enhanced(input_path: str, kernel_size: int = 3) -> tuple:
     """
     # abandon low quality images, mask enhanced
     img = np.rollaxis(gdal.Open(input_path).ReadAsArray().astype(np.uint8), 0, 3)
-    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB if img.shape[2] < 4 else cv2.COLOR_BGRA2RGBA)
 
     # calculate image scores
     # pylint: disable=unused-variable
@@ -277,10 +298,11 @@ def gen_cc_enhanced(input_path: str, kernel_size: int = 3) -> tuple:
     # aveValue is average pixel value of grayscale image, if aveValue lower than 30 or higher than 195, return
     # quality_score is a score from Multiscale Autocorrelation (MAC), if quality_score lower than 13, return
 
-    # aveValue = check_brightness(img)
-    # quality_score = get_image_quality(input_path)
-    # if low_rate > 0.1 or aveValue < 30 or aveValue > 195 or quality_score < 13:
-    #    return None, None, None
+    ave_value = __internal__.check_brightness(img)
+    # if not quality_score:
+    #     quality_score = getImageQuality(input_path)
+    if low_rate > 0.1 or ave_value < 30 or ave_value > 195:
+        return None, None
 
     # saturated image process
     # over_rate is percentage of high value pixels(higher than SATURATE_THRESHOLD) in the grayscale image, if
@@ -306,6 +328,16 @@ def supported_file_ext(self) -> tuple:
         """Returns a tuple of supported file extensions in lowercase (with the preceeding dot: eg '.tif')"""
         return '.tiff', '.tif'
 
+    def add_parameters(self, parser: argparse.ArgumentParser) -> None:
+        """Adds parameters
+        Arguments:
+            parser: instance of argparse
+        """
+        # pylint: disable=no-self-use
+        parser.add_argument('--out_file', type=str, help='the path to save the masked file to')
+
+        parser.epilog += '  Mask files are saved with the .msk filename extension added when not specified.'
+
     def check_continue(self, environment: Environment, check_md: dict, transformer_md: list,
                        full_md: list) -> tuple:
         """Checks if conditions are right for continuing processing
@@ -331,6 +363,8 @@ def check_continue(self, environment: Environment, check_md: dict, transformer_m
                         result['code'] = 0
                         break
             except Exception as ex:
+                if logging.getLogger().level == logging.DEBUG:
+                    logging.exception("Exception caught in check_continue")
                 result['code'] = -1
                 result['error'] = "Exception caught processing file list: %s" % str(ex)
         else:
@@ -379,15 +413,24 @@ def perform_process(self, environment: Environment, check_md: dict, transformer_
                                     os.path.basename(one_file))
                     continue
 
-                # Get the mask name using the original name as reference
-                rgb_mask_tif = os.path.join(check_md['working_folder'], __internal__.get_maskfilename(one_file))
+                # Get the mask name
+                if environment.args.out_file:
+                    rgb_mask_tif = environment.args.out_file
+                    if not os.path.dirname(rgb_mask_tif):
+                        rgb_mask_tif = os.path.join(check_md['working_folder'], rgb_mask_tif)
+                else:
+                    # Use the original name
+                    rgb_mask_tif = os.path.join(check_md['working_folder'], __internal__.get_maskfilename(one_file))
 
                 # Create the mask file
                 logging.debug("Creating mask file '%s'", rgb_mask_tif)
                 mask_ratio, mask_rgb = gen_cc_enhanced(one_file)
+                if mask_rgb is None:
+                    logging.warning("Skipping over image that failed quality check: %s", one_file)
+                    continue
 
                 # Bands must be reordered to avoid swapping R and B
-                mask_rgb = cv2.cvtColor(mask_rgb, cv2.COLOR_BGR2RGB)
+                mask_rgb = cv2.cvtColor(mask_rgb, cv2.COLOR_BGR2RGB if mask_rgb.shape[2] < 4 else cv2.COLOR_BGRA2RGBA)
 
                 transformer_info = environment.generate_transformer_md()
 
@@ -412,6 +455,8 @@ def perform_process(self, environment: Environment, check_md: dict, transformer_
             result['file'] = file_md
 
         except Exception as ex:
+            if logging.getLogger().level == logging.DEBUG:
+                logging.exception("Exception caught in perform_process")
             result['code'] = -1001
             result['error'] = "Exception caught masking files: %s" % str(ex)
 

tests/test_soilmask.py

@@ -70,7 +70,7 @@ def test_prepare_metadata_for_geotiff():
                             assert value == str(test[METADATA_KEY_TRANSLATION['transformer_repo']]['repUrl'])
 
 
-def test_command_line():
+def test_simple_line():
     """Runs the command line and tests the result"""
     orthomosaic_mask_name = 'orthomosaic_mask.tif'
     result_name = 'result.json'
@@ -98,3 +98,34 @@ def test_command_line():
     img = gdal.Open(os.path.join(working_space, orthomosaic_mask_name)).ReadAsArray()
     assert img is not None
     assert isinstance(img, np.ndarray)
+
+
+def test_outputfile_command_line():
+    """Runs the command line and tests the result"""
+    orthomosaic_mask_name = 'soilmask.tif'
+    result_name = 'result.json'
+    source_image = os.path.join(TESTING_FILE_PATH, 'orthomosaic.tif')
+    source_metadata = os.path.join(TESTING_FILE_PATH, 'experiment.yaml')
+    assert os.path.exists(source_image)
+    assert os.path.exists(source_metadata)
+
+    working_space = os.path.realpath('./test_results')
+    os.makedirs(working_space, exist_ok=True)
+
+    command_line = [SOURCE_PATH, '--metadata', source_metadata, '--working_space', working_space,
+                    '--out_file', orthomosaic_mask_name, source_image]
+    subprocess.run(command_line, check=True)
+
+    # Check that the expected files were created
+    for expected_file in [result_name, orthomosaic_mask_name]:
+        assert os.path.exists(os.path.join(working_space, expected_file))
+
+    # Inspect the created files
+    with open(os.path.join(working_space, result_name)) as in_file:
+        res = json.load(in_file)
+        assert 'code' in res
+        assert res['code'] == 0
+
+    img = gdal.Open(os.path.join(working_space, orthomosaic_mask_name)).ReadAsArray()
+    assert img is not None
+    assert isinstance(img, np.ndarray)