Merge pull request #16 from Chris-Schnaufer/scif

Scif merge from personal repo

.github/workflows/action_pylint.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+cp ./.github/workflows/*.py ./
+

.github/workflows/pylint_check.yaml

@@ -0,0 +1,82 @@
+name: Enforcing pylint checks
+on:
+  push:
+    branches:
+      - master
+      - develop
+  pull_request:
+    branches:
+      - master
+      - develop
+    tags:
+      - v*
+
+jobs:
+  update_python:
+    runs-on: ubuntu-latest
+    name: Running pylint checks
+    steps:
+      - name: Current python version
+        run: python3 --version || echo python3 not installed
+      - name: Install Python 3.7
+        run: sudo apt-get install -y --no-install-recommends python3.7 python3-pip && sudo ln -sfn /usr/bin/python3.7 /usr/bin/python3
+        id: install_python_3_7
+      - name: Updated python version
+        run: python3 --version
+      - name: PYTHONPATH environment variable
+        run: echo ${PYTHONPATH}
+      - name: Update pip
+        run: python3 -m pip install --upgrade --no-cache-dir pip
+        id: pip-install
+      - name: Fetch/update setuptools
+        run: python3 -m pip install --upgrade --no-cache-dir setuptools
+        id: setuptools-install
+      - name: Install python-apt
+        run: sudo apt-get install -y python-apt
+      - name: HACK to fix apt-get update problem w/ different python versions
+        run: 'cd /usr/lib/python3/dist-packages && sudo cp apt_pkg.cpython-36m-x86_64-linux-gnu.so apt_pkg.so'
+      - name: Update apt-get
+        run: sudo apt-get update
+      - name: Fetch/update pylint
+        run: python3 -m pip install --upgrade --no-cache-dir pylint
+        id: pylint-install
+      - name: Fetch source code
+        uses: actions/checkout@v2
+        id: fetch-source
+      - name: Finding files
+        run: find . -type f -name "*.py" > action_pylint_files.txt
+        id: find-python-files
+      - name: Install system requirements
+        shell: bash
+        run: 'sudo apt-get install -y python3-gdal gdal-bin libgdal-dev gcc g++ python3.7-dev'
+        id: install-gdal-other-reqs
+      - name: Install Python numpy
+        shell: bash
+        run: 'python3 -m pip install --upgrade --no-cache-dir numpy wheel terrautils'
+        id: install-python-numpy
+      - name: Install Python pygdal
+        shell: bash
+        run: 'sudo python3 -m pip install --no-cache-dir pygdal==2.2.3.5'
+        id: install-python-pygdal
+      - name: Install system requirements from source
+        shell: bash
+        run: '[ -s "packages.txt" ] && (cat packages.txt | xargs apt-get install -y --no-install-recommends) || (echo "No addtional packages to install")'
+        id: install-system-reqs
+      - name: Install Python requirements from source
+        shell: bash
+        run: '[ -s "requirements.txt" ] && (python3 -m pip install --no-cache-dir -r requirements.txt) || (echo "No Python packages to install")'
+        id: install-python-reqs
+      - name: Run action pylint script
+        shell: bash
+        run: '[ -s ".github/workflows/action_pylint.sh" ] && (chmod +x ".github/workflows/action_pylint.sh" && ./.github/workflows/action_pylint.sh) || (echo "Error running shell script")'
+        id: run-special-action-script
+      - name: Fetching pylint.rc file
+        run: wget https://raw.githubusercontent.com/AgPipeline/Organization-info/master/pylint.rc
+        id: fetch_pylint_resource
+      - name: Listing
+        run: ls -la
+      - name: Files to be linted
+        run: cat action_pylint_files.txt
+      - name: Running pylint
+        run: cat action_pylint_files.txt | xargs python3 -m pylint --rcfile ./pylint.rc --extension-pkg-whitelist=cv2
+

.gitignore

@@ -1,4 +1,111 @@
-.idea/*
-*.pyc
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# pycharm
+.idea
+
+# Testing folder
 test
-.venv
+

Dockerfile

@@ -1,24 +1,36 @@
-FROM phusion/baseimage
-# Env variables
-ENV DEBIAN_FRONTEND noninteractive
+FROM agdrone/agpypeline:1.0
+LABEL maintainer="Chris Schnaufer <[email protected]>"
 
-RUN apt-get update -y \
-    && apt-get install --no-install-recommends -y \
-    wget \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh \
-    && bash miniconda.sh -b
-RUN  ~/miniconda3/bin/conda update -n base -c defaults conda
-COPY . /scif/apps/soilmask/src/
+# Start installing things
+COPY requirements.txt packages.txt /home/extractor/
 
-# Install the filesystem from the recipe
-COPY *.scif /
-RUN scif install /recipe.scif
+USER root
 
-# Cleanup APT
-RUN apt-get clean \
-  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+RUN [ -s /home/extractor/packages.txt ] && \
+    (echo 'Installing packages' && \
+        apt-get update && \
+        cat /home/extractor/packages.txt | xargs apt-get install -y --no-install-recommends && \
+        rm /home/extractor/packages.txt && \
+        apt-get autoremove -y && \
+        apt-get clean && \
+        rm -rf /var/lib/apt/lists/*) || \
+    (echo 'No packages to install' && \
+        rm /home/extractor/packages.txt)
 
-# SciF Entrypoint
-ENTRYPOINT ["scif"]
+RUN [ -s /home/extractor/requirements.txt ] && \
+    (echo "Install python modules" && \
+    python3 -m pip install -U --no-cache-dir pip && \
+    python3 -m pip install --no-cache-dir setuptools && \
+    python3 -m pip install --no-cache-dir -r /home/extractor/requirements.txt && \
+    rm /home/extractor/requirements.txt) || \
+    (echo "No python modules to install" && \
+    rm /home/extractor/requirements.txt)
+
+USER extractor
+COPY configuration.py soilmask.py /home/extractor/
+
+USER root
+RUN chmod a+x /home/extractor/soilmask.py
+
+USER extractor
+ENTRYPOINT ["/home/extractor/soilmask.py"]

README.md

@@ -1,5 +1,88 @@
 # Transformer Soil Mask
 
-Masks out the soil in images
+Converts an RGB image into a soil mask in which the soil is represented as black.
 
-## TODO
+The core idea for this Transformer is a plant-soil segmentation. 
+
+## Algorithm Description
+
+The core idea for this Transformer is a plant-soil segmentation. We apply a threshold to differentiate plant and soil, and do a smoothing after binary processing. Saturated portions of the image are removed. At the end, it returns the plant area ratio (canopy cover) within a bounding box.
+
+Steps:
+
+1. Split image data into R,G,B channel, and make a tmp image.
+2. For each pixel, if G value is T(threshold) higher than R value, make this pixel as foreground, and set the tmp pixel value to 255, so all tmp pixels are 0 or 255.
+3. Use a filter to blur this tmp image
+4. Remove anomalies (small areas incorrectly classified as plant of interest)
+4. Threshold the blurred tmp image with a threshold of 128 to get a new mask image that represents our plant (foreground) detections.
+5. Remove saturated pixels
+5. Output ratio = foreground pixel count / total pixel count
+
+### Parameters
+
+* G - R Threshold is set to 2 for normal situation.
+* Blur: image to new mask threshold is set to 128; passed to the OpenCV blur function.
+* Saturation threshold: threshold for classifying a pixel as saturated. Default is 245 in a greyscale imagess
+* Small Area Threshold: Used to remove anomalies from the image - this parameter is the size of a mask fragment in pixels that is removed. 
+
+### Quality Statement
+
+Currently, this algorithm has been used on wheat and sorghum; it has been tested on lettuce but only works when the leaves are green (fails if they are red or purple).
+
+We believe the tested threshold works well in a normal illumination. Below are three examples of successful segmentation:
+
+![cc1](figures/normal_canopy_cover.png)
+![cc2](figures/normal_canopy_cover2.png)
+
+![cc3](figures/normal_canopy_cover3.png)
+
+At the same time, there are some limitations with the current threshold. Here are some examples:
+
+1. Image captured in a low illumination.
+
+![2016-10-07__03-06-00-741](figures/low_illumination.jpg)
+
+2. Image captured in a very high illumination.
+
+![2016-09-28__12-19-06-452](figures/high_illumination.jpg)
+
+3. In late season, panicle is covering a lot in the image, and leaves is getting yellow.
+
+![2016-11-15__09-45-50-604](figures/yellow_plant.jpg)
+
+4. Sometimes an unidentified sensor problem results in a blank image.
+
+![2016-10-10__11-04-18-165](figures/sensor_problem.jpg)
+
+For more details, see related discussions, including: https://github.com/terraref/reference-data/issues/186#issuecomment-333631648
+
+### Sample Docker Command line
+
+Below is a sample command line that shows how the soil mask Docker image could be run.
+An explanation of the command line options used follows.
+Be sure to read up on the [docker run](https://docs.docker.com/engine/reference/run/) command line for more information.
+
+```docker run --rm --mount "src=/home/test,target=/mnt,type=bind" agpipeline/soilmask:2.0 --working_space "/mnt" --metadata "/mnt/08f445ef-b8f9-421a-acf1-8b8c206c1bb8_metadata_cleaned.json" "/mnt/08f445ef-b8f9-421a-acf1-8b8c206c1bb8_left.tif" ```
+
+This example command line assumes the source files are located in the `/home/test` folder of the local machine.
+The name of the image to run is `agpipeline/soilmask:2.0`.
+
+We are using the same folder for the source files and the output files.
+By using multiple `--mount` options, the source and output files can be separated.
+
+**Docker commands** \
+Everything between 'docker' and the name of the image are docker commands.
+
+- `run` indicates we want to run an image
+- `--rm` automatically delete the image instance after it's run
+- `--mount "src=/home/test,target=/mnt,type=bind"` mounts the `/home/test` folder to the `/mnt` folder of the running image
+
+We mount the `/home/test` folder to the running image to make files available to the software in the image.
+
+**Image's commands** \
+The command line parameters after the image name are passed to the software inside the image.
+Note that the paths provided are relative to the running image (see the --mount option specified above).
+
+- `--working_space "/mnt"` specifies the folder to use as a workspace
+- `--metadata "/mnt/08f445ef-b8f9-421a-acf1-8b8c206c1bb8_metadata.cleaned.json"` is the name of the source metadata to be cleaned
+- `"/mnt/08f445ef-b8f9-421a-acf1-8b8c206c1bb8_left.tif"` is the name of the image to mask

configuration.py

@@ -0,0 +1,36 @@
+"""Contains transformer configuration information
+"""
+from agpypeline.configuration import Configuration
+
+
+class ConfigurationSoilmask(Configuration):
+    """Configuration information for Soil Mask transformer"""
+    # Silence this error until we have public methods
+    # pylint: disable=too-few-public-methods
+
+    # The version number of the transformer
+    transformer_version = '2.0'
+
+    # The transformer description
+    transformer_description = 'Stereo RGB Image Enhancement & Masking'
+
+    # Short name of the transformer
+    transformer_name = 'terra.stereo-rgb.rgbmask'
+
+    # The name of the author of the extractor
+    author_name = 'Chris Schnaufer'
+
+    # The email of the author of the extractor
+    author_email = '[email protected]'
+
+    # Repository URI of where the source code lives
+    repository = 'https://github.com/Chris-Schnaufer/rgbmask.git'
+
+    # Contributors to this transformer
+    contributors = ['Max Burnette', 'Zongyang Li', 'Todd Nicholson']
+
+    # The sensor associated with the transformer
+    transformer_sensor = 'stereoTop'
+
+    # The transformer type (eg: 'rgbmask', 'plotclipper')
+    transformer_type = 'rgb_mask'