Merge branch 'release/5.0.0'

.github/workflows/publish.yaml

@@ -12,36 +12,85 @@ env:
   REGISTRY_OLD: docker.io
   BASE_IMAGE_USER: ghcr.io/driplineorg
   BASE_IMAGE_REPO: dripline-cpp
-  BASE_IMAGE_VER: 'v2.9.1'
+  DEV_SUFFIX: '-dev'
+  BASE_IMAGE_TAG: 'v2.10.0'
+#  BASE_IMAGE_TAG: 'develop'
+#  DEV_SUFFIX: ''
 
 jobs:
 
   test_docker:
+
     runs-on: ubuntu-latest
+
+    env:
+      TAG: gha-test
+      INT_TAG: gha-int-test
+
     steps:
+
+      - name: Concatenate env variables
+        run: |
+          echo "DL_PY_TAG=${{ env.REGISTRY }}/${{ github.repository }}:${{ env.TAG }}" >> $GITHUB_ENV
+          echo "DL_PY_INT_TAG=${{ env.REGISTRY }}/${{ github.repository }}:${{ env.INT_TAG }}" >> $GITHUB_ENV
+
       - name: Checkout the repo 
         uses: actions/checkout@v4
         with:
           submodules: recursive
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
+
       - name: Set up Docker Buildx
         id: setup_buildx
         uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker
+
       - name: Build
         id: build
         uses: docker/build-push-action@v5
-        env:
-          BASE_IMAGE_TAG: ${{ env.BASE_IMAGE_VER }}-dev
         with:
           context: .
           push: false
+          load: true
           build-args: |
             img_user=${{ env.BASE_IMAGE_USER }}
             img_repo=${{ env.BASE_IMAGE_REPO }}
-            img_tag=${{ env.BASE_IMAGE_TAG }}
+            img_tag=${{ env.BASE_IMAGE_TAG }}${{ env.DEV_SUFFIX }}
           platforms: linux/amd64
+          tags: ${{ env.DL_PY_TAG }}
+
+      - name: Build Integration Tests
+        id: build_int_tests
+        uses: docker/build-push-action@v5
+        with:
+          context: ./tests/integration
+          push: false
+          load: true
+          build-args: |
+            img_user=${{ env.BASE_IMAGE_USER }}
+            img_repo=dripline-python
+            img_tag=${{ env.TAG }}
+          platforms: linux/amd64
+          tags: ${{ env.DL_PY_INT_TAG }}
+
+      - name: Unit Tests
+        run: |
+          docker run --rm ${{ env.DL_PY_INT_TAG }} bash -c "
+            cd /usr/local/src_py/tests
+            pytest"
+
+      - name: Integration Tests
+        run: |
+          cd tests/integration
+          ./do-testing.sh ${{ env.INT_TAG }}
 
+# For debugging
+#      - name: Setup tmate session
+#        if: ${{ ! success() }}
+#        uses: mxschmitt/action-tmate@v3
 
   build_and_push:
 
@@ -59,8 +108,12 @@ jobs:
         include:
           - build: dev
             tag-suffix: '-dev'
+            do-integration: true
+            int-tag-suffix: '-int-test'
           - build: release
             tag-suffix: ''
+            do-integration: false
+            int-tag-suffix: ''
 
     steps:
 
@@ -85,6 +138,22 @@ jobs:
             type=ref,event=pr
             type=raw,value=latest,enable=${{ github.ref == format('refs/heads{0}', 'main') }}
 
+      - name: Docker meta - integration
+        id: docker_meta_integration
+        if: ${{ matrix.do-integration }}
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ${{ env.REGISTRY }}/driplineorg/dripline-python
+          flavor: |
+            latest=auto
+            suffix=${{ matrix.int-tag-suffix }},onlatest=true
+          tags: |
+            type=semver,pattern={{raw}}
+            type=ref,event=branch
+            type=ref,event=pr
+            type=raw,value=latest,enable=${{ github.ref == format('refs/heads{0}', 'main') }}
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
@@ -111,18 +180,30 @@ jobs:
       - name: Build and push
         id: build_push
         uses: docker/build-push-action@v5
-        env:
-          BASE_IMAGE_TAG: ${{ env.BASE_IMAGE_VER }}${{ matrix.tag-suffix }}
         with:
           context: .
           push: ${{ github.event_name == 'push' }}  # limited to develop, main, and tags; don't push on PR
           build-args: | 
             img_user=${{ env.BASE_IMAGE_USER }}
             img_repo=${{ env.BASE_IMAGE_REPO }}
-            img_tag=${{ env.BASE_IMAGE_TAG }}
+            img_tag=${{ env.BASE_IMAGE_TAG }}${{ env.DEV_SUFFIX }}
           tags: ${{ steps.docker_meta.outputs.tags }}
-          platforms: linux/amd64,linux/arm/v7,linux/arm64
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
+
+      - name: Build and push - integration
+        id: build_push_integration
+        if: ${{ matrix.do-integration }}
+        uses: docker/build-push-action@v5
+        with:
+          context: ./tests/integration
+          push: ${{ github.event_name == 'push' }}  # limited to develop, main, and tags; don't push on PR
+          build-args: | 
+            img_user=${{ env.REGISTRY }}/driplineorg
+            img_repo=dripline-python
+            img_tag=latest-dev
+          tags: ${{ steps.docker_meta_integration.outputs.tags }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7
 
       - name: Release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         if: ${{ github.event_name == 'push' && contains(github.ref, 'refs/tags/') }}

.gitignore

@@ -65,6 +65,7 @@ build*/
 
 # Virtual environment
 venv/
+.venv/
 
 # Pycharm
 .idea

.readthedocs.yaml

@@ -26,6 +26,6 @@ sphinx:
 # Optional but recommended, declare the Python requirements required
 # to build your documentation
 # See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
-#python:
-#  install:
-#  - requirements: Nymph/Scarab/documentation/cpp/requirements.txt
+python:
+  install:
+  - requirements: doc/requirements.txt

CMakeLists.txt

@@ -1,8 +1,11 @@
 cmake_minimum_required(VERSION 3.5)  # <3.5 is deprecated by CMake
-project( DriplinePy VERSION 4.7.1 )
+project( DriplinePy VERSION 5.0.0 )
 
 cmake_policy( SET CMP0074 NEW )
 
+# Required by Scarab
+set( CMAKE_CXX_STANDARD 17 )
+
 # standard install directories
 include( GNUInstallDirs )
 
@@ -21,6 +24,10 @@ list( APPEND PUBLIC_EXT_LIBS Threads::Threads )
 
 find_package( pybind11 2.6.0 REQUIRED )
 list( APPEND PUBLIC_EXT_LIBS pybind11::module )
+string( TOLOWER ${CMAKE_BUILD_TYPE} build_type_lower )
+if( "${build_type_lower}" STREQUAL "debug" )
+    add_compile_definitions( PYBIND11_DETAILED_ERROR_MESSAGES=1 )
+endif()
 
 find_package( Dripline CONFIG REQUIRED )
 

Dockerfile

@@ -1,39 +1,22 @@
 ARG img_user=ghcr.io/driplineorg
 ARG img_repo=dripline-cpp
-#ARG img_tag=hotfix_2.6.2
-ARG img_tag=v2.9.1
-#ARG img_arch=arm
+#ARG img_tag=develop
+ARG img_tag=v2.10.0
 
 FROM ${img_user}/${img_repo}:${img_tag}
-#FROM ${img_user}/${img_repo}:${img_tag}-${img_arch}
-#FROM dlc_temp
 
 ## would prefer not to do this, just run ldconfig after the build to get things
 ## into the ld.so.conf cache... use this only when developing and adding libs
-ENV LD_LIBRARY_PATH /usr/local/lib
+ENV LD_LIBRARY_PATH=/usr/local/lib
 
 RUN apt-get update && \
     apt-get clean && \
     apt-get --fix-missing  -y install \
         libpq-dev && \
-    rm -rf /var/lib/apt/lists/*
-
-
-RUN pip install ipython
-RUN pip install pytest
-
-COPY module_bindings /usr/local/src_py/module_bindings
-COPY dripline /usr/local/src_py/dripline
-COPY bin /usr/local/src_py/bin
-COPY .git /usr/local/src_py/.git
-COPY pyproject.toml /usr/local/src_py/pyproject.toml
-COPY setup.py /usr/local/src_py/setup.py
-COPY CMakeLists.txt /usr/local/src_py/CMakeLists.txt
-COPY tests /usr/local/src_py/tests
-COPY LICENSE /usr/local/src_py/LICENSE
-COPY README.md /usr/local/src_py/README.md
-COPY pytest.ini /usr/local/src_py/pytest.ini
+    rm -rf /var/lib/apt/lists/* && \
+    pip install ipython pytest
 
+COPY . /usr/local/src_py/
 
 RUN pip install -v /usr/local/src_py
 

bin/dl-http-server

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+
+import scarab
+import dripline
+
+import logging
+logger = logging.getLogger('dripline')
+logger.setLevel(logging.DEBUG)
+logger.propagate = False
+
+def setup_logging():
+    handler = logging.StreamHandler()
+    handler.setLevel(logging.WARNING)
+    logger.addHandler(handler)
+    time_format = '%Y-%m-%dT%H:%M:%S.%fZ'
+    base_format = '%(asctime)s{}[%(levelname)-8s] %(name)s(%(lineno)d) -> {}%(message)s'
+    _logging_format = None
+    try:
+        import colorlog
+        modified_format = base_format.format('%(log_color)s', '%(purple)s')
+        _logging_format = colorlog.ColoredFormatter(modified_format, datefmt=time_format[:-4])
+    except ImportError:
+        modified_format = base_format.format(' ', '')
+        _logging_format = logging.Formatter(modified_format, time_format[:-4])
+    handler.setFormatter(_logging_format)
+    return handler
+
+def run_server(the_app):
+    log_handler = setup_logging()
+
+    this_config_param = the_app.primary_config
+    this_auth = the_app.auth
+
+    verbosity = the_app.global_verbosity
+    #print(f"verbosity is {verbosity}")
+    log_handler.setLevel(verbosity)
+
+    logger.debug(f'dl-http-server got this primary config:\n{this_config_param}')
+
+    # Convert config to a dict
+    this_config = this_config_param.to_python()
+
+    server = dripline.implementations.HTTPServer(**this_config)
+    server.http_listen()
+
+if __name__ == '__main__':
+
+    # App object
+    the_main = scarab.MainApp()
+
+    # Default configuration for a Dripline command
+    the_main.default_config = scarab.ParamNode()
+    the_main.default_config.add('dripline_mesh', dripline.core.DriplineConfig())
+    # We want scheduling to be on
+    the_main.default_config.add('http_host', scarab.ParamValue('0.0.0.0'))
+    the_main.default_config.add('http_port', scarab.ParamValue(8080))
+    the_main.default_config.add('web_root', scarab.ParamValue('/html'))
+
+    # Add the default dripline authentication specification and CL options
+    dripline.core.add_dripline_options(the_main)
+
+    # Package version information
+    the_main.set_version(dripline.core.get_version( "dripline-python" ))
+
+    # Callback function that will run everything
+    the_main.set_callback(lambda : run_server(the_main))
+
+    # Execute the callback (done via CLI11_PARSE)
+    import sys
+    the_main.execute(sys.argv)