2nd Party

.github/workflows/alice_please_contribute_recommended_community_standards.yml

@@ -18,11 +18,10 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.9
-    - name: Install github CLI
+    - name: Configure git user info
       run: |
-        set -x
-        python -c 'import sys, functools, urllib.request; print(urllib.request.urlretrieve(sys.argv[-2], filename=sys.argv[-1], reporthook=lambda n, c, t: print(f"{round(((n*c)/t) * 100, 2)}%", end="\r", file=sys.stderr))[0])' https://github.com/cli/cli/releases/download/v2.12.1/gh_2.12.1_linux_amd64.deb gh_2.12.1_linux_amd64.deb
-        dpkg -i gh_2.12.1_linux_amd64.deb
+       git config --global user.email "[email protected]"
+       git config --global user.name "Alice"
     - name: Configure github CLI auth
       env:
         GH_USER: aliceoa

.github/workflows/alice_shouldi_contribute.yml

@@ -58,8 +58,6 @@ jobs:
       run: |
         set -x
         pwd
-        cd $(mktemp -d)
-        pwd
         alice shouldi contribute -keys "${REPO_URL}" -log debug
     - uses: actions/upload-artifact@v3
       with:

.github/workflows/new_2nd_party_tcb_eval.yml

@@ -0,0 +1,36 @@
+# Upstream: https://github.com/actions/importer-issue-ops/blob/f5fc2781a33d92f232f415c62d021dcf6dae1e06/.github/workflows/issue_ops.yml#license=MIT
+# Overlays:
+# - Upstream: https://github.com/stefanbuck/github-issue-parser/blob/145e4dc5ee373ffe0129d3a3d9fc44dce64f7491/README.md#license=MIT
+name: 'IssueOps: Alice: ShouldI: Contribute'
+on:
+  issues:
+    types: [created]
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  issue-parser:
+    outputs:
+      repo_name: ${{ steps.issue-parser.outputs.issueparser_repo_name }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: stefanbuck/github-issue-parser@v3
+      id: issue-parser
+      with:
+        template-path: .github/ISSUE_TEMPLATE/new_plugin.yml
+    - name: Add comment
+      uses: peter-evans/create-or-update-comment@a35cf36e5301d70b76f316e867e7788a55a31dae
+      with:
+        issue-number: ${{ github.event.issue.number }}
+        body: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+  contribute:
+    needs: issue-parser
+    uses: "./.github/workflows/alice_shouldi_contribute.yml"
+    with:
+      repo_url: "${{ needs.issue-parser.outputs.repo_url }}"
+    secrets:
+      GH_USER: "aliceoa"
+      GH_ACCESS_TOKEN: ${{ secrets.ALICE_SHOULDI_CONTRIBUTE_ISSUE_OPS }}

.github/workflows/testing.yml

@@ -9,8 +9,6 @@ on:
   pull_request:
     paths-ignore:
     - 'docs/arch/**'
-  schedule:
-  - cron:  '0 3 * * *'
 
 jobs:
   lint:
@@ -99,7 +97,11 @@ jobs:
         - feature/auth
         - service/http
         - configloader/yaml
-        python-version: [3.7]
+        - configloader/image
+        - source/mongodb
+        - entities/alice
+        python-version:
+        - "3.12"
 
     steps:
     - uses: actions/checkout@v2
@@ -161,6 +163,7 @@ jobs:
         model/pytorch=${{ secrets.PYPI_MODEL_PYTORCH }}
         model/xgboost=${{ secrets.PYPI_MODEL_XGBOOST }}
         source/mysql=${{ secrets.PYPI_SOURCE_MYSQL }}
+        source/mongodb=${{ secrets.PYPI_SOURCE_MONGODB }}
         feature/git=${{ secrets.PYPI_FEATURE_GIT }}
         feature/auth=${{ secrets.PYPI_FEATURE_AUTH }}
         operations/binsec=${{ secrets.PYPI_OPERATIONS_BINSEC }}
@@ -200,6 +203,7 @@ jobs:
         - docs/examples/shouldi.rst
         - docs/examples/innersource/swportal.rst
         - docs/examples/innersource/microservice.rst
+        - docs/examples/innersource/kubernetes.rst
         - docs/examples/icecream_sales.rst
         - docs/examples/data_cleanup/data_cleanup.rst
         - docs/examples/data_cleanup/data_cleanup_classfication.rst

.gitignore

@@ -39,3 +39,5 @@ tests/notebooks/data/
 .venv39/
 .venv37/
 .DS_Store
+.output
+.output*

CHANGELOG.md

@@ -38,6 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   `$ dffml service dev create blank mypackage`
 - `is_trained` flag to all models
 - Dynamic `location` property to `Model` baseclass.
+- Pandas dataframe source can read from Excel files
 ### Changed
 - Calls to hashlib now go through helper functions
 - Build docs using `dffml service dev docs`
@@ -56,6 +57,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Record object key properties are now always strings
 - High level functions (`train()`, etc.) now work on existing open contexts
 - Issue of download progress being logged only on first download
+- Operation Implementations now get instantiated with an instance of their
+  config object if they have one rather than an empty BaseConfig object if the
+  dataflow does not provide a config for the operation.
 ### Removed
 - `dffml/skel/common/REPLACE_IMPORT_PACKAGE_NAME/version.py`
 

configloader/jsonschema/.coveragerc

@@ -0,0 +1,13 @@
+[run]
+source =
+    dffml_config_jsonschema
+    tests
+branch = True
+
+[report]
+exclude_lines =
+    no cov
+    no qa
+    noqa
+    pragma: no cover
+    if __name__ == .__main__.:

configloader/jsonschema/.gitignore

@@ -0,0 +1,21 @@
+*.log
+*.pyc
+.cache/
+.coverage
+.idea/
+.vscode/
+*.egg-info/
+build/
+dist/
+docs/build/
+venv/
+wheelhouse/
+*.egss
+.mypy_cache/
+*.swp
+.venv/
+.eggs/
+*.modeldir
+*.db
+htmlcov/
+built_html_docs/

configloader/jsonschema/LICENSE

@@ -0,0 +1,21 @@
+Copyright (c) 2022 Intel
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

configloader/jsonschema/MANIFEST.in

@@ -0,0 +1,3 @@
+include README.rst
+include LICENSE
+recursive-include dffml_config_jsonschema *

configloader/jsonschema/README.rst

@@ -0,0 +1,22 @@
+DFFML JSON Schema Configloader
+==============================
+
+Used to export the JSON schema associated with an object.
+
+Install from pip
+
+.. code-block:: console
+
+    $ pip install dffml-config-jsonschema
+
+Usage
+
+.. code-block:: console
+
+    $ dffml service dev export -configloader jsonschema example:mydict
+
+Testing
+
+.. code-block:: console
+
+    $ python -m unittest discover -v

configloader/jsonschema/dffml_config_jsonschema/configloader.py

@@ -0,0 +1,34 @@
+"""
+Description of what this config does
+"""
+import ast
+from typing import Dict
+
+from dffml.util.entrypoint import entrypoint
+from dffml.util.cli.arg import Arg
+from dffml.base import BaseConfig
+from dffml.configloader.configloader import (
+    BaseConfigLoaderContext,
+    BaseConfigLoader,
+)
+
+
+class JSONSchemaConfigLoaderContext(BaseConfigLoaderContext):
+    async def loadb(self, resource: bytes) -> Dict:
+        return ast.literal_eval(resource.decode())
+
+    async def dumpb(self, resource: Dict) -> bytes:
+        return repr(resource).encode()
+
+
+@entrypoint("jsonschema")
+class JSONSchemaConfigLoader(BaseConfigLoader):
+    CONTEXT = JSONSchemaConfigLoaderContext
+
+    @classmethod
+    def args(cls, args, *above) -> Dict[str, Arg]:
+        return args
+
+    @classmethod
+    def config(cls, config, *above) -> BaseConfig:
+        return BaseConfig()

configloader/jsonschema/dffml_config_jsonschema/version.py

@@ -0,0 +1 @@
+VERSION = "0.0.1"

configloader/jsonschema/entry_points.txt

@@ -0,0 +1,2 @@
+[dffml.configloader]
+jsonschema = dffml_config_jsonschema.configloader:JSONSchemaConfigLoader

configloader/jsonschema/pyproject.toml

@@ -0,0 +1,20 @@
+requires = ["setuptools>=44", "wheel", "setuptools_scm[toml]>=3.4.3"]
+build-backend = "setuptools.build_meta"
+
+[tool.black]
+exclude = '''
+(
+  /(
+      \.eggs         # exclude a few common directories in the
+    | \.git          # root of the project
+    | \.hg
+    | \.mypy_cache
+    | \.tox
+    | \.venv
+    | _build
+    | buck-out
+    | build
+    | dist
+  )
+)
+'''

configloader/jsonschema/setup.cfg

@@ -0,0 +1,40 @@
+[metadata]
+name = dffml-config-jsonschema
+description = DFFML config dffml-config-jsonschema
+version = attr: dffml_config_jsonschema.version.VERSION
+long_description = file: README.rst
+author = Unknown
+author_email = [email protected]
+maintainer = Unknown
+maintainer_email = [email protected]
+url = https://github.com/coder/dffml-config-jsonschema
+license = MIT
+keywords = dffml
+classifiers =
+    Development Status :: 3 - Alpha
+    Intended Audience :: Developers
+    License :: OSI Approved :: MIT License
+    Natural Language :: English
+    Operating System :: OS Independent
+    Programming Language :: Python :: 3 :: Only
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: Implementation :: CPython
+    Programming Language :: Python :: Implementation :: PyPy
+
+[options]
+zip_safe = False
+include_package_data = True
+packages = find:
+entry_points = file: entry_points.txt
+install_requires =
+    dffml>=0.4.0
+    pydantic>=1.10.2
+
+[options.extras_require]
+dev =
+    coverage
+    codecov
+    sphinx
+    twine
+    black==19.10b0
+    importlib_metadata>=4.8.1;python_version<"3.8"

configloader/jsonschema/setup.py

@@ -0,0 +1,8 @@
+import sys
+import site
+import setuptools
+
+# See https://github.com/pypa/pip/issues/7953
+site.ENABLE_USER_SITE = "--user" in sys.argv[1:]
+
+setuptools.setup()