tests: Create a separate group for feature flag suites

.github/workflows/test-integrations-flags.yml

@@ -0,0 +1,163 @@
+# Do not edit this YAML file. This file is generated automatically by executing
+# python scripts/split_tox_gh_actions/split_tox_gh_actions.py
+# The template responsible for it is in
+# scripts/split_tox_gh_actions/templates/base.jinja
+name: Test Flags
+on:
+  push:
+    branches:
+      - master
+      - release/**
+      - potel-base
+  pull_request:
+# Cancel in progress workflows on pull_requests.
+# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+permissions:
+  contents: read
+env:
+  BUILD_CACHE_KEY: ${{ github.sha }}
+  CACHED_BUILD_PATHS: |
+    ${{ github.workspace }}/dist-serverless
+jobs:
+  test-flags-latest:
+    name: Flags (latest)
+    timeout-minutes: 30
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8","3.12","3.13"]
+        # python3.6 reached EOL and is no longer being supported on
+        # new versions of hosted runners on Github Actions
+        # ubuntu-20.04 is the last version that supported python3.6
+        # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877
+        os: [ubuntu-20.04]
+    steps:
+      - uses: actions/[email protected]
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          allow-prereleases: true
+      - name: Setup Test Env
+        run: |
+          pip install "coverage[toml]" tox
+      - name: Erase coverage
+        run: |
+          coverage erase
+      - name: Test launchdarkly latest
+        run: |
+          set -x # print commands that are executed
+          ./scripts/runtox.sh "py${{ matrix.python-version }}-launchdarkly-latest"
+      - name: Test openfeature latest
+        run: |
+          set -x # print commands that are executed
+          ./scripts/runtox.sh "py${{ matrix.python-version }}-openfeature-latest"
+      - name: Test unleash latest
+        run: |
+          set -x # print commands that are executed
+          ./scripts/runtox.sh "py${{ matrix.python-version }}-unleash-latest"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }}
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
+      - name: Generate coverage XML
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }}
+        run: |
+          coverage combine .coverage-sentry-*
+          coverage xml
+      - name: Upload coverage to Codecov
+        if: ${{ !cancelled() }}
+        uses: codecov/[email protected]
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage.xml
+          # make sure no plugins alter our coverage reports
+          plugin: noop
+          verbose: true
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() }}
+        uses: codecov/test-results-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: .junitxml
+          verbose: true
+  test-flags-pinned:
+    name: Flags (pinned)
+    timeout-minutes: 30
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8","3.12","3.13"]
+        # python3.6 reached EOL and is no longer being supported on
+        # new versions of hosted runners on Github Actions
+        # ubuntu-20.04 is the last version that supported python3.6
+        # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877
+        os: [ubuntu-20.04]
+    steps:
+      - uses: actions/[email protected]
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          allow-prereleases: true
+      - name: Setup Test Env
+        run: |
+          pip install "coverage[toml]" tox
+      - name: Erase coverage
+        run: |
+          coverage erase
+      - name: Test launchdarkly pinned
+        run: |
+          set -x # print commands that are executed
+          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-launchdarkly"
+      - name: Test openfeature pinned
+        run: |
+          set -x # print commands that are executed
+          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-openfeature"
+      - name: Test unleash pinned
+        run: |
+          set -x # print commands that are executed
+          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-unleash"
+      - name: Generate coverage XML (Python 3.6)
+        if: ${{ !cancelled() && matrix.python-version == '3.6' }}
+        run: |
+          export COVERAGE_RCFILE=.coveragerc36
+          coverage combine .coverage-sentry-*
+          coverage xml --ignore-errors
+      - name: Generate coverage XML
+        if: ${{ !cancelled() && matrix.python-version != '3.6' }}
+        run: |
+          coverage combine .coverage-sentry-*
+          coverage xml
+      - name: Upload coverage to Codecov
+        if: ${{ !cancelled() }}
+        uses: codecov/[email protected]
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage.xml
+          # make sure no plugins alter our coverage reports
+          plugin: noop
+          verbose: true
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() }}
+        uses: codecov/test-results-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: .junitxml
+          verbose: true
+  check_required_tests:
+    name: All pinned Flags tests passed
+    needs: test-flags-pinned
+    # Always run this, even if a dependent job failed
+    if: always()
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Check for failures
+        if: contains(needs.test-flags-pinned.result, 'failure') || contains(needs.test-flags-pinned.result, 'skipped')
+        run: |
+          echo "One of the dependent jobs has failed. You may need to re-run it." && exit 1

.github/workflows/test-integrations-misc.yml

@@ -47,18 +47,10 @@ jobs:
       - name: Erase coverage
         run: |
           coverage erase
-      - name: Test launchdarkly latest
-        run: |
-          set -x # print commands that are executed
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-launchdarkly-latest"
       - name: Test loguru latest
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh "py${{ matrix.python-version }}-loguru-latest"
-      - name: Test openfeature latest
-        run: |
-          set -x # print commands that are executed
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-openfeature-latest"
       - name: Test opentelemetry latest
         run: |
           set -x # print commands that are executed
@@ -79,10 +71,6 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh "py${{ matrix.python-version }}-typer-latest"
-      - name: Test unleash latest
-        run: |
-          set -x # print commands that are executed
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-unleash-latest"
       - name: Generate coverage XML (Python 3.6)
         if: ${{ !cancelled() && matrix.python-version == '3.6' }}
         run: |
@@ -135,18 +123,10 @@ jobs:
       - name: Erase coverage
         run: |
           coverage erase
-      - name: Test launchdarkly pinned
-        run: |
-          set -x # print commands that are executed
-          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-launchdarkly"
       - name: Test loguru pinned
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-loguru"
-      - name: Test openfeature pinned
-        run: |
-          set -x # print commands that are executed
-          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-openfeature"
       - name: Test opentelemetry pinned
         run: |
           set -x # print commands that are executed
@@ -167,10 +147,6 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-typer"
-      - name: Test unleash pinned
-        run: |
-          set -x # print commands that are executed
-          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-unleash"
       - name: Generate coverage XML (Python 3.6)
         if: ${{ !cancelled() && matrix.python-version == '3.6' }}
         run: |

scripts/split_tox_gh_actions/split_tox_gh_actions.py

@@ -76,16 +76,6 @@
         "cloud_resource_context",
         "gcp",
     ],
-    "Tasks": [
-        "arq",
-        "beam",
-        "celery",
-        "dramatiq",
-        "huey",
-        "ray",
-        "rq",
-        "spark",
-    ],
     "DBs": [
         "asyncpg",
         "clickhouse_driver",
@@ -94,12 +84,27 @@
         "redis_py_cluster_legacy",
         "sqlalchemy",
     ],
+    "Flags": [
+        "launchdarkly",
+        "openfeature",
+        "unleash",
+    ],
     "GraphQL": [
         "ariadne",
         "gql",
         "graphene",
         "strawberry",
     ],
+    "Tasks": [
+        "arq",
+        "beam",
+        "celery",
+        "dramatiq",
+        "huey",
+        "ray",
+        "rq",
+        "spark",
+    ],
     "Network": [
         "gevent",
         "grpc",
@@ -125,15 +130,12 @@
         "tornado",
     ],
     "Misc": [
-        "launchdarkly",
         "loguru",
-        "openfeature",
         "opentelemetry",
         "potel",
         "pure_eval",
         "trytond",
         "typer",
-        "unleash",
     ],
 }