diff --git a/.github/workflows/ci-code-style.yml b/.github/workflows/ci-code-style.yml index 83bddb21154..5339b32bf80 100644 --- a/.github/workflows/ci-code-style.yml +++ b/.github/workflows/ci-code-style.yml @@ -20,10 +20,10 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Install Tox + - name: Install Poetry run: | python -m pip install --upgrade pip - python -m pip install tox + python -m pip install poetry - name: Run code checks run: | - tox -e check + make check diff --git a/.github/workflows/ci-config-functions.yml b/.github/workflows/ci-config-functions.yml index 021ee3a4a9c..25230b217f4 100644 --- a/.github/workflows/ci-config-functions.yml +++ b/.github/workflows/ci-config-functions.yml @@ -9,16 +9,14 @@ on: - "kiwi/config/functions.sh" - ".github/workflows/**" - "test/scripts/**" - - "tox.ini" - - "*requirements.txt" + - "pyproject.toml" pull_request: paths: # copy-pasta from the above because workflows don't support anchors :-( - "kiwi/config/functions.sh" - ".github/workflows/**" - "test/scripts/**" - - "tox.ini" - - "*requirements.txt" + - "pyproject.toml" jobs: scripts_integration_tests: @@ -28,14 +26,13 @@ jobs: fail-fast: false matrix: container_runtime: - - DOCKER - PODMAN steps: - uses: actions/cache@v3 with: path: | ~/.cache/pip - .tox/ + ~/.cache/pypoetry key: pip-${{ hashFiles('.virtualenv.dev-requirements.txt') }}-${{ hashFiles('.virtualenv.requirements.txt') }} restore-keys: | -pip- @@ -44,10 +41,12 @@ jobs: uses: actions/checkout@v3 - name: set up Python uses: actions/setup-python@v4 - - name: Install Tox - run: sudo apt update && sudo apt install tox + - name: Install Poetry + run: | + sudo python -m pip install --upgrade pip + sudo python -m pip install poetry - name: Run the scripts integration tests - run: sudo tox -e scripts + run: sudo make test_scripts env: CONTAINER_RUNTIME: ${{ matrix.container_runtime }} diff --git a/.github/workflows/ci-documentation.yml b/.github/workflows/ci-documentation.yml index 266263f6bfd..8d3a0811c47 100644 --- a/.github/workflows/ci-documentation.yml +++ b/.github/workflows/ci-documentation.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: [3.11] steps: - uses: actions/checkout@v3 @@ -20,9 +20,10 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Install Tox + - name: Install Poetry run: | - sudo apt update && sudo apt install tox + python -m pip install --upgrade pip + python -m pip install poetry - name: Run sphinx build run: | - tox -e doc + make docs diff --git a/.github/workflows/ci-publish-pages.yml b/.github/workflows/ci-publish-pages.yml index 5ea9440bca6..69b2d77c23a 100644 --- a/.github/workflows/ci-publish-pages.yml +++ b/.github/workflows/ci-publish-pages.yml @@ -22,10 +22,10 @@ jobs: - name: Install run: | python -m pip install --upgrade pip - python -m pip install tox + python -m pip install poetry - name: Render run: | - tox -e doc_gh_pages + make prepare_for_docs - name: Deploy uses: peaceiris/actions-gh-pages@v3 with: diff --git a/.github/workflows/ci-publish-to-pypi.yml b/.github/workflows/ci-publish-to-pypi.yml index ebee7bc37f5..d21679e8c88 100644 --- a/.github/workflows/ci-publish-to-pypi.yml +++ b/.github/workflows/ci-publish-to-pypi.yml @@ -28,9 +28,9 @@ jobs: - name: Install run: | python -m pip install --upgrade pip - python -m pip install tox + python -m pip install poetry - name: Prepare run: | - tox -e doc_man,release + make prepare_for_pypi - name: Publish uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/ci-units-types.yml b/.github/workflows/ci-units-types.yml index b596325ff00..960e7533650 100644 --- a/.github/workflows/ci-units-types.yml +++ b/.github/workflows/ci-units-types.yml @@ -20,11 +20,11 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Install Tox + - name: Install Poetry run: | python -m pip install --upgrade pip - python -m pip install tox + python -m pip install poetry - name: Run unit and type tests - run: tox -e "unit_py${PY_VER/./_}" -- -n auto + run: make test env: PY_VER: ${{ matrix.python-version }} diff --git a/.virtualenv.dev-requirements.txt b/.virtualenv.dev-requirements.txt deleted file mode 100644 index ce0edc73248..00000000000 --- a/.virtualenv.dev-requirements.txt +++ /dev/null @@ -1,59 +0,0 @@ -# -# Requirement files for Python Virtual Environments -# - --r .virtualenv.requirements.txt - -# poetry for building/installing/upload -poetry - -# virtualenv-based automation of test activities -tox - -# python unit testing framework -pytest -pytest-cov -pytest-xdist - -# Rolling backport of unittest.mock for all Pythons -mock - -# Version-bump your software with a single command! -bumpversion - -# A built-package format for Python -wheel - -# Python style guide checker -flake8 -mypy -types-requests -types-PyYAML -types-mock -types-simplejson - -# Generate Data Structures from XML Schema -# http://pythonhosted.org/generateDS -generateDS==2.29.24 - -# for building documentation -sphinx>=5.0.0 - -git+https://github.com/openSUSE/rstxml2docbook.git@feature/kiwi - -sphinx_rtd_theme -sphinxcontrib-spelling -pyenchant - -# for github pages deployment tasks -travis-sphinx -ghp-import - -# for helper tools -python-dateutil - -# for release -twine - -# for scripts testing -pytest-container diff --git a/.virtualenv.requirements.txt b/.virtualenv.requirements.txt deleted file mode 100644 index 53440a1043e..00000000000 --- a/.virtualenv.requirements.txt +++ /dev/null @@ -1,35 +0,0 @@ -# -# Requirement files for Python Virtual Environments -# - -# Shell interface for docopt, the command-line interface description language -docopt-ng - -# Powerful and Pythonic XML processing library -lxml - -# YAML file parsing -PyYAML - -# TOML file parsing -toml - -# Network request/response library -requests - -# markup support -anymarkup-core -xmltodict - -# provides pkg_resources -setuptools - -# json support that also knows about tuples -simplejson - -# TOML file parsing -toml - -# markup support -anymarkup-core -xmltodict diff --git a/Makefile b/Makefile index af6e51e0f50..43d1e3b2fa0 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,7 @@ docdir = /usr/share/doc/packages python_version = 3 python_lookup_name = python$(python_version) python = $(shell which $(python_lookup_name)) +sc_disable = SC1091,SC1090,SC2001,SC2174,SC1117,SC2048,SC2004 LC = LC_MESSAGES @@ -43,9 +44,6 @@ install: install -d -m 755 ${buildroot}usr/share/kiwi cp -a helper/xsl_to_v74 ${buildroot}usr/share/kiwi/ -tox: - tox -- "-n 5" - kiwi/schema/kiwi.rng: kiwi/schema/kiwi.rnc # whenever the schema is changed this target will convert # the short form of the RelaxNG schema to the format used @@ -92,16 +90,48 @@ clean_git_attributes: # for details on when this target is called see setup.py git checkout kiwi/version.py -build: clean +setup: poetry install --all-extras - bash -c 'shellcheck -e SC1091,SC1090,SC2001,SC2174,SC1117,SC2048,SC2004 dracut/modules.d/*/*.sh -s bash' - bash -c 'shellcheck -e SC1091,SC1090,SC2001,SC2174,SC1117,SC2048,SC2004 kiwi/config/functions.sh -s bash' - poetry run make -C doc man + +docs: setup + poetry run make -C doc man html + +docs_suse: setup + poetry run make -C doc xml + rm -rf doc/build/restxml + mv doc/build/xml doc/build/restxml + poetry run pip install \ + git+https://github.com/openSUSE/rstxml2docbook.git@feature/kiwi + poetry run bash -c 'pushd doc && rstxml2docbook \ + -v --no-split -o build/xml/book.xml build/restxml/index.xml' + bash -c 'mkdir -p doc/build/images/src/png && \ + cp -a doc/source/.images/* doc/build/images/src/png' + cp doc/DC-kiwi doc/build/ + bash -c 'pushd doc/build && daps -d DC-kiwi html' + +test_scripts: setup + poetry run bash -c \ + 'pip install pytest-container && pushd test/scripts && pytest -s -vv' + +check: setup + # shell code checks + bash -c 'shellcheck -e ${sc_disable} dracut/modules.d/*/*.sh -s bash' + bash -c 'shellcheck -e ${sc_disable} kiwi/config/functions.sh -s bash' + # python flake tests poetry run flake8 --statistics -j auto --count kiwi poetry run flake8 --statistics -j auto --count test/unit poetry run flake8 --statistics -j auto --count test/scripts + +test: setup + # python static code checks poetry run mypy kiwi - poetry run bash -c 'pushd test/unit && pytest -n 5 --doctest-modules --no-cov-on-fail --cov=kiwi --cov-report=term-missing --cov-fail-under=100 --cov-config .coveragerc' + # unit tests + poetry run bash -c 'pushd test/unit && pytest -n 5 \ + --doctest-modules --no-cov-on-fail --cov=kiwi \ + --cov-report=term-missing --cov-fail-under=100 \ + --cov-config .coveragerc' + +build: clean check test # build the sdist source tarball poetry build --format=sdist # provide rpm source tarball @@ -124,9 +154,21 @@ build: clean # provide patches cp package/*.patch dist -pypi: clean tox +prepare_for_pypi: clean setup + # documentation render and tests + poetry run make -C doc man + # sdist tarball, the actual publishing happens via the + # ci-publish-to-pypi.yml github action poetry build --format=sdist - poetry publish --repository=pypi + +prepare_for_docs: clean setup + # documentation man pages + poetry run make -C doc man + # documentation github pages, the actual publishing via + # the ci-publish-pages.yml github action + poetry run bash -c 'pushd doc && \ + travis-sphinx --outdir build_gh_pages build --nowarn --source ./source' + bash -c 'touch ./doc/build_gh_pages/.nojekyll' clean: clean_git_attributes rm -rf dist diff --git a/doc/source/troubleshooting/boxbuild_tweaks.rst b/doc/source/troubleshooting/boxbuild_tweaks.rst index e336cebae06..8003f684826 100644 --- a/doc/source/troubleshooting/boxbuild_tweaks.rst +++ b/doc/source/troubleshooting/boxbuild_tweaks.rst @@ -29,7 +29,7 @@ To do so, follow these steps: $ qemu-img resize Ubuntu-Box.x86_64-1.22.04-System-BuildBox.qcow2 +20G -2. When relaunching your `kiwi` box build, make sure you use `--no-snapshot` +2. When relaunching your `kiwi-ng` box build, make sure you use `--no-snapshot` and `--box-debug` options within your build command/script. Example: .. code:: bash diff --git a/pyproject.toml b/pyproject.toml index 524b5534ed4..dc0ff3b432f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,6 +108,8 @@ sphinx = ">=5.0.0" sphinx_rtd_theme = "*" sphinxcontrib-spelling = "*" pyenchant = "*" +travis-sphinx = "*" +ghp-import = "*" [build-system] requires = ["poetry-core>=1.2.0"] diff --git a/test/scripts/conftest.py b/test/scripts/conftest.py index b7cd4f285c5..b14c943fab6 100644 --- a/test/scripts/conftest.py +++ b/test/scripts/conftest.py @@ -8,48 +8,27 @@ CONTAINERS = [ DerivedContainer(base=url, containerfile=FUNCTIONS_COPY_CONTAINERFILE) for url in [ - "registry.opensuse.org/opensuse/leap:15.4", - "registry.opensuse.org/opensuse/leap:15.3", - "registry.opensuse.org/opensuse/leap:15.2", + "registry.opensuse.org/opensuse/leap:15.5", "registry.opensuse.org/opensuse/tumbleweed:latest", - "registry.suse.com/suse/sle15:15.4", - "registry.suse.com/suse/sle15:15.3", - "registry.suse.com/suse/sle15:15.2", - "registry.suse.com/suse/sle15:15.1", - "registry.suse.com/suse/sles12sp5:latest", - "quay.io/centos/centos:stream8", - "quay.io/centos/centos:stream9", + "registry.fedoraproject.org/fedora:latest" ] ] ( - LEAP_15_4, - LEAP_15_3, - LEAP_15_2, + LEAP_15_5, TUMBLEWEED, - SLE_15_SP4, - SLE_15_SP3, - SLE_15_SP2, - SLE_15_SP1, - SLE_12_SP5, - CENTOS_STREAM_8, - CENTOS_STREAM_9, + FEDORA ) = CONTAINERS CONTAINERS_WITH_ZYPPER = [ - LEAP_15_4, - LEAP_15_3, - LEAP_15_2, - TUMBLEWEED, - SLE_15_SP4, - SLE_15_SP3, + LEAP_15_5, + TUMBLEWEED ] -CONTAINERS_WITH_YUM = [ - CENTOS_STREAM_8, - CENTOS_STREAM_9, +CONTAINERS_WITH_DNF = [ + FEDORA ] diff --git a/test/scripts/test_baseVagrantSetup.py b/test/scripts/test_baseVagrantSetup.py index d0c20986851..36db52cbbb8 100644 --- a/test/scripts/test_baseVagrantSetup.py +++ b/test/scripts/test_baseVagrantSetup.py @@ -1,7 +1,7 @@ from pytest_container.container import DerivedContainer from .conftest import ( - CONTAINERS_WITH_YUM, CONTAINERS_WITH_ZYPPER, + CONTAINERS_WITH_DNF ) import pytest @@ -19,18 +19,20 @@ ) +DNF_IN_CMD_CONTAINERFILE = ( + """RUN dnf -y install openssh-server && /usr/libexec/openssh/sshd-keygen ed25519 +""" + VAGRANT_SETUP_CONTAINERFILE +) + + @pytest.mark.parametrize( "container_per_test", [ DerivedContainer(base=cont, containerfile=ZYPPER_IN_CMD_CONTAINERFILE) for cont in CONTAINERS_WITH_ZYPPER ] + [ - DerivedContainer( - base=cont, - containerfile="""RUN yum -y install openssh-server sudo && /usr/libexec/openssh/sshd-keygen ed25519 -""" + VAGRANT_SETUP_CONTAINERFILE, - ) - for cont in CONTAINERS_WITH_YUM + DerivedContainer(base=cont, containerfile=DNF_IN_CMD_CONTAINERFILE) + for cont in CONTAINERS_WITH_DNF ], indirect=["container_per_test"], ) diff --git a/test/scripts/test_suseSetupProduct.py b/test/scripts/test_suseSetupProduct.py index a5f5a7d5801..366b67b81c9 100644 --- a/test/scripts/test_suseSetupProduct.py +++ b/test/scripts/test_suseSetupProduct.py @@ -1,14 +1,7 @@ import pytest from .conftest import ( - LEAP_15_2, - LEAP_15_3, - LEAP_15_4, - SLE_12_SP5, - SLE_15_SP1, - SLE_15_SP2, - SLE_15_SP3, - SLE_15_SP4, + LEAP_15_5, TUMBLEWEED, ) @@ -30,18 +23,11 @@ def test_does_nothing_when_product_correct(auto_container_per_test): "container_per_test,product_name", ( (TUMBLEWEED, "openSUSE.prod"), - (LEAP_15_2, "openSUSE.prod"), - (LEAP_15_3, "Leap.prod"), - (LEAP_15_4, "Leap.prod"), - (SLE_15_SP4, "SLES.prod"), - (SLE_15_SP3, "SLES.prod"), - (SLE_15_SP2, "SLES.prod"), - (SLE_15_SP1, "SLES.prod"), - (SLE_12_SP5, "SLES.prod"), + (LEAP_15_5, "Leap.prod"), ), indirect=["container_per_test"], ) -def test_sets_baseproduct_from_etc_os_relesae(container_per_test, product_name): +def test_sets_baseproduct_from_etc_os_release(container_per_test, product_name): assert not container_per_test.connection.file("/etc/SuSE-brand").exists container_per_test.connection.run_expect([0], "rm /etc/products.d/baseproduct") diff --git a/test/unit/path_test.py b/test/unit/path_test.py index 4246d78b5fd..05c31fd080f 100644 --- a/test/unit/path_test.py +++ b/test/unit/path_test.py @@ -93,10 +93,10 @@ def test_which(self, mock_exists, mock_env, mock_access): def test_which_with_real_data(self, pytestconfig: pytest.Config): assert Path.which( - 'tox.ini', + 'pyproject.toml', custom_env={'PATH': str(pytestconfig.rootpath)}, access_mode=os.F_OK - ) == str(pytestconfig.rootpath / "tox.ini") + ) == str(pytestconfig.rootpath / "pyproject.toml") assert Path.which( '__init__.py', diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 1e282838ad1..00000000000 --- a/tox.ini +++ /dev/null @@ -1,265 +0,0 @@ -# Tox configuration file -# -# For more information, see https://tox.readthedocs.org -# -# Run it with -# a) all targets -# $ tox -# -# b) with specific targets (build only documentation): -# $ tox -e doc -# - -[tox] -minversion = 3.3.0 -isolated_build = True -skip_missing_interpreters = True -skipsdist = True -envlist = - check, - unit_py3_13, - unit_py3_12, - unit_py3_11, - unit_py3_10, - unit_py3_9, - packagedoc - - -[testenv] -description = - {unit_py3_9,unit_py3_10,unit_py3_11,unit_py3_12,unit_py3_13}: Unit Test run with basepython set to {basepython} - devel: Test KIWI -allowlist_externals = - bash - travis-sphinx - rstxml2docbook - cp - make - rm - mv - flake8 - poetry - python - pytest -basepython = - unit_py3_13: python3.13 - unit_py3_12: python3.12 - unit_py3_11: python3.11 - unit_py3_10: python3.10 - unit_py3_9: python3.9 - release: python3.10 - check: python3 - devel: python3 - packagedoc: python3 - doc: python3 -passenv = - * -usedevelop = True -deps = - -r.virtualenv.dev-requirements.txt - - -# Test run with basepython set to 3.9 -[testenv:unit_py3_9] -setenv = - PYTHONPATH={toxinidir}/test -changedir=test/unit -commands = - {[testenv:mypy]commands} - {[testenv:unit]commands} - - -# Test run with basepython set to 3.10 -[testenv:unit_py3_10] -setenv = - PYTHONPATH={toxinidir}/test -changedir=test/unit -commands = - {[testenv:mypy]commands} - {[testenv:unit]commands} - - -# Test run with basepython set to 3.11 -[testenv:unit_py3_11] -setenv = - PYTHONPATH={toxinidir}/test -changedir=test/unit -commands = - {[testenv:mypy]commands} - {[testenv:unit]commands} - - -# Test run with basepython set to 3.12 -[testenv:unit_py3_12] -setenv = - PYTHONPATH={toxinidir}/test -changedir=test/unit -commands = - {[testenv:mypy]commands} - {[testenv:unit]commands} - -# Test run with basepython set to 3.13 -[testenv:unit_py3_13] -setenv = - PYTHONPATH={toxinidir}/test -changedir=test/unit -commands = - {[testenv:mypy]commands} - {[testenv:unit]commands} - - -[testenv:mypy] -description = Static Type Checking Base -skip_install = True -usedevelop = True -setenv = - PYTHONUNBUFFERED=yes - WITH_COVERAGE=yes -passenv = - * -deps = {[testenv]deps} -changedir=test/unit -commands = - bash -c 'cd ../../ && mypy kiwi' - - -[testenv:unit] -description = Unit Test Base -skip_install = True -usedevelop = True -setenv = - PYTHONUNBUFFERED=yes - WITH_COVERAGE=yes -passenv = - * -deps = {[testenv]deps} -changedir=test/unit -commands = - bash -c 'cd ../../ && poetry install --all-extras' - bash -c 'cd ../../ && mypy kiwi' - pytest --doctest-modules --no-cov-on-fail --cov=kiwi \ - --cov-report=term-missing --cov-fail-under=100 \ - --cov-config .coveragerc {posargs} - - -[testenv:doc] -description = Documentation build suitable for local review -skip_install = True -usedevelop = True -deps = {[testenv]deps} -changedir=doc -commands = - {[testenv:doc.html]commands} - {[testenv:doc.man]commands} - - -[testenv:doc_man] -description = Documentation build run suitable for PyPi release -skip_install = True -usedevelop = True -deps = {[testenv]deps} -changedir=doc -commands = - {[testenv:doc.man]commands} - - -[testenv:packagedoc] -description = Documentation build run suitable for doc deployment in package(rpm) -skip_install = True -usedevelop = True -deps = {[testenv]deps} -changedir=doc -commands = - {[testenv:doc.man]commands} - - -[testenv:doc_gh_pages] -description = Documentation build suitable for doc deployment to github pages -skip_install = True -usedevelop = True -deps = {[testenv:doc]deps} -changedir=doc -commands = - {[testenv:doc.man]commands} - travis-sphinx --outdir build_gh_pages build --nowarn --source ./source - bash -c 'touch ./build_gh_pages/.nojekyll' - - -[testenv:doc_suse] -description = Documentation build suitable for SUSE documentation -skip_install = True -usedevelop = True -deps = {[testenv:doc]deps} -changedir=doc -commands = - {[testenv:doc.xml]commands} - rstxml2docbook -v --no-split -o build/xml/book.xml build/restxml/index.xml - bash -c 'mkdir -p build/images/src/png && cp -a source/.images/* build/images/src/png' - cp DC-kiwi build/ - bash -c 'cd build && daps -d DC-kiwi html' - - -[testenv:doc.html] -description = Documentation build html result -skip_install = True -deps = {[testenv:doc]deps} -changedir=doc -commands = - make html - - -[testenv:doc.xml] -description = Documentation build xml result -skip_install = True -deps = {[testenv:doc]deps} -changedir=doc -commands = - make xml - rm -rf build/restxml - mv build/xml build/restxml - - -[testenv:doc.man] -description = Documentation build man pages -skip_install = True -deps = {[testenv:doc]deps} -changedir=doc -commands = - make man - - -[testenv:check] -description = Source code quality/integrity check -deps = {[testenv]deps} -skip_install = True -usedevelop = True -commands = - flake8 --statistics -j auto --count {toxinidir}/kiwi - flake8 --statistics -j auto --count {toxinidir}/test/unit - flake8 --statistics -j auto --count {toxinidir}/test/scripts - bash -c 'shellcheck -e SC1091,SC1090,SC2001,SC2174,SC1117,SC2048,SC2004 {toxinidir}/dracut/modules.d/*/*.sh -s bash' - bash -c 'shellcheck -e SC1091,SC1090,SC2001,SC2174,SC1117,SC2048,SC2004 {toxinidir}/kiwi/config/functions.sh -s bash' - - -# PyPi upload -[testenv:release] -deps = {[testenv]deps} -skip_install = True -usedevelop = True -commands = - poetry build --format=sdist - - -[testenv:scripts] -description = Integration tests of the builtin shell scripts -deps = - {[testenv]deps} -changedir=test/scripts -commands = - pytest -s -vv {posargs} - - -[testenv:devel] -passenv = * -allowlist_externals = * -commands = {posargs} []