Merge branch 'dev'

.github/workflows/cicd.yml

@@ -12,16 +12,16 @@ jobs:
       matrix:
         python-version:
         # This list must be kept in sync with setup.py.
-        - "3.7"
         - "3.8"
         - "3.9"
         - "3.10"
-        - "3.11"  # Reference version used to run integration tests;
+        - "3.11"
+        - "3.12"  # Reference version used to run integration tests;
                   # to keep in sync with 'if: ' statements in the rest
                   # of this file.
-        - "pypy3.9"  # To keep in sync with other pypy releases referenced in the
-                     # "Docker image tests" and "Push Docker image to Docker Hub"
-                     # sections of this file.
+        - "pypy3.10"  # To keep in sync with other pypy releases referenced in the
+                      # "Docker image tests" and "Push Docker image to Docker Hub"
+                      # sections of this file.
 
     steps:
       - uses: actions/checkout@v2
@@ -46,7 +46,7 @@ jobs:
         run: flake8
 
       - name: Test CLI
-        if: ${{ matrix.python-version == '3.11' }}
+        if: ${{ matrix.python-version == '3.12' }}
         run: ./tests/cli
         env:
           SECRET_PEERINGDB_API_KEY: ${{ secrets.SECRET_PEERINGDB_API_KEY }}
@@ -58,21 +58,21 @@ jobs:
           SECRET_PEERINGDB_API_KEY: ${{ secrets.SECRET_PEERINGDB_API_KEY }}
 
       - name: Config build tests
-        if: ${{ matrix.python-version != '3.11' }}
+        if: ${{ matrix.python-version != '3.12' }}
         run: BUILD_ONLY=1 pytest -vs -x tests/live_tests/
         env:
           TRAVIS: 1
           SECRET_PEERINGDB_API_KEY: ${{ secrets.SECRET_PEERINGDB_API_KEY }}
 
       - name: Test external resources
-        if: ${{ matrix.python-version == '3.11' }}
+        if: ${{ matrix.python-version == '3.12' }}
         run: pytest -vs tests/external_resources/
         env:
           TRAVIS: 1
           SECRET_PEERINGDB_API_KEY: ${{ secrets.SECRET_PEERINGDB_API_KEY }}
 
       - name: Fetch Docker images used by tests
-        if: ${{ matrix.python-version == '3.11' }}
+        if: ${{ matrix.python-version == '3.12' }}
         run: |
           echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
           docker pull pierky/bird:1.6.8
@@ -87,7 +87,7 @@ jobs:
           DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Integration tests
-        if: ${{ matrix.python-version == '3.11' }}
+        if: ${{ matrix.python-version == '3.12' }}
         run: pytest -vs -x tests/live_tests/
         env:
           TRAVIS: 1
@@ -101,9 +101,9 @@ jobs:
     strategy:
       matrix:
         docker-image:
-        - base_image: python:3.11
+        - base_image: python:3.12
           target_image: latest
-        - base_image: pypy:3.9
+        - base_image: pypy:3.10
           target_image: latest-pypy3
 
     steps:
@@ -385,15 +385,17 @@ jobs:
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
-          python-version: 3.11
+          python-version: 3.12
 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          pip install --upgrade setuptools
 
       - name: Build and push the release
         run: >-
+          pip install --upgrade packaging
           python setup.py sdist
 
       # All the tags go to PyPI test
@@ -438,38 +440,38 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Push to Docker Hub (current release, CPython)
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           push: true
           file: ./docker/Dockerfile
           build-args: |
-            base_image=python:3.11
+            base_image=python:3.12
           tags: pierky/arouteserver:${{ steps.get_the_release.outputs.CURRENT_RELEASE }}
 
       - name: Push to Docker Hub (current release, PyPy)
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           push: true
           file: ./docker/Dockerfile
           build-args: |
-            base_image=pypy:3.9
+            base_image=pypy:3.10
           tags: pierky/arouteserver:${{ steps.get_the_release.outputs.CURRENT_RELEASE }}-pypy3
 
       - name: Push to Docker Hub (latest, CPython)
         # Only stable releases
         if: steps.is_stable_release.outputs.IS_STABLE_RELEASE == 'True'
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           push: true
           file: ./docker/Dockerfile
           build-args: |
-            base_image=python:3.11
+            base_image=python:3.12
           tags: pierky/arouteserver:latest
 
       - name: Push to Docker Hub (latest, PyPy)
         # Only stable releases
         if: steps.is_stable_release.outputs.IS_STABLE_RELEASE == 'True'
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           push: true
           file: ./docker/Dockerfile

.github/workflows/test_external_resources.yml

@@ -12,7 +12,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ["3.11"]
+        python-version: ["3.12"]
 
     steps:
       - uses: actions/checkout@v2

.readthedocs.yaml

@@ -8,7 +8,7 @@ version: 2
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.11"
+    python: "3.12"
 
 formats:
   - epub

CHANGES.rst

@@ -3,6 +3,54 @@ Change log
 
 .. note:: **Upgrade notes**: after upgrading, run the ``arouteserver setup-templates`` command to sync the local templates with those distributed with the new version. More details on the `Upgrading <https://arouteserver.readthedocs.io/en/latest/INSTALLATION.html#upgrading>`__ section of the documentation.
 
+1.23
+----
+
+- Fix: Remove second double string 'AS'.
+
+  Cosmetic: remove the double 'AS' prefix from the comments in AS-SET names in BIRD configurations.
+
+  See also `GitHub issue 132 <https://github.com/pierky/arouteserver/issues/132>`__.
+
+- Fix: parse Registro.br records gracefully.
+
+  In case of parsing issues in the records exported by the Registro.br dataset, the tool now logs a warning message and skips the record.
+  The configuration generation no longer fails in this case.
+
+  See also `GitHub PR 135 <https://github.com/pierky/arouteserver/pull/135>`__, `GitHub issue 138 <https://github.com/pierky/arouteserver/issues/138>`__.
+
+- Fix: just log failed Registro.br record content in case of issues.
+
+  When a parsing error occurs in the Registro.br dataset, the tool now logs the content of the record that caused the issue, instead of the whole file.
+
+  See also `GitHub issue 138 <https://github.com/pierky/arouteserver/issues/138>`__.
+
+- Fix: ``clients-from-peeringdb``, reduce the number of PeeringDB API queries per minute.
+
+  To avoid hitting the PeeringDB API rate limit, a delay is introduced between the queries performed to fetch clients' details when the ``clients-from-peeringdb`` command is used.
+
+  See also `GitHub issue 139 <https://github.com/pierky/arouteserver/issues/139>`__.
+
+- Improvement: avoid warnings when ``bgpq4`` prints ``Invalid AS number:`` errors.
+
+  When ``Invalid AS number:`` lines are printed by ``bgpq4``, avoid generating the ``bgpq4 succeeded but an error was printed`` warnings.
+
+- Improvement (Docker image): allow generation of clients.yml from Euro-IX files.
+
+  A new environment variable is introduced (``EURO_IX_URL``) to instruct the Docker container to build the clients.yml file from the Euro-IX JSON export file.
+
+  See also `GitHub PR 136 <https://github.com/pierky/arouteserver/pull/136>`__.
+
+- Improvement (Docker image): allow use of custom templates.
+
+  A new environment variable is introduced (``TEMPLATES_DIR``) to instruct the Docker container to use custom templates for the configuration generation.
+
+- Tests are no longer performed for Python 3.7 (EoL).
+
+- Tests for Python 3.12 are introduced: that's the release used to perform the integration tests.
+
+- Docker images are built on top of Python 3.12 and PyPy 3.10, and use ``bgpq4`` version 1.12.
+
 1.22.1
 ------
 

docker/Dockerfile

@@ -1,4 +1,4 @@
-ARG base_image="python:3.11"
+ARG base_image="python:3.12"
 
 FROM ${base_image}
 
@@ -8,7 +8,7 @@ WORKDIR /arouteserver
 RUN mkdir /bgpq4 && \
     cd /bgpq4 && \
     git clone https://github.com/bgp/bgpq4.git ./ && \
-    git checkout tags/1.4 && \
+    git checkout tags/1.12 && \
     ./bootstrap && \
     ./configure && \
     make && \

docker/README.md

@@ -47,6 +47,7 @@ docker run \
     -e IP_VER=4 \
     -e DAEMON=bird \
     -e VERSION=1.6.8 \
+    -e SECRET_PEERINGDB_API_KEY="your secret PeeringDB API key" \
     pierky/arouteserver:latest
 ```
 
@@ -86,6 +87,17 @@ docker run \
     pierky/arouteserver:latest
 ```
 
+In case custom templates are used to generate the final configurations, it's possible to pass the `TEMPLATES_DIR` environment variable to the container and set it to a custom path (on the container side) where the local directory containing the customized files is mounted:
+
+```bash
+docker run \
+    ...
+    -e TEMPLATES_DIR="/custom_templates" \
+    -v /path/to/local/templates/directory:/custom_templates:ro
+```
+
+The local directory (*/path/to/local/templates/directory* in the example above) must have the same structure of the official *templates* directory (so, with sub-directories like *bird*, *openbgpd*, *html*, *md*, ...).
+
 ### Textual representation
 
 A [textual representation](https://arouteserver.readthedocs.io/en/latest/USAGE.html#textual-representation) of the route server's options and policies will be generated automatically before the route server configuration is generated if the directory `/root/arouteserver_html` exists on the container.

docker/run.sh

@@ -28,6 +28,9 @@ LOCAL_PREFIXES="${LOCAL_PREFIXES}"
 IP_VER="${IP_VER}"
 LOCAL_FILES="${LOCAL_FILES}"
 LOCAL_FILES_DIR="${LOCAL_FILES_DIR}"
+TEMPLATES_DIR="${TEMPLATES_DIR}"
+EURO_IX_URL="${EURO_IX_URL}"
+EURO_IX_IXP_ID="${EURO_IX_IXP_ID}"
 
 CLIENTS_FILE_PATH="/root/clients.yml"
 OUTPUT_DIR="/root/arouteserver_configs"
@@ -40,6 +43,13 @@ if [[ ! -e "${OUTPUT_DIR}" && ! -d "${OUTPUT_DIR}" ]]; then
           You can run the container passing the '-v PATH_OF_DIRECTORY_ON_THE_HOST:${OUTPUT_DIR}' argument."
 fi
 
+if [[ -n "${TEMPLATES_DIR}" ]]; then
+    if [[ ! -e "${TEMPLATES_DIR}" ]]; then
+        error "Couldn't find the directory ${TEMPLATES_DIR} on the container.\n
+               Please mount the local directory where the templates are stored using '-v PATH_OF_TEMPLATES_DIR_ON_THE_HOST:${TEMPLATES_DIR}:ro' argument."
+    fi
+fi
+
 TEST_WRITABLE_FILE="${OUTPUT_DIR}/.test_dir_is_writable"
 
 set +e
@@ -83,6 +93,14 @@ if [[ -z "${VERSION}" ]]; then
     error_envvar_not_set "VERSION"
 fi
 
+if [[ -n "${EURO_IX_URL}" ]]; then
+
+    echo ""
+    echo "Generating clients.yaml from EURO IX-F"
+    echo ""
+    arouteserver clients-from-euroix -o "${CLIENTS_FILE_PATH}" --url "${EURO_IX_URL}" "${EURO_IX_IXP_ID}"
+fi
+
 if [[ ! -e "${CLIENTS_FILE_PATH}" ]]; then
     error "Couldn't find the file ${CLIENTS_FILE_PATH} on the container.\n
            Please mount the local file where the list of clients is defined using '-v PATH_OF_CLIENTS_FILE_ON_THE_HOST:${CLIENTS_FILE_PATH}:ro' argument."
@@ -183,10 +201,17 @@ if [[ -n "${LOCAL_FILES}" ]]; then
     fi
 fi
 
+TEMPLATES_DIR_ARG=""
+
+if [[ -n "${TEMPLATES_DIR}" ]]; then
+    TEMPLATES_DIR_ARG="--templates-dir ${TEMPLATES_DIR}"
+fi
+
 OUTPUT_PATH="${OUTPUT_DIR}/${OUTPUT_FILE}"
 
 arouteserver \
     "${DAEMON}" \
+    ${TEMPLATES_DIR_ARG} \
     --target-version "${VERSION}" \
     ${IP_VER_ARG} \
     --clients "${CLIENTS_FILE_PATH}" \

docs/INSTALLATION.rst

@@ -32,7 +32,7 @@ Strongly suggested: setup a `Virtualenv <https://virtualenv.pypa.io/>`_.
 .. code:: bash
 
    # on Debian/Ubuntu:
-   sudo apt-get install python-pip python-virtualenv
+   sudo apt-get install python3-pip python3-virtualenv
 
    # on CentOS:
    sudo yum install epel-release
@@ -48,7 +48,7 @@ Strongly suggested: setup a `Virtualenv <https://virtualenv.pypa.io/>`_.
    source ~/.virtualenvs/arouteserver/bin/activate
 
    # install the program
-   pip install arouteserver
+   pip3 install arouteserver
 
 More: ``virtualenv`` `installation <https://virtualenv.pypa.io/en/latest/installation.html>`_ and `usage <https://virtualenv.pypa.io/en/latest/userguide.html>`_.
 

pierky/arouteserver/irrdb.py

@@ -162,18 +162,33 @@ def _run_cmd(self, cmd):
             proc.communicate()
             raise
 
+        if err is not None:
+            err = err.decode("utf-8").strip()
+
         if proc.returncode != 0:
             err_msg = "{} exit code is {}".format(self.bgpq, proc.returncode)
-            if err is not None and err.strip():
+            if err:
                 err_msg += ", stderr: {}".format(err)
             raise ValueError(err_msg)
 
-        if err is not None and err.strip():
-            logging.warning("{} succeeded but an error was "
-                            "printed when executing '{}': {}".format(
-                                self.bgpq,
-                                " ".join(cmd), err.strip()
-                            ))
+        if err:
+            # If an error was returned, remove any line containing
+            # "Invalid AS number:". This is printed to stderr when
+            # private ASNs are found in the output, but it's just
+            # confusing and not useful for the user.
+            err_lines = err.split("\n")
+            err = "\n".join([
+                line
+                for line in err_lines
+                if "Invalid AS number:" not in line
+            ])
+
+            if err:
+                logging.warning("{} succeeded but an error was "
+                                "printed when executing '{}': {}".format(
+                                    self.bgpq,
+                                    " ".join(cmd), err
+                                ))
 
         return out