From d8d9c006a9ff7430c1f03d891dc006670c89eefb Mon Sep 17 00:00:00 2001 From: Lukas Holecek Date: Fri, 10 Nov 2023 09:53:57 +0100 Subject: [PATCH 1/2] Install latest dependencies from PyPI --- Dockerfile | 13 ++++--------- requirements.txt | 14 ++------------ resultsdb_frontend/controllers/main.py | 9 +-------- 3 files changed, 7 insertions(+), 29 deletions(-) diff --git a/Dockerfile b/Dockerfile index 693fb96..ae4ef82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,3 @@ -# This will produce an image to be used in Openshift -# Build should be triggered from repo root like: -# docker build -f Dockerfile --tag - FROM registry.fedoraproject.org/fedora-minimal:38 ARG GITHUB_SHA @@ -32,18 +28,17 @@ ENV \ COPY . /opt/app-root/src/resultsdb_frontend/ RUN microdnf -y install \ - findutils \ httpd \ mod_ssl \ python3-mod_wsgi \ python3-pip \ - python3-cachelib \ - python3-flask \ - python3-iso8601 \ python3-resultsdb_api \ rpm-build \ + && pip3 install --upgrade --upgrade-strategy eager \ + -r /opt/app-root/src/resultsdb_frontend/requirements.txt \ + && pip3 install --no-deps /opt/app-root/src/resultsdb_frontend \ + && microdnf -y remove python3-pip \ && microdnf -y clean all \ - && pip3 install /opt/app-root/src/resultsdb_frontend \ && install -d /usr/share/resultsdb_frontend/conf \ && install -p -m 0644 \ /opt/app-root/src/resultsdb_frontend/conf/resultsdb_frontend.conf \ diff --git a/requirements.txt b/requirements.txt index 27eda8a..12565f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,4 @@ -# This is a list of pypi packages to be installed into virtualenv. Alternatively, -# you can install these as RPMs instead of pypi packages. See the dependecies -# with: -# $ rpmspec -q --requires resultsdb.spec -# $ rpmspec -q --buildrequires resultsdb.spec - -# A note for maintainers: Please keep this list in sync and in the same order -# as the spec file. - -Flask >= 0.10.1 +Flask >= 2.2.5 +cachelib iso8601 >= 0.1.11 resultsdb_api >= 2.0 -six >= 1.10.0 -cachelib diff --git a/resultsdb_frontend/controllers/main.py b/resultsdb_frontend/controllers/main.py index f2053f4..136dd5e 100644 --- a/resultsdb_frontend/controllers/main.py +++ b/resultsdb_frontend/controllers/main.py @@ -34,18 +34,11 @@ CACHE = SimpleCache() CACHE_TIMEOUT = 60 - -RDB_API = None +RDB_API = ResultsDBapi(app.config["RDB_URL"]) main = Blueprint("main", __name__) -@app.before_first_request -def before_first_request(): - global RDB_API - RDB_API = ResultsDBapi(app.config["RDB_URL"]) - - @main.route("/") @main.route("/index") def index(): From 562b94f664fedd69be188d1fb252c94ca64559dc Mon Sep 17 00:00:00 2001 From: Lukas Holecek Date: Fri, 10 Nov 2023 10:20:40 +0100 Subject: [PATCH 2/2] Drop unmaintained ResultsDB package --- Dockerfile | 1 - requirements.txt | 2 +- resultsdb_frontend/config.py | 3 ++ resultsdb_frontend/controllers/main.py | 27 +++-------- resultsdb_frontend/requests_session.py | 63 ++++++++++++++++++++++++++ resultsdb_frontend/resultsdb_api.py | 38 ++++++++++++++++ 6 files changed, 111 insertions(+), 23 deletions(-) create mode 100644 resultsdb_frontend/requests_session.py create mode 100644 resultsdb_frontend/resultsdb_api.py diff --git a/Dockerfile b/Dockerfile index ae4ef82..306d7b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,6 @@ RUN microdnf -y install \ mod_ssl \ python3-mod_wsgi \ python3-pip \ - python3-resultsdb_api \ rpm-build \ && pip3 install --upgrade --upgrade-strategy eager \ -r /opt/app-root/src/resultsdb_frontend/requirements.txt \ diff --git a/requirements.txt b/requirements.txt index 12565f8..86e18fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ Flask >= 2.2.5 cachelib iso8601 >= 0.1.11 -resultsdb_api >= 2.0 +requests diff --git a/resultsdb_frontend/config.py b/resultsdb_frontend/config.py index 0b4739e..20f5cd9 100644 --- a/resultsdb_frontend/config.py +++ b/resultsdb_frontend/config.py @@ -38,6 +38,9 @@ class Config: FEDMENU_URL = "https://apps.fedoraproject.org/fedmenu" FEDMENU_DATA_URL = "https://apps.fedoraproject.org/js/data.js" + # Options for outbound HTTP requests made by python-requests + REQUESTS_TIMEOUT = (6.1, 15) + class ProductionConfig(Config): DEBUG = False diff --git a/resultsdb_frontend/controllers/main.py b/resultsdb_frontend/controllers/main.py index 136dd5e..1865594 100644 --- a/resultsdb_frontend/controllers/main.py +++ b/resultsdb_frontend/controllers/main.py @@ -28,9 +28,9 @@ request, url_for, ) -from resultsdb_api import ResultsDBapi, ResultsDBapiException from resultsdb_frontend import app +from resultsdb_frontend.resultsdb_api import ResultsDBapi CACHE = SimpleCache() CACHE_TIMEOUT = 60 @@ -73,19 +73,13 @@ def testcase_tokenizer(): @main.route("/groups") def groups(): - try: - groups = RDB_API.get_groups(**dict(request.args)) - except ResultsDBapiException as e: - return str(e) + groups = RDB_API.get_groups(**dict(request.args)) return render_template("groups.html", groups=groups) @main.route("/groups/") def group(group_id): - try: - group = RDB_API.get_group(group_id) - except ResultsDBapiException as e: - return str(e) + group = RDB_API.get_group(group_id) groups = dict(prev=None, next=None, data=[group]) return render_template("groups.html", groups=groups) @@ -93,10 +87,7 @@ def group(group_id): @main.route("/results") def results(): args = dict(request.args) - try: - results = RDB_API.get_results(**args) - except ResultsDBapiException as e: - return str(e) + results = RDB_API.get_results(**args) for result in results["data"]: result["groups"] = (len(result["groups"]), ",".join(result["groups"])) return render_template("results.html", results=results) @@ -104,10 +95,7 @@ def results(): @main.route("/results/") def result(result_id): - try: - result = RDB_API.get_result(id=result_id) - except ResultsDBapiException as e: - return str(e) + result = RDB_API.get_result(id=result_id) try: result["groups"] = (len(result["groups"]), ",".join(result["groups"])) except KeyError: @@ -124,9 +112,6 @@ def testcases(): @main.route("/testcases/") def testcase(testcase_name): - try: - tc = RDB_API.get_testcase(name=testcase_name) - except ResultsDBapiException as e: - return str(e) + tc = RDB_API.get_testcase(name=testcase_name) tcs = dict(prev=None, next=None, data=[tc]) return render_template("testcases.html", testcases=tcs) diff --git a/resultsdb_frontend/requests_session.py b/resultsdb_frontend/requests_session.py new file mode 100644 index 0000000..6e066db --- /dev/null +++ b/resultsdb_frontend/requests_session.py @@ -0,0 +1,63 @@ +import logging +from json import dumps + +import requests +from flask import current_app, has_app_context +from requests.adapters import HTTPAdapter +from requests.exceptions import ConnectionError, ConnectTimeout, RetryError +from urllib3.exceptions import ProxyError, SSLError +from urllib3.util.retry import Retry + +log = logging.getLogger(__name__) + + +class ErrorResponse(requests.Response): + def __init__(self, status_code, error_message, url): + super().__init__() + self.status_code = status_code + self._error_message = error_message + self.url = url + self.reason = error_message.encode() + + @property + def content(self): + return dumps({"message": self._error_message}).encode() + + +class RequestsSession(requests.Session): + def request(self, *args, **kwargs): # pylint:disable=arguments-differ + log.debug("Request: args=%r, kwargs=%r", args, kwargs) + + req_url = kwargs.get("url", args[1]) + + kwargs.setdefault("headers", {"Content-Type": "application/json"}) + if has_app_context(): + kwargs.setdefault("timeout", current_app.config["REQUESTS_TIMEOUT"]) + + try: + ret_val = super().request(*args, **kwargs) + except (ConnectTimeout, RetryError) as e: + ret_val = ErrorResponse(504, str(e), req_url) + except (ConnectionError, ProxyError, SSLError) as e: + ret_val = ErrorResponse(502, str(e), req_url) + + log.debug("Request finished: %r", ret_val) + return ret_val + + +def get_requests_session(): + """Get http(s) session for request processing.""" + + session = RequestsSession() + retry = Retry( + total=3, + read=3, + connect=3, + backoff_factor=1, + status_forcelist=(500, 502, 503, 504), + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount("http://", adapter) + session.mount("https://", adapter) + session.headers["User-Agent"] = "resultsdb_frontend" + return session diff --git a/resultsdb_frontend/resultsdb_api.py b/resultsdb_frontend/resultsdb_api.py new file mode 100644 index 0000000..7922175 --- /dev/null +++ b/resultsdb_frontend/resultsdb_api.py @@ -0,0 +1,38 @@ +from resultsdb_frontend.requests_session import get_requests_session + + +def _prepare_params(**kwargs): + return { + key: ",".join(str(v) for v in value) if isinstance(value, list) else str(value) + for key, value in kwargs.items() + if value is not None + } + + +class ResultsDBapi: + def __init__(self, api_url): + self.url = api_url.rstrip("/") + self.session = get_requests_session() + + def _get(self, api, **kwargs): + r = self.session.get(f"{self.url}{api}", **kwargs) + r.raise_for_status() + return r.json() + + def get_group(self, uuid): + return self._get(f"/groups/{uuid}") + + def get_groups(self, **kwargs): + return self._get("/groups", params=_prepare_params(**kwargs)) + + def get_result(self, id): + return self._get(f"/results/{id}") + + def get_results(self, **kwargs): + return self._get("/results", params=_prepare_params(**kwargs)) + + def get_testcase(self, name): + return self._get(f"/testcases/{name}") + + def get_testcases(self, **kwargs): + return self._get("/testcases", params=_prepare_params(**kwargs))