Refactor/pick a faster json lib (#1156)

* use rapidjson instead

* use orjson instead

* fix tests; relocate some report properties

* fix pyproject.toml

* add newsfrag; some tweaks

* misc

- fix pytest warnings
- remove unused requirements* files
- adjust ci

.github/actions/pip-cache/action.yml

@@ -6,5 +6,5 @@ runs:
     - name: Restore Pip Cache
       uses: actions/cache@v4
       with:
-        path: ~/.cache/pip
-        key: ${{ hashFiles('setup.py') }}-${{ hashFiles('requirements.txt') }}
+        path: ${{ runner.os == 'Linux' && '~/.cache/pip' || '~\AppData\Local\pip\Cache' }}
+        key: ${{ runner.os }}-${{ hashFiles('pyproject.toml') }}

doc/en/getting_started.rst

@@ -76,7 +76,7 @@ Also find all our downloadable examples :ref:`here <download>`.
 
 Working with the source
 -----------------------
- 
+
 You will need a working python 3.7+ interrpreter preferably a venv, and for the interactive ui you need node installed. 
 We are using `doit <https://pydoit.org/contents.html>`_ as the taskrunner ``doit list`` can show all the commands.
 
@@ -85,10 +85,10 @@ We are using `doit <https://pydoit.org/contents.html>`_ as the taskrunner ``doit
     git clone https://github.com/morganstanley/testplan.git
     cd testplan
 
-    # install all dev requirements
-    pip install -r requirements-txt  # this install testplan in editable mode
+    # install testplan in editable mode & all dev requirements
+    pip install -e .
 
-    #build the interactive UI (if you do not like it is opening a browserwindow remove the `-o`)
+    # build the interactive UI (if you do not like it is opening a browserwindow remove the `-o`)
     doit build_ui -o
 
 Internal tests

doc/newsfragments/3147_changed.another_json_lib.rst

@@ -0,0 +1 @@
+Use a new JSON library ``orjson`` to improve performance when using Python 3.8 or later versions.

examples/ExecutionPools/Discover/test_plan.py

@@ -45,6 +45,7 @@ def main(plan):
 
 if __name__ == "__main__":
     res = main()
-    assert len(res.report.entries) == 5
+    if res.report.entries:
+        assert len(res.report.entries) == 5
     print("Exiting code: {}".format(res.exit_code))
     sys.exit(res.exit_code)

pyproject.toml

@@ -34,12 +34,9 @@
         "setuptools",
         "pytest",
         "pytest-mock",
-        "py",
         "psutil",
         "schema",
-        "pytz",
         "lxml",
-        "python-dateutil",
         "reportlab",
         "marshmallow",
         "termcolor",
@@ -66,7 +63,9 @@
         "typing_extensions",
         "dill",
         "gherkin-official==4.1.3",
-        "parse"
+        "parse",
+        "orjson; python_version>='3.8'",
+        "flask-orjson; python_version>='3.8'"
     ]
     requires-python = ">=3.7"
 

pytest.ini

@@ -1,10 +1,12 @@
 [pytest]
 filterwarnings =
     ignore::pytest.PytestWarning
-    ignore::DeprecationWarning:flask_restx.*:
-    ignore::DeprecationWarning:jinja2.*:
-    ignore::DeprecationWarning:jsonschema.*:
-    ignore::DeprecationWarning:marshmallow.*:
-    ignore::DeprecationWarning:werkzeug.*:
+    ignore:.*flask_restx.*:DeprecationWarning
+    ; jsonschema warning from flask_restx
+    ignore:.*jsonschema.*:DeprecationWarning
+    ignore:.*load_module.*:DeprecationWarning
+    ignore:.*LogMatcher.*:UserWarning
+    ; under most cases, included files are not hit
+    ignore:No data was collected:coverage.exceptions.CoverageWarning
 
 norecursedirs=tests/helpers

testplan/common/report/base.py

@@ -263,8 +263,8 @@ def __init__(
         uid: Optional[str] = None,
         entries: Optional[list] = None,
         parent_uids: Optional[List[str]] = None,
-        status_override=None,
-        status_reason=None,
+        status_override: Optional[Status] = None,
+        status_reason: Optional[str] = None,
     ):
         self.name = name
         self.description = description
@@ -433,6 +433,56 @@ def is_empty(self) -> bool:
         """
         return len(self.entries) == len(self.logs) == 0
 
+    @property
+    def passed(self) -> bool:
+        """Shortcut for getting if report status should be considered passed."""
+        return self.status.normalised() == Status.PASSED
+
+    @property
+    def failed(self) -> bool:
+        """
+        Shortcut for checking if report status should be considered failed.
+        """
+        return self.status <= Status.FAILED
+
+    @property
+    def unstable(self) -> bool:
+        """
+        Shortcut for checking if report status should be considered unstable.
+        """
+        return self.status.normalised() == Status.UNSTABLE
+
+    @property
+    def unknown(self) -> bool:
+        """
+        Shortcut for checking if report status is unknown.
+        """
+        return self.status.normalised() == Status.UNKNOWN
+
+    @property
+    def status(self) -> Status:
+        """Return the report status."""
+        if self.status_override:
+            return self.status_override
+        return self._status
+
+    @status.setter
+    def status(self, new_status: Status):
+        self._status = new_status
+
+    @property
+    def runtime_status(self) -> RuntimeStatus:
+        """
+        Used for interactive mode, the runtime status of a testcase will be one
+        of ``RuntimeStatus``.
+        """
+        return self._runtime_status
+
+    @runtime_status.setter
+    def runtime_status(self, new_status: RuntimeStatus):
+        """Set the runtime status."""
+        self._runtime_status = new_status
+
     @property
     def hash(self):
         """Return a hash of all entries in this report."""
@@ -468,34 +518,8 @@ def __init__(self, name, **kwargs):
         for child in self.entries:
             self.set_parent_uids(child)
 
-    @property
-    def passed(self):
-        """Shortcut for getting if report status should be considered passed."""
-        return self.status.normalised() == Status.PASSED
-
-    @property
-    def failed(self):
-        """
-        Shortcut for checking if report status should be considered failed.
-        """
-        return self.status <= Status.FAILED
-
-    @property
-    def unstable(self):
-        """
-        Shortcut for checking if report status should be considered unstable.
-        """
-        return self.status.normalised() == Status.UNSTABLE
-
-    @property
-    def unknown(self):
-        """
-        Shortcut for checking if report status is unknown.
-        """
-        return self.status.normalised() == Status.UNKNOWN
-
-    @property
-    def status(self):
+    @Report.status.getter
+    def status(self) -> Status:
         """
         Status of the report, will be used to decide
         if a Testplan run has completed successfully or not.
@@ -513,12 +537,8 @@ def status(self):
 
         return self._status
 
-    @status.setter
-    def status(self, new_status):
-        self._status = new_status
-
     @property
-    def runtime_status(self):
+    def runtime_status(self) -> RuntimeStatus:
         """
         The runtime status is used for interactive running, and reports
         whether a particular entry is READY, WAITING, RUNNING, RESETTING,
@@ -534,7 +554,7 @@ def runtime_status(self):
         return self._runtime_status
 
     @runtime_status.setter
-    def runtime_status(self, new_status):
+    def runtime_status(self, new_status: RuntimeStatus):
         """Set the runtime_status of all child entries."""
         for entry in self:
             if entry.category != ReportCategories.SYNTHESIZED:
@@ -635,11 +655,11 @@ def remove_by_uid(self, uid):
 
     __delitem__ = remove_by_uid
 
-    def pre_order_reports(self):
+    def pre_order_iterate(self):
         yield self
         for e in self:
             if isinstance(e, BaseReportGroup):
-                yield from e.pre_order_reports()
+                yield from e.pre_order_iterate()
             elif isinstance(e, Report):
                 yield e
 
@@ -961,7 +981,7 @@ def set_runtime_status_filtered(
     ) -> None:
         """
         Alternative setter for the runtime status of an entry. Propagates only
-          to the specified entries.
+        to the specified entries.
 
         :param new_status: new runtime status to be set
         :param entries: tree-like structure of entries names

testplan/common/report/schemas.py

@@ -4,12 +4,16 @@
 from marshmallow import Schema, fields, post_load
 from marshmallow.utils import EXCLUDE
 
+from testplan.common.report.base import (
+    BaseReportGroup,
+    Report,
+    RuntimeStatus,
+    Status,
+)
 from testplan.common.serialization import fields as custom_fields
 from testplan.common.serialization import schemas
 from testplan.common.utils import timing
 
-from .base import Report, BaseReportGroup, Status, RuntimeStatus
-
 __all__ = ["ReportLogSchema", "ReportSchema", "BaseReportGroupSchema"]
 
 # pylint: disable=unused-argument
@@ -91,6 +95,14 @@ class Meta:
         allow_none=True,
     )
     status_reason = fields.String(allow_none=True)
+    status = fields.Function(
+        lambda x: x.status.to_json_compatible(),
+        Status.from_json_compatible,
+    )
+    runtime_status = fields.Function(
+        lambda x: x.runtime_status.to_json_compatible(),
+        RuntimeStatus.from_json_compatible,
+    )
     logs = fields.Nested(ReportLogSchema, many=True)
     hash = fields.Integer(dump_only=True)
     parent_uids = fields.List(fields.String())
@@ -127,14 +139,6 @@ class BaseReportGroupSchema(ReportSchema):
         },
         many=True,
     )
-    status = fields.Function(
-        lambda x: x.status.to_json_compatible(),
-        Status.from_json_compatible,
-    )
-    runtime_status = fields.Function(
-        lambda x: x.runtime_status.to_json_compatible(),
-        RuntimeStatus.from_json_compatible,
-    )
     counter = fields.Dict(dump_only=True)
     children = fields.List(fields.Nested(ReportLinkSchema))
 

testplan/common/utils/json.py

@@ -0,0 +1,43 @@
+import json
+from pathlib import Path
+from typing import Union
+
+_USE_ORJSON = False
+
+try:
+    import orjson
+except ImportError:
+    pass
+else:
+    _USE_ORJSON = True
+
+
+def json_loads(data: str):
+    if _USE_ORJSON:
+        return orjson.loads(data)
+    else:
+        return json.loads(data)
+
+
+def json_dumps(data, indent_2=False, default=None) -> str:
+    if _USE_ORJSON:
+        return orjson.dumps(
+            data,
+            default=default,
+            option=orjson.OPT_INDENT_2 if indent_2 else 0,
+        ).decode()
+    else:
+        if default:
+
+            class _E(json.JSONEncoder):
+                def default(self, o):
+                    return default(o)
+
+        else:
+            _E = None
+        return json.dumps(data, cls=_E, indent=2 if indent_2 else None)
+
+
+def json_load_from_path(path: Union[str, Path]) -> dict:
+    with open(path) as fp:
+        return json_loads(fp.read())