v2.3.0 (#12)

README.md

@@ -53,10 +53,11 @@ hoyolab-rss-feeds -c path/to/config.toml
 It is also possible to generate the feeds via code:
 
 ```python
+from pathlib import Path
 from hoyolabrssfeeds import FeedConfigLoader, GameFeed, GameFeedCollection, Game
 
 async def generate_feeds():
-    loader = FeedConfigLoader("path/to/config.toml")
+    loader = FeedConfigLoader(Path("path/to/config.toml"))
 
     # all games in config
     all_configs = await loader.get_all_feed_configs()
@@ -113,6 +114,20 @@ for each feed.
 should be used to avoid wrong auto-escaping of backslashes. More info about the TOML
 format can be found in the [official documentation](https://toml.io/en/).
 
+### Logging
+
+Simple logs at level `INFO` are written to the terminal by default. If a file path is given
+via parameter (`-l /path/to/out.log`), the logs are written to this file.
+
+If the application is run via code, the logger must be 
+[configured](https://docs.python.org/3.11/howto/logging.html#configuring-logging) separately. 
+The application specific logger is available by:
+
+```python
+import logging
+logger = logging.getLogger("hoyolabrssfeeds")
+```
+
 ### Options
 
 #### Games

RELEASENOTES.md

@@ -1,5 +1,7 @@
+## New Features
+
+- Added simple logging (for level `INFO`)
+
 ## Fixes
 
-- Added temporary notification message for #10 
-- Updated dependencies
-- Minor bug fixes 
+- Added a Hoyolab structured content parser to fix #10

src/hoyolabrssfeeds/__main__.py

@@ -1,26 +1,26 @@
 import argparse
 import asyncio
+import logging
 from pathlib import Path
 from platform import system
 from typing import Optional
 
 from .configs import FeedConfigLoader
 from .feeds import GameFeedCollection
 
+logger = logging.getLogger(__package__)
+
 
 async def create_feeds(config_path: Optional[Path] = None) -> None:
     # fallback path defined in config loader if no path given
     config_loader = FeedConfigLoader(config_path)
 
     if not config_loader.path.exists():
         await config_loader.create_default_config_file()
-        print(
-            'Default config file created at "{}"!'.format(config_loader.path.resolve())
-        )
+        logger.info("Default config file created at %s.", config_loader.path.resolve())
         return
 
     feed_configs = await config_loader.get_all_feed_configs()
-
     game_feed = GameFeedCollection.from_configs(feed_configs)
     await game_feed.create_feeds()
 
@@ -35,11 +35,27 @@ def cli() -> None:
     )
 
     arg_parser.add_argument(
-        "-c", "--config-path", help="path to the TOML config file", type=Path
+        "-c", "--config-path", help="Path to the TOML config file", type=Path
+    )
+
+    arg_parser.add_argument(
+        "-l",
+        "--log-path",
+        required=False,
+        default=None,
+        help="Path to the written log file",
+        type=Path,
     )
 
     args = arg_parser.parse_args()
 
+    logging.basicConfig(
+        filename=args.log_path,
+        filemode="a",
+        format="%(asctime)s | %(levelname)-8s | %(message)s",
+        level=logging.INFO,
+    )
+
     asyncio.run(create_feeds(args.config_path))
 
 

src/hoyolabrssfeeds/feeds.py

@@ -1,5 +1,5 @@
 import asyncio
-import warnings
+import logging
 from typing import List
 from typing import Optional
 from typing import Type
@@ -21,6 +21,8 @@
 _GF = TypeVar("_GF", bound="GameFeed")
 _GFC = TypeVar("_GFC", bound="GameFeedCollection")
 
+logger = logging.getLogger(__name__)
+
 
 class GameFeed:
     """Feed generator for a single game."""
@@ -34,10 +36,9 @@ def __init__(
         # warn if identical paths for writers are found
         writer_paths = [str(writer.config.path) for writer in feed_writers]
         if len(writer_paths) != len(set(writer_paths)):
-            warnings.warn(
-                "Writers for {} game feed contain identical paths".format(
-                    feed_meta.game.name.title()
-                )
+            logger.warning(
+                'Writers for "%s" feed contain identical paths!',
+                feed_meta.title or feed_meta.game.name.title(),
             )
 
         if feed_loader is None:
@@ -79,6 +80,13 @@ async def create_feed(
     ) -> None:
         """Create or update a feed and write it to files."""
 
+        logger.info(
+            '%s "%s" feed in %s format...',
+            "Updating" if self._feed_loader.config.path.exists() else "Creating",
+            self._feed_meta.title or self._feed_meta.game.name.title(),
+            " & ".join([w.config.feed_type.title() for w in self._feed_writers]),
+        )
+
         local_session = session or aiohttp.ClientSession()
         feed_categories = self._feed_meta.categories or [c for c in FeedItemCategory]
         self._was_updated = False
@@ -115,6 +123,16 @@ async def create_feed(
                 ]
             )
 
+            logger.info(
+                'The "%s" feed was successfully updated.',
+                self._feed_meta.title or self._feed_meta.game.name.title(),
+            )
+        else:
+            logger.info(
+                'The "%s" feed is still uptodate.',
+                self._feed_meta.title or self._feed_meta.game.name.title(),
+            )
+
     async def _update_category_feed(
         self,
         session: aiohttp.ClientSession,
@@ -143,6 +161,12 @@ async def _update_category_feed(
         }
 
         if len(new_or_outdated_ids) > 0:
+            logger.info(
+                'Found %d new or outdated posts for "%s" category.',
+                len(new_or_outdated_ids),
+                category.name.title(),
+            )
+
             # remove outdated items from feed because they will be re-fetched
             category_items = list(
                 filter(lambda item: item.id not in new_or_outdated_ids, category_items)

src/hoyolabrssfeeds/hoyolab.py

@@ -1,3 +1,4 @@
+import json
 import re
 from typing import Any
 from typing import Dict
@@ -53,10 +54,16 @@ async def _request(
 
         return response_json
 
-    @staticmethod
-    def _transform_post(post: Dict[str, Any]) -> Dict[str, Any]:
+    def _transform_post(self, post: Dict[str, Any]) -> Dict[str, Any]:
         """Transform (i.e. apply fixes) post of Hoyolab API response."""
 
+        # weird hoyolab bug/feature, where the content html is just a language code.
+        # this needs to be first to also apply the other fixes.
+        if re.fullmatch(r"^[a-z]{2}-[a-z]{2}$", post["post"]["content"]):
+            post["post"]["content"] = self._parse_structured_content(
+                post["post"]["structured_content"]
+            )
+
         # remove empty leading paragraphs
         if post["post"]["content"].startswith(
             ("<p></p>", "<p>&nbsp;</p>", "<p><br></p>")
@@ -68,13 +75,48 @@ def _transform_post(post: Dict[str, Any]) -> Dict[str, Any]:
             "hoyolab-upload-private", "upload-os-bbs"
         )
 
-        # weird hoyolab bug/feature, where the content html is just a language code.
-        # could be fixed by parsing the structured_content and creating html from it.
-        if re.fullmatch(r"^[a-z]{2}-[a-z]{2}$", post["post"]["content"]):
-            post["post"]["content"] = "<em>Content not available...</em>"
-
         return post
 
+    @staticmethod
+    def _parse_structured_content(structured_content: str) -> str:
+        """Parse the Hoyolab structured content and return the constructed HTML."""
+
+        structured_content = re.sub(r"(\\)?\\n", "<br>", structured_content)
+        html_content = []
+
+        try:
+            json_content: List[Dict[str, Any]] = json.loads(structured_content)
+        except json.JSONDecodeError as err:
+            raise HoyolabApiError(
+                "Could not decode structured content to JSON!"
+            ) from err
+
+        for node in json_content:
+            if type(node["insert"]) is str:
+                if "attributes" in node and "link" in node["attributes"]:
+                    text = '<a href="{}">{}</a>'.format(
+                        node["attributes"]["link"], node["insert"]
+                    )
+                else:
+                    text = node["insert"]
+
+                if "attributes" in node and "bold" in node["attributes"]:
+                    text = "<p><strong>{}</strong></p>".format(text)
+                elif "attributes" in node and "italic" in node["attributes"]:
+                    text = "<p><em>{}</em></p>".format(text)
+                else:
+                    text = "<p>{}</p>".format(text)
+
+                html_content.append(text)
+            elif "image" in node["insert"]:
+                html_content.append('<img src="{}">'.format(node["insert"]["image"]))
+            elif "video" in node["insert"]:
+                html_content.append(
+                    '<iframe src="{}"></iframe>'.format(node["insert"]["video"])
+                )
+
+        return "".join(html_content)
+
     async def get_news_list(
         self,
         session: aiohttp.ClientSession,

tests/conftest.py

@@ -281,6 +281,7 @@ def mocked_loader(mocker: pytest_mock.MockFixture) -> MagicMock:
 @pytest.fixture
 def mocked_writers(mocker: pytest_mock.MockFixture) -> List[MagicMock]:
     writer: MagicMock = mocker.create_autospec(AbstractFeedFileWriter, instance=True)
+    writer.config.feed_type = models.FeedType.JSON  # needed for logger calls
 
     return [writer]
 
@@ -312,6 +313,9 @@ def validate_hoyolab_post(post: Dict[str, Any], is_full_post: bool) -> None:
         assert type(post["post"]["content"]) is str
         assert len(post["post"]["content"]) > 0
 
+        assert type(post["post"]["structured_content"]) is str
+        assert len(post["post"]["structured_content"]) > 0
+
         assert type(post["post"]["subject"]) is str
         assert len(post["post"]["subject"]) > 0
 

tests/test_feeds.py

@@ -23,15 +23,17 @@ def test_game_feed_was_updated(
 
 
 def test_same_path_warning(
+    caplog: pytest.LogCaptureFixture,
     feed_meta: models.FeedMeta,
     mocked_loader: AbstractFeedFileLoader,
     json_feed_file_writer_config: models.FeedFileWriterConfig,
 ) -> None:
     writer: AbstractFeedFileWriter = JSONFeedFileWriter(json_feed_file_writer_config)
     duplicate_writers = [writer, writer]
 
-    with pytest.warns(UserWarning, match="identical paths"):
+    with caplog.at_level("WARNING"):
         feeds.GameFeed(feed_meta, duplicate_writers, mocked_loader)
+        assert "identical paths" in caplog.text
 
 
 def test_from_config(feed_config: models.FeedConfig) -> None:

tests/test_hoyolab.py

@@ -172,7 +172,8 @@ def test_leading_line_breaks() -> None:
 
     expected = {"post": {"content": "Hello World"}}
 
-    transformed_post = hoyolab.HoyolabNews._transform_post(post)
+    api = hoyolab.HoyolabNews(models.Game.GENSHIN)
+    transformed_post = api._transform_post(post)
 
     assert transformed_post == expected
 
@@ -188,20 +189,53 @@ def test_private_link_bug() -> None:
         "post": {"content": '<img src="https://upload-os-bbs.hoyolab.com/test.jpg">'}
     }
 
-    transformed_post = hoyolab.HoyolabNews._transform_post(post)
+    api = hoyolab.HoyolabNews(models.Game.GENSHIN)
+    transformed_post = api._transform_post(post)
 
     assert transformed_post == expected
 
 
 def test_content_html_bug() -> None:
-    post = {"post": {"content": "en-us"}}
+    post = {"post": {"content": "en-us", "structured_content": "[]"}}
 
-    transformed_post = hoyolab.HoyolabNews._transform_post(post)
+    api = hoyolab.HoyolabNews(models.Game.GENSHIN)
+    transformed_post = api._transform_post(post)
 
     # test that the content was replaced/fixed
     assert transformed_post["post"]["content"] != "en-us"
 
 
+def test_structured_content_parser() -> None:
+    raw_sc = """
+        [{"insert":"Hello World!"},
+        {"insert":"Hello bold World!","attributes":{"bold":true}},
+        {"insert":"Hello italic World!","attributes":{"italic":true}},
+        {"insert":"\\n","attributes":{"align":"center"}},
+        {"insert":"Hello Link!","attributes":{"link":"https://example.com"}},
+        {"insert":{"image":"https://example.com/image.jpg"}},
+        {"insert":{"video":"https://example.com/video.mp4"}}]
+    """
+
+    expected_html = """
+        <p>Hello World!</p>
+        <p><strong>Hello bold World!</strong></p>
+        <p><em>Hello italic World!</em></p>
+        <p><br></p>
+        <p><a href="https://example.com">Hello Link!</a></p>
+        <img src="https://example.com/image.jpg">
+        <iframe src="https://example.com/video.mp4"></iframe>
+    """.replace(
+        "    ", ""
+    ).replace(
+        "\n", ""
+    )
+
+    assert hoyolab.HoyolabNews._parse_structured_content(raw_sc) == expected_html
+
+    with pytest.raises(errors.HoyolabApiError):
+        hoyolab.HoyolabNews._parse_structured_content("###")
+
+
 # ---- HELPER FUNCTIONS ----