From 5afdbaf00846fd631d9e2918242dc6cb0fc61795 Mon Sep 17 00:00:00 2001 From: izum1 Date: Sat, 4 May 2024 16:26:56 +0800 Subject: [PATCH 01/16] feat: add limit for the number of logs to save --- config/template.json | 4 + gui.py | 2 +- module/config/argument/args.json | 10 + module/config/argument/argument.yaml | 3 + module/config/argument/task.yaml | 1 + module/config/config_generated.py | 4 + module/config/i18n/en-US.json | 14 ++ module/config/i18n/ja-JP.json | 14 ++ module/config/i18n/zh-CN.json | 14 ++ module/config/i18n/zh-TW.json | 14 ++ module/logger.py | 265 ++++++++++++++++++++++++--- 11 files changed, 314 insertions(+), 31 deletions(-) diff --git a/config/template.json b/config/template.json index 21450d3031..eed2ab1579 100644 --- a/config/template.json +++ b/config/template.json @@ -42,6 +42,10 @@ } }, "General": { + "Log": { + "LogBackUpCount": 7, + "EnableZip": true + }, "Retirement": { "RetireMode": "one_click_retire" }, diff --git a/gui.py b/gui.py index 97b5f3cf4f..953cc93cbc 100644 --- a/gui.py +++ b/gui.py @@ -72,7 +72,7 @@ def func(ev: threading.Event): should_exit = False while not should_exit: event = Event() - process = Process(target=func, args=(event,)) + process = Process(target=func, args=(event,), name="gui") process.start() while not should_exit: try: diff --git a/module/config/argument/args.json b/module/config/argument/args.json index 551c710580..7d629f8b21 100644 --- a/module/config/argument/args.json +++ b/module/config/argument/args.json @@ -296,6 +296,16 @@ } }, "General": { + "Log": { + "LogBackUpCount": { + "type": "input", + "value": 7 + }, + "EnableZip": { + "type": "checkbox", + "value": true + } + }, "Retirement": { "RetireMode": { "type": "select", diff --git a/module/config/argument/argument.yaml b/module/config/argument/argument.yaml index cc20982a17..d0c9b75fec 100644 --- a/module/config/argument/argument.yaml +++ b/module/config/argument/argument.yaml @@ -119,6 +119,9 @@ DropRecord: MeowfficerTalent: value: do_not option: [ do_not, save, upload, save_and_upload ] +Log: + LogBackUpCount: 7 + EnableZip: true Retirement: RetireMode: value: one_click_retire diff --git a/module/config/argument/task.yaml b/module/config/argument/task.yaml index 74e40c0883..c3512ca718 100644 --- a/module/config/argument/task.yaml +++ b/module/config/argument/task.yaml @@ -15,6 +15,7 @@ Alas: - Optimization - DropRecord General: + - Log - Retirement - OneClickRetire - Enhance diff --git a/module/config/config_generated.py b/module/config/config_generated.py index 8ac69edaa6..c5283c1709 100644 --- a/module/config/config_generated.py +++ b/module/config/config_generated.py @@ -54,6 +54,10 @@ class GeneratedConfig: DropRecord_MeowfficerBuy = 'do_not' # do_not, save DropRecord_MeowfficerTalent = 'do_not' # do_not, save, upload, save_and_upload + # Group `Log` + Log_LogBackUpCount = 7 + Log_EnableZip = True + # Group `Retirement` Retirement_RetireMode = 'one_click_retire' # one_click_retire, enhance, old_retire diff --git a/module/config/i18n/en-US.json b/module/config/i18n/en-US.json index 927dc0c21b..626c09e397 100644 --- a/module/config/i18n/en-US.json +++ b/module/config/i18n/en-US.json @@ -573,6 +573,20 @@ "save_and_upload": "Save and upload" } }, + "Log": { + "_info": { + "name": "Log._info.name", + "help": "Log._info.help" + }, + "LogBackUpCount": { + "name": "Log.LogBackUpCount.name", + "help": "Log.LogBackUpCount.help" + }, + "EnableZip": { + "name": "Log.EnableZip.name", + "help": "Log.EnableZip.help" + } + }, "Retirement": { "_info": { "name": "Retirement Settings", diff --git a/module/config/i18n/ja-JP.json b/module/config/i18n/ja-JP.json index 52ceba496a..daf86c9c81 100644 --- a/module/config/i18n/ja-JP.json +++ b/module/config/i18n/ja-JP.json @@ -573,6 +573,20 @@ "save_and_upload": "save_and_upload" } }, + "Log": { + "_info": { + "name": "Log._info.name", + "help": "Log._info.help" + }, + "LogBackUpCount": { + "name": "Log.LogBackUpCount.name", + "help": "Log.LogBackUpCount.help" + }, + "EnableZip": { + "name": "Log.EnableZip.name", + "help": "Log.EnableZip.help" + } + }, "Retirement": { "_info": { "name": "Retirement._info.name", diff --git a/module/config/i18n/zh-CN.json b/module/config/i18n/zh-CN.json index 24e75d5dde..a909ecf7a6 100644 --- a/module/config/i18n/zh-CN.json +++ b/module/config/i18n/zh-CN.json @@ -573,6 +573,20 @@ "save_and_upload": "保存并上传" } }, + "Log": { + "_info": { + "name": "Log._info.name", + "help": "Log._info.help" + }, + "LogBackUpCount": { + "name": "Log.LogBackUpCount.name", + "help": "Log.LogBackUpCount.help" + }, + "EnableZip": { + "name": "Log.EnableZip.name", + "help": "Log.EnableZip.help" + } + }, "Retirement": { "_info": { "name": "退役设置", diff --git a/module/config/i18n/zh-TW.json b/module/config/i18n/zh-TW.json index 00b9a450b6..a4762a2a32 100644 --- a/module/config/i18n/zh-TW.json +++ b/module/config/i18n/zh-TW.json @@ -573,6 +573,20 @@ "save_and_upload": "保存並上傳" } }, + "Log": { + "_info": { + "name": "Log._info.name", + "help": "Log._info.help" + }, + "LogBackUpCount": { + "name": "Log.LogBackUpCount.name", + "help": "Log.LogBackUpCount.help" + }, + "EnableZip": { + "name": "Log.EnableZip.name", + "help": "Log.EnableZip.help" + } + }, "Retirement": { "_info": { "name": "退役設定", diff --git a/module/logger.py b/module/logger.py index 97dc148dec..4f4ff51ff2 100644 --- a/module/logger.py +++ b/module/logger.py @@ -1,11 +1,20 @@ +import bz2 import datetime +import gzip +import io import logging +import multiprocessing import os +import pathlib +import shutil import sys +import threading +import time +from logging.handlers import TimedRotatingFileHandler from typing import Callable, List from rich.console import Console, ConsoleOptions, ConsoleRenderable, NewLine -from rich.highlighter import RegexHighlighter, NullHighlighter +from rich.highlighter import NullHighlighter, RegexHighlighter from rich.logging import RichHandler from rich.rule import Rule from rich.style import Style @@ -86,6 +95,169 @@ def handle(self, record: logging.LogRecord) -> bool: super().handle(record) +class RichTimedRotatingHandler(TimedRotatingFileHandler): + ZIPMAP= { + "gz" : (gzip.open, ".gz"), + "bz2" : (bz2.open, ".bz2") + } + def __init__(self, zip = False, *args, **kwargs) -> None: + TimedRotatingFileHandler.__init__(self, *args, **kwargs) + self.console = Console(file=io.StringIO(), no_color=True, highlight=False, width=119) + self.richd = RichHandler( + console=self.console, + show_path=False, + show_time=False, + show_level=False, + rich_tracebacks=True, + tracebacks_show_locals=True, + tracebacks_extra_lines=3, + highlighter=NullHighlighter(), + ) + # Keep the same format + self.richd.setFormatter( + logging.Formatter( + fmt="%(asctime)s.%(msecs)03d | %(levelname)s | %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + ) + # To handle the API of logger.print() + self.console = self.richd.console + # To handle the API of alas.save_error_log() + self.log_file = None + self.zip = zip + + # Override the initial rolloverAt + self.rolloverAt = time.time() + self.doRollover() + + # Close unnecessary stream + self.stream.close() + self.stream = None + + def getFilesToDelete(self) -> list: + """ + Determine the files to delete when rolling over.\n + Override the original method to use RichHandler + """ + dirName, baseName = os.path.split(self.baseFilename) + fileNames = os.listdir(dirName) + result = [] + suffix = "_" + baseName + plen = len(suffix) + for fileName in fileNames: + if fileName[-plen:] == suffix: + prefix = fileName[:-plen] + if self.extMatch.match(prefix): + result.append(os.path.join(dirName, fileName)) + if len(result) < self.backupCount: + result = [] + else: + result.sort() + result = result[: len(result) - self.backupCount] + return result + + def doRollover(self) -> None: + """ + Do a rollover.\n + Override the original method to use RichHandler + """ + if self.richd.console: + self.richd.console.file.close() + self.richd.console.file = None + + currentTime = int(time.time()) + dstNow = time.localtime(currentTime)[-1] + t = self.rolloverAt + if self.utc: + timeTuple = time.gmtime(t) + else: + timeTuple = time.localtime(t) + dstThen = timeTuple[-1] + if dstNow != dstThen: + if dstNow: + addend = 3600 + else: + addend = -3600 + timeTuple = time.localtime(t + addend) + + path = pathlib.Path(self.baseFilename) + # 2021-08-01 + _ + alas.txt -> "2021-08-01_alas.txt" + newPath = path.with_name( + time.strftime(self.suffix, timeTuple) + "_" + path.name + ) + self.richd.console.file = open( + newPath, "a" if os.path.exists(newPath) else "w", encoding="utf-8" + ) + + if self.backupCount > 0: + for s in self.getFilesToDelete(): + os.remove(s) + + newRolloverAt = self.computeRollover(currentTime) + while newRolloverAt <= currentTime: + newRolloverAt = newRolloverAt + self.interval + # If DST changes and midnight or weekly rollover, adjust for this. + if (self.when == "MIDNIGHT" or self.when.startswith("W")) and not self.utc: + dstAtRollover = time.localtime(newRolloverAt)[-1] + if dstNow != dstAtRollover: + if ( + not dstNow + ): # DST kicks in before next rollover, so we need to deduct an hour + addend = -3600 + else: # DST bows out before next rollover, so we need to add an hour + addend = 3600 + newRolloverAt += addend + self.rolloverAt = newRolloverAt + + if self.zip: + thread = threading.Thread(target=self._compress, args=(self.log_file,)) + thread.daemon = True + thread.start() + self.log_file = str(newPath.resolve()) + + def _compress(self, file, compression="bz2") -> None: + """ + Compress a file with gzip\n + If file is None (In the initialization), compress the last log file\n + Template: ./log/2021-08-01_alas.txt to ./log/bak/2021-08-01_alas.txt.gz + """ + try: + if file is None: + basePath = pathlib.Path(self.baseFilename) + name = basePath.name + logFiles = [file for file in basePath.parent.glob("*_" + name)] + if len(logFiles) < 2: + return + file = logFiles[-2] + + path = pathlib.Path(file) + parent = path.parent + (path.parent / "bak").mkdir(exist_ok=True) + cmp_func, ext = self.ZIPMAP.get(compression, (gzip.open, ".gz")) + zipFile = parent.joinpath("bak").joinpath(path.name).with_suffix(ext) + + if zipFile.exists(): + return + with file.open("rb") as f_in: + with cmp_func(zipFile, "wb") as f_out: + shutil.copyfileobj(f_in, f_out) + except Exception as e: + logger.exception(e) + + + def print(self, *objects: ConsoleRenderable, **kwargs) -> None: + Console.print(self.console, *objects, **kwargs) + + # @override + def emit(self, record: logging.LogRecord) -> None: + try: + if self.shouldRollover(record): + self.doRollover() + RichHandler.emit(self.richd, record) + except Exception: + RichHandler.handleError(self.richd, record) + + class HTMLConsole(Console): """ Force full feature console @@ -186,38 +358,69 @@ def _set_file_logger(name=pyw_name): def set_file_logger(name=pyw_name): - if '_' in name: - name = name.split('_', 1)[0] - log_file = f'./log/{datetime.date.today()}_{name}.txt' - try: - file = open(log_file, mode='a', encoding='utf-8') - except FileNotFoundError: - os.mkdir('./log') - file = open(log_file, mode='a', encoding='utf-8') - - file_console = Console( - file=file, - no_color=True, - highlight=False, - width=119, - ) - - hdlr = RichFileHandler( - console=file_console, - show_path=False, - show_time=False, - show_level=False, - rich_tracebacks=True, - tracebacks_show_locals=True, - tracebacks_extra_lines=3, - highlighter=NullHighlighter(), + import json + if "_" in name: + name = name.split("_", 1)[0] + pname = multiprocessing.current_process().name.replace(":", "_") + + log_dir = pathlib.Path("./log") + log_file = log_dir.joinpath(f"{pname}.txt" if name == "gui" else f"{name}.txt") + os.makedirs(log_dir, exist_ok=True) + + # These process needn't to save log file. + process = ["SyncManager-", "MainProcess", "Process-"] + if any(p in log_file.name for p in process): + hdlr = RichFileHandler(console=Console(file=io.StringIO())) + logger.addHandler(hdlr) + logger.log_file = str(log_file.resolve()) + if os.path.exists(log_file): + os.remove(log_file) + return + + valid_cfg = [ + file + for file in pathlib.Path("./config").glob("*.json") + if not file.name.endswith((".maa.json", ".fpy.json", "template.json")) + ] + if len(valid_cfg) > 0: + try: + with open(str(valid_cfg[0]),"r") as f: + data_dict = json.load(f) + cnt = data_dict.get("General", {}).get("Log", {}).get("LogBackUpCount") + count = cnt if cnt is not None and isinstance(cnt, int) and cnt >= 0 else 7 + zip = data_dict.get("General", {}).get("Log", {}).get("EnableZip") + zip = zip if zip is not None and isinstance(zip, bool) else False + except Exception as e: + count = 7 + zip = False + logger.exception(e) + else: + count = 7 + zip = False + + hdlr = RichTimedRotatingHandler( + zip=zip, + filename=str(log_file), + when="midnight", + # when="S", + interval=1, + backupCount=count, + encoding="utf-8", ) - hdlr.setFormatter(file_formatter) - logger.handlers = [h for h in logger.handlers if not isinstance( - h, (logging.FileHandler, RichFileHandler))] + logger.handlers = [ + h + for h in logger.handlers + if not isinstance( + h, (logging.FileHandler, RichTimedRotatingHandler, RichFileHandler) + ) + ] logger.addHandler(hdlr) - logger.log_file = log_file + logger.log_file = hdlr.log_file + + # Delete the default log file after initialize the handler + if os.path.exists(log_file): + os.remove(log_file) def set_func_logger(func): @@ -280,6 +483,8 @@ def print(*objects: ConsoleRenderable, **kwargs): hdlr._func(renderable) elif isinstance(hdlr, RichHandler): hdlr.console.print(*objects) + elif isinstance(hdlr, RichTimedRotatingHandler): + hdlr.print(*objects, **kwargs) def rule(title="", *, characters="─", style="rule.line", end="\n", align="center"): From 418a2a43375ce26015f95a6fe8c831141d0db078 Mon Sep 17 00:00:00 2001 From: izum1 Date: Sat, 4 May 2024 17:14:00 +0800 Subject: [PATCH 02/16] fix: fix logger to handle linux os --- module/logger.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/module/logger.py b/module/logger.py index 4f4ff51ff2..a2151b0afe 100644 --- a/module/logger.py +++ b/module/logger.py @@ -361,14 +361,16 @@ def set_file_logger(name=pyw_name): import json if "_" in name: name = name.split("_", 1)[0] - pname = multiprocessing.current_process().name.replace(":", "_") + # Handler Windows : Windows have "SyncManager-N:N", "MainProcess", "Process-N", "gui" 4 Processes + # There have no process named "gui", only "MainProcess" in Linux + pname = multiprocessing.current_process().name.replace(":", "_") if os.name == "nt" else name log_dir = pathlib.Path("./log") log_file = log_dir.joinpath(f"{pname}.txt" if name == "gui" else f"{name}.txt") os.makedirs(log_dir, exist_ok=True) - # These process needn't to save log file. - process = ["SyncManager-", "MainProcess", "Process-"] + # These process needn't to save log file in Windows + process = ["SyncManager-", "MainProcess", "Process-"] if os.name == "nt" else [] if any(p in log_file.name for p in process): hdlr = RichFileHandler(console=Console(file=io.StringIO())) logger.addHandler(hdlr) From 1fd9b0937ec70bdb330cf3793d1689a8c15666c7 Mon Sep 17 00:00:00 2001 From: izum1 Date: Sat, 4 May 2024 19:56:37 +0800 Subject: [PATCH 03/16] fix: fix some logger's bugs --- config/template.json | 5 +- module/config/argument/args.json | 22 +++++-- module/config/argument/argument.yaml | 9 ++- module/config/config_generated.py | 5 +- module/config/i18n/en-US.json | 24 +++++--- module/config/i18n/ja-JP.json | 24 +++++--- module/config/i18n/zh-CN.json | 24 +++++--- module/config/i18n/zh-TW.json | 24 +++++--- module/logger.py | 88 +++++++++++++++------------- 9 files changed, 147 insertions(+), 78 deletions(-) diff --git a/config/template.json b/config/template.json index eed2ab1579..52603643c4 100644 --- a/config/template.json +++ b/config/template.json @@ -43,8 +43,9 @@ }, "General": { "Log": { - "LogBackUpCount": 7, - "EnableZip": true + "LogKeepCount": 7, + "LogBackUpMethod": "delete", + "ZipMethod": "bz2" }, "Retirement": { "RetireMode": "one_click_retire" diff --git a/module/config/argument/args.json b/module/config/argument/args.json index 7d629f8b21..9973a43d69 100644 --- a/module/config/argument/args.json +++ b/module/config/argument/args.json @@ -297,13 +297,27 @@ }, "General": { "Log": { - "LogBackUpCount": { + "LogKeepCount": { "type": "input", "value": 7 }, - "EnableZip": { - "type": "checkbox", - "value": true + "LogBackUpMethod": { + "type": "select", + "value": "delete", + "option": [ + "delete", + "zip", + "none" + ] + }, + "ZipMethod": { + "type": "select", + "value": "bz2", + "option": [ + "bz2", + "gzip", + "none" + ] } }, "Retirement": { diff --git a/module/config/argument/argument.yaml b/module/config/argument/argument.yaml index d0c9b75fec..671f7bf0e2 100644 --- a/module/config/argument/argument.yaml +++ b/module/config/argument/argument.yaml @@ -120,8 +120,13 @@ DropRecord: value: do_not option: [ do_not, save, upload, save_and_upload ] Log: - LogBackUpCount: 7 - EnableZip: true + LogKeepCount: 7 + LogBackUpMethod: + value: delete + option: [ delete, zip, none ] + ZipMethod: + value: bz2 + option: [ bz2, gzip, none] Retirement: RetireMode: value: one_click_retire diff --git a/module/config/config_generated.py b/module/config/config_generated.py index c5283c1709..2f4dc9ddfa 100644 --- a/module/config/config_generated.py +++ b/module/config/config_generated.py @@ -55,8 +55,9 @@ class GeneratedConfig: DropRecord_MeowfficerTalent = 'do_not' # do_not, save, upload, save_and_upload # Group `Log` - Log_LogBackUpCount = 7 - Log_EnableZip = True + Log_LogKeepCount = 7 + Log_LogBackUpMethod = 'delete' # delete, zip, none + Log_ZipMethod = 'bz2' # bz2, gzip, none # Group `Retirement` Retirement_RetireMode = 'one_click_retire' # one_click_retire, enhance, old_retire diff --git a/module/config/i18n/en-US.json b/module/config/i18n/en-US.json index 626c09e397..cdaea0921c 100644 --- a/module/config/i18n/en-US.json +++ b/module/config/i18n/en-US.json @@ -578,13 +578,23 @@ "name": "Log._info.name", "help": "Log._info.help" }, - "LogBackUpCount": { - "name": "Log.LogBackUpCount.name", - "help": "Log.LogBackUpCount.help" - }, - "EnableZip": { - "name": "Log.EnableZip.name", - "help": "Log.EnableZip.help" + "LogKeepCount": { + "name": "Log.LogKeepCount.name", + "help": "Log.LogKeepCount.help" + }, + "LogBackUpMethod": { + "name": "Log.LogBackUpMethod.name", + "help": "Log.LogBackUpMethod.help", + "delete": "delete", + "zip": "zip", + "none": "none" + }, + "ZipMethod": { + "name": "Log.ZipMethod.name", + "help": "Log.ZipMethod.help", + "bz2": "bz2", + "gzip": "gzip", + "none": "none" } }, "Retirement": { diff --git a/module/config/i18n/ja-JP.json b/module/config/i18n/ja-JP.json index daf86c9c81..46174a5ee8 100644 --- a/module/config/i18n/ja-JP.json +++ b/module/config/i18n/ja-JP.json @@ -578,13 +578,23 @@ "name": "Log._info.name", "help": "Log._info.help" }, - "LogBackUpCount": { - "name": "Log.LogBackUpCount.name", - "help": "Log.LogBackUpCount.help" - }, - "EnableZip": { - "name": "Log.EnableZip.name", - "help": "Log.EnableZip.help" + "LogKeepCount": { + "name": "Log.LogKeepCount.name", + "help": "Log.LogKeepCount.help" + }, + "LogBackUpMethod": { + "name": "Log.LogBackUpMethod.name", + "help": "Log.LogBackUpMethod.help", + "delete": "delete", + "zip": "zip", + "none": "none" + }, + "ZipMethod": { + "name": "Log.ZipMethod.name", + "help": "Log.ZipMethod.help", + "bz2": "bz2", + "gzip": "gzip", + "none": "none" } }, "Retirement": { diff --git a/module/config/i18n/zh-CN.json b/module/config/i18n/zh-CN.json index a909ecf7a6..0f0510d022 100644 --- a/module/config/i18n/zh-CN.json +++ b/module/config/i18n/zh-CN.json @@ -578,13 +578,23 @@ "name": "Log._info.name", "help": "Log._info.help" }, - "LogBackUpCount": { - "name": "Log.LogBackUpCount.name", - "help": "Log.LogBackUpCount.help" - }, - "EnableZip": { - "name": "Log.EnableZip.name", - "help": "Log.EnableZip.help" + "LogKeepCount": { + "name": "Log.LogKeepCount.name", + "help": "Log.LogKeepCount.help" + }, + "LogBackUpMethod": { + "name": "Log.LogBackUpMethod.name", + "help": "Log.LogBackUpMethod.help", + "delete": "delete", + "zip": "zip", + "none": "none" + }, + "ZipMethod": { + "name": "Log.ZipMethod.name", + "help": "Log.ZipMethod.help", + "bz2": "bz2", + "gzip": "gzip", + "none": "none" } }, "Retirement": { diff --git a/module/config/i18n/zh-TW.json b/module/config/i18n/zh-TW.json index a4762a2a32..012e25deaf 100644 --- a/module/config/i18n/zh-TW.json +++ b/module/config/i18n/zh-TW.json @@ -578,13 +578,23 @@ "name": "Log._info.name", "help": "Log._info.help" }, - "LogBackUpCount": { - "name": "Log.LogBackUpCount.name", - "help": "Log.LogBackUpCount.help" - }, - "EnableZip": { - "name": "Log.EnableZip.name", - "help": "Log.EnableZip.help" + "LogKeepCount": { + "name": "Log.LogKeepCount.name", + "help": "Log.LogKeepCount.help" + }, + "LogBackUpMethod": { + "name": "Log.LogBackUpMethod.name", + "help": "Log.LogBackUpMethod.help", + "delete": "delete", + "zip": "zip", + "none": "none" + }, + "ZipMethod": { + "name": "Log.ZipMethod.name", + "help": "Log.ZipMethod.help", + "bz2": "bz2", + "gzip": "gzip", + "none": "none" } }, "Retirement": { diff --git a/module/logger.py b/module/logger.py index a2151b0afe..82c16f08c7 100644 --- a/module/logger.py +++ b/module/logger.py @@ -8,6 +8,7 @@ import pathlib import shutil import sys +import tarfile import threading import time from logging.handlers import TimedRotatingFileHandler @@ -96,11 +97,12 @@ def handle(self, record: logging.LogRecord) -> bool: class RichTimedRotatingHandler(TimedRotatingFileHandler): - ZIPMAP= { - "gz" : (gzip.open, ".gz"), - "bz2" : (bz2.open, ".bz2") + ZIPMAP = { + "none": (open, ""), + "gzip": (gzip.open, "gz"), + "bz2" : (bz2.open, "bz2") } - def __init__(self, zip = False, *args, **kwargs) -> None: + def __init__(self, bak="none", compression="bz2", *args, **kwargs) -> None: TimedRotatingFileHandler.__init__(self, *args, **kwargs) self.console = Console(file=io.StringIO(), no_color=True, highlight=False, width=119) self.richd = RichHandler( @@ -124,11 +126,13 @@ def __init__(self, zip = False, *args, **kwargs) -> None: self.console = self.richd.console # To handle the API of alas.save_error_log() self.log_file = None - self.zip = zip + self.bak = bak.lower() + self.compression = compression.lower() # Override the initial rolloverAt self.rolloverAt = time.time() self.doRollover() + # self.rolloverAt = self.computeRollover(int(time.time())) # Close unnecessary stream self.stream.close() @@ -209,38 +213,37 @@ def doRollover(self) -> None: newRolloverAt += addend self.rolloverAt = newRolloverAt - if self.zip: - thread = threading.Thread(target=self._compress, args=(self.log_file,)) - thread.daemon = True - thread.start() + thread = threading.Thread(target=self._compress, args=(self.log_file, self.compression, self.bak,)) + thread.daemon = True + thread.start() self.log_file = str(newPath.resolve()) - def _compress(self, file, compression="bz2") -> None: + def _compress(self, file, compression="none", bak="none") -> None: """ Compress a file with gzip\n If file is None (In the initialization), compress the last log file\n Template: ./log/2021-08-01_alas.txt to ./log/bak/2021-08-01_alas.txt.gz """ - try: - if file is None: - basePath = pathlib.Path(self.baseFilename) - name = basePath.name - logFiles = [file for file in basePath.parent.glob("*_" + name)] - if len(logFiles) < 2: - return - file = logFiles[-2] - - path = pathlib.Path(file) - parent = path.parent - (path.parent / "bak").mkdir(exist_ok=True) - cmp_func, ext = self.ZIPMAP.get(compression, (gzip.open, ".gz")) - zipFile = parent.joinpath("bak").joinpath(path.name).with_suffix(ext) - - if zipFile.exists(): + basePath = pathlib.Path(self.baseFilename) + if file is None: + logFiles = [file for file in basePath.parent.glob("*_" + basePath.name)] + if len(logFiles) < 2: return - with file.open("rb") as f_in: - with cmp_func(zipFile, "wb") as f_out: - shutil.copyfileobj(f_in, f_out) + file = sorted(logFiles, key=lambda x: str(x))[-2] + logger.info(logFiles) + try: + logFile = pathlib.Path(file) + parent = logFile.parent + cmpFunc, ext = self.ZIPMAP.get(compression, (gzip.open, "gz")) + zipFile = parent.joinpath("bak").joinpath(logFile.name).with_suffix('.'+ext) + + if not zipFile.exists() and compression != "none" and bak == "zip": + (parent / "bak").mkdir(exist_ok=True) + with logFile.open("rb") as f_in: + with cmpFunc(zipFile, "wb") as f_out: + shutil.copyfileobj(f_in, f_out) + if bak == "none": + shutil.copy2(logFile, zipFile.with_name(logFile.name)) except Exception as e: logger.exception(e) @@ -388,24 +391,29 @@ def set_file_logger(name=pyw_name): try: with open(str(valid_cfg[0]),"r") as f: data_dict = json.load(f) - cnt = data_dict.get("General", {}).get("Log", {}).get("LogBackUpCount") + cnt = data_dict.get("General", {}).get("Log", {}).get("LogKeepCount") count = cnt if cnt is not None and isinstance(cnt, int) and cnt >= 0 else 7 - zip = data_dict.get("General", {}).get("Log", {}).get("EnableZip") - zip = zip if zip is not None and isinstance(zip, bool) else False + bkm = data_dict.get("General", {}).get("Log", {}).get("LogBackUpMethod") + bak_method = bkm if bkm is not None and bkm in ["none", "delete", "zip"] else "none" + method = data_dict.get("General", {}).get("Log", {}).get("ZipMethod") + zip_method = method if method is not None and method in ["none", "gzip", "bz2"] else "none" except Exception as e: - count = 7 - zip = False + count=7 + bak_method="none" + zip_method="none" logger.exception(e) else: - count = 7 - zip = False + count=7 + zip_method="none" + bak_method="none" hdlr = RichTimedRotatingHandler( - zip=zip, + bak=bak_method, filename=str(log_file), - when="midnight", - # when="S", - interval=1, + compression=zip_method, + # when="midnight", + when="S", + interval=10, backupCount=count, encoding="utf-8", ) From e71396d0bd76f2a960aaff078eaf108bff090640 Mon Sep 17 00:00:00 2001 From: izum1 Date: Sat, 4 May 2024 20:46:27 +0800 Subject: [PATCH 04/16] add: GUI translation for General Log --- module/config/argument/args.json | 3 +- module/config/argument/argument.yaml | 2 +- module/config/config_generated.py | 2 +- module/config/i18n/en-US.json | 21 ++++++----- module/config/i18n/ja-JP.json | 3 +- module/config/i18n/zh-CN.json | 25 +++++++------ module/config/i18n/zh-TW.json | 25 +++++++------ module/logger.py | 53 +++++++++++++++------------- 8 files changed, 66 insertions(+), 68 deletions(-) diff --git a/module/config/argument/args.json b/module/config/argument/args.json index 9973a43d69..e6b82dc06d 100644 --- a/module/config/argument/args.json +++ b/module/config/argument/args.json @@ -315,8 +315,7 @@ "value": "bz2", "option": [ "bz2", - "gzip", - "none" + "gzip" ] } }, diff --git a/module/config/argument/argument.yaml b/module/config/argument/argument.yaml index 671f7bf0e2..f2609053f7 100644 --- a/module/config/argument/argument.yaml +++ b/module/config/argument/argument.yaml @@ -126,7 +126,7 @@ Log: option: [ delete, zip, none ] ZipMethod: value: bz2 - option: [ bz2, gzip, none] + option: [ bz2, gzip ] Retirement: RetireMode: value: one_click_retire diff --git a/module/config/config_generated.py b/module/config/config_generated.py index 2f4dc9ddfa..28ff2e6f4d 100644 --- a/module/config/config_generated.py +++ b/module/config/config_generated.py @@ -57,7 +57,7 @@ class GeneratedConfig: # Group `Log` Log_LogKeepCount = 7 Log_LogBackUpMethod = 'delete' # delete, zip, none - Log_ZipMethod = 'bz2' # bz2, gzip, none + Log_ZipMethod = 'bz2' # bz2, gzip # Group `Retirement` Retirement_RetireMode = 'one_click_retire' # one_click_retire, enhance, old_retire diff --git a/module/config/i18n/en-US.json b/module/config/i18n/en-US.json index cdaea0921c..2dbe3aac31 100644 --- a/module/config/i18n/en-US.json +++ b/module/config/i18n/en-US.json @@ -575,26 +575,25 @@ }, "Log": { "_info": { - "name": "Log._info.name", - "help": "Log._info.help" + "name": "Log", + "help": "General settings, effective after the scheduler is restarted" }, "LogKeepCount": { - "name": "Log.LogKeepCount.name", - "help": "Log.LogKeepCount.help" + "name": "Count of log rotation", + "help": "Number of days to rotate logs" }, "LogBackUpMethod": { - "name": "Log.LogBackUpMethod.name", - "help": "Log.LogBackUpMethod.help", + "name": "Log backup method", + "help": "Back up logs or delete expired logs directly", "delete": "delete", - "zip": "zip", - "none": "none" + "zip": "create zip in ./log/bak", + "none": "copy logs to ./log/bak" }, "ZipMethod": { - "name": "Log.ZipMethod.name", + "name": "Zip Method", "help": "Log.ZipMethod.help", "bz2": "bz2", - "gzip": "gzip", - "none": "none" + "gzip": "gzip" } }, "Retirement": { diff --git a/module/config/i18n/ja-JP.json b/module/config/i18n/ja-JP.json index 46174a5ee8..ad853525b9 100644 --- a/module/config/i18n/ja-JP.json +++ b/module/config/i18n/ja-JP.json @@ -593,8 +593,7 @@ "name": "Log.ZipMethod.name", "help": "Log.ZipMethod.help", "bz2": "bz2", - "gzip": "gzip", - "none": "none" + "gzip": "gzip" } }, "Retirement": { diff --git a/module/config/i18n/zh-CN.json b/module/config/i18n/zh-CN.json index 0f0510d022..4031784c63 100644 --- a/module/config/i18n/zh-CN.json +++ b/module/config/i18n/zh-CN.json @@ -575,26 +575,25 @@ }, "Log": { "_info": { - "name": "Log._info.name", - "help": "Log._info.help" + "name": "日志", + "help": "通用设置，调度器重启后生效" }, "LogKeepCount": { - "name": "Log.LogKeepCount.name", - "help": "Log.LogKeepCount.help" + "name": "Log保留数量", + "help": "Log过期时间(天)" }, "LogBackUpMethod": { - "name": "Log.LogBackUpMethod.name", - "help": "Log.LogBackUpMethod.help", - "delete": "delete", - "zip": "zip", - "none": "none" + "name": "Log处理方式", + "help": "备份目录为./log/bak", + "delete": "删除", + "zip": "压缩备份", + "none": "拷贝备份" }, "ZipMethod": { - "name": "Log.ZipMethod.name", - "help": "Log.ZipMethod.help", + "name": "压缩格式", + "help": "", "bz2": "bz2", - "gzip": "gzip", - "none": "none" + "gzip": "gzip" } }, "Retirement": { diff --git a/module/config/i18n/zh-TW.json b/module/config/i18n/zh-TW.json index 012e25deaf..652ccf224e 100644 --- a/module/config/i18n/zh-TW.json +++ b/module/config/i18n/zh-TW.json @@ -575,26 +575,25 @@ }, "Log": { "_info": { - "name": "Log._info.name", - "help": "Log._info.help" + "name": "Log", + "help": "通用設置，調度器重啟後生效" }, "LogKeepCount": { - "name": "Log.LogKeepCount.name", - "help": "Log.LogKeepCount.help" + "name": "Log保留數量", + "help": "Log過期時間(天)" }, "LogBackUpMethod": { - "name": "Log.LogBackUpMethod.name", - "help": "Log.LogBackUpMethod.help", - "delete": "delete", - "zip": "zip", - "none": "none" + "name": "Log處理方式", + "help": "備份目錄為./log/bak", + "delete": "刪除", + "zip": "壓縮備份", + "none": "拷貝備份" }, "ZipMethod": { - "name": "Log.ZipMethod.name", - "help": "Log.ZipMethod.help", + "name": "壓縮格式", + "help": "", "bz2": "bz2", - "gzip": "gzip", - "none": "none" + "gzip": "gzip" } }, "Retirement": { diff --git a/module/logger.py b/module/logger.py index 82c16f08c7..7dee2a93e5 100644 --- a/module/logger.py +++ b/module/logger.py @@ -98,7 +98,6 @@ def handle(self, record: logging.LogRecord) -> bool: class RichTimedRotatingHandler(TimedRotatingFileHandler): ZIPMAP = { - "none": (open, ""), "gzip": (gzip.open, "gz"), "bz2" : (bz2.open, "bz2") } @@ -132,7 +131,6 @@ def __init__(self, bak="none", compression="bz2", *args, **kwargs) -> None: # Override the initial rolloverAt self.rolloverAt = time.time() self.doRollover() - # self.rolloverAt = self.computeRollover(int(time.time())) # Close unnecessary stream self.stream.close() @@ -159,7 +157,7 @@ def getFilesToDelete(self) -> list: result.sort() result = result[: len(result) - self.backupCount] return result - + def doRollover(self) -> None: """ Do a rollover.\n @@ -183,7 +181,7 @@ def doRollover(self) -> None: else: addend = -3600 timeTuple = time.localtime(t + addend) - + path = pathlib.Path(self.baseFilename) # 2021-08-01 + _ + alas.txt -> "2021-08-01_alas.txt" newPath = path.with_name( @@ -213,40 +211,45 @@ def doRollover(self) -> None: newRolloverAt += addend self.rolloverAt = newRolloverAt - thread = threading.Thread(target=self._compress, args=(self.log_file, self.compression, self.bak,)) + thread = threading.Thread( + target=self._compress, + args=(self.log_file, self.compression, self.bak,) + ) thread.daemon = True thread.start() self.log_file = str(newPath.resolve()) - def _compress(self, file, compression="none", bak="none") -> None: + def _compress(self, file, compression="gzip", bak="none") -> None: """ Compress a file with gzip\n If file is None (In the initialization), compress the last log file\n Template: ./log/2021-08-01_alas.txt to ./log/bak/2021-08-01_alas.txt.gz """ basePath = pathlib.Path(self.baseFilename) + if bak == "delete": + return if file is None: logFiles = [file for file in basePath.parent.glob("*_" + basePath.name)] if len(logFiles) < 2: return file = sorted(logFiles, key=lambda x: str(x))[-2] - logger.info(logFiles) try: logFile = pathlib.Path(file) parent = logFile.parent cmpFunc, ext = self.ZIPMAP.get(compression, (gzip.open, "gz")) - zipFile = parent.joinpath("bak").joinpath(logFile.name).with_suffix('.'+ext) + zipFile = parent.joinpath("bak").joinpath(logFile.name).with_suffix("." + ext) - if not zipFile.exists() and compression != "none" and bak == "zip": - (parent / "bak").mkdir(exist_ok=True) - with logFile.open("rb") as f_in: - with cmpFunc(zipFile, "wb") as f_out: - shutil.copyfileobj(f_in, f_out) if bak == "none": shutil.copy2(logFile, zipFile.with_name(logFile.name)) + return + elif bak == "zip": + if not zipFile.exists(): + (parent / "bak").mkdir(exist_ok=True) + with logFile.open("rb") as f_in: + with cmpFunc(zipFile, "wb") as f_out: + shutil.copyfileobj(f_in, f_out) except Exception as e: logger.exception(e) - def print(self, *objects: ConsoleRenderable, **kwargs) -> None: Console.print(self.console, *objects, **kwargs) @@ -396,24 +399,24 @@ def set_file_logger(name=pyw_name): bkm = data_dict.get("General", {}).get("Log", {}).get("LogBackUpMethod") bak_method = bkm if bkm is not None and bkm in ["none", "delete", "zip"] else "none" method = data_dict.get("General", {}).get("Log", {}).get("ZipMethod") - zip_method = method if method is not None and method in ["none", "gzip", "bz2"] else "none" + zip_method = method if method is not None and method in ["gzip", "bz2"] else "gzip" except Exception as e: - count=7 - bak_method="none" - zip_method="none" + count = 7 + bak_method = "none" + zip_method = "gzip" logger.exception(e) else: - count=7 - zip_method="none" - bak_method="none" - + count = 7 + zip_method = "none" + bak_method = "gzip" + hdlr = RichTimedRotatingHandler( bak=bak_method, filename=str(log_file), compression=zip_method, - # when="midnight", - when="S", - interval=10, + when="midnight", + # when="S", + interval=1, backupCount=count, encoding="utf-8", ) From e37e6657d4522b955b1e7b5d403d24945cf7ea82 Mon Sep 17 00:00:00 2001 From: izum1 Date: Sat, 4 May 2024 21:09:31 +0800 Subject: [PATCH 05/16] fix: fix some bugs of file logger --- module/logger.py | 55 +++++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/module/logger.py b/module/logger.py index 7dee2a93e5..cf02d4b138 100644 --- a/module/logger.py +++ b/module/logger.py @@ -8,7 +8,6 @@ import pathlib import shutil import sys -import tarfile import threading import time from logging.handlers import TimedRotatingFileHandler @@ -223,17 +222,18 @@ def _compress(self, file, compression="gzip", bak="none") -> None: """ Compress a file with gzip\n If file is None (In the initialization), compress the last log file\n - Template: ./log/2021-08-01_alas.txt to ./log/bak/2021-08-01_alas.txt.gz + Template: ./log/2021-08-01_alas.txt to ./log/bak/2021-08-01_alas.gz """ - basePath = pathlib.Path(self.baseFilename) if bak == "delete": return - if file is None: - logFiles = [file for file in basePath.parent.glob("*_" + basePath.name)] - if len(logFiles) < 2: - return - file = sorted(logFiles, key=lambda x: str(x))[-2] + basePath = pathlib.Path(self.baseFilename) try: + if file is None: + logFiles = [file for file in basePath.parent.glob("*_" + basePath.name)] + if len(logFiles) < 2: + return + file = sorted(logFiles, key=lambda x: str(x))[-2] + logFile = pathlib.Path(file) parent = logFile.parent cmpFunc, ext = self.ZIPMAP.get(compression, (gzip.open, "gz")) @@ -373,7 +373,7 @@ def set_file_logger(name=pyw_name): log_dir = pathlib.Path("./log") log_file = log_dir.joinpath(f"{pname}.txt" if name == "gui" else f"{name}.txt") - os.makedirs(log_dir, exist_ok=True) + log_dir.mkdir(parents=True, exist_ok=True) # These process needn't to save log file in Windows process = ["SyncManager-", "MainProcess", "Process-"] if os.name == "nt" else [] @@ -381,41 +381,34 @@ def set_file_logger(name=pyw_name): hdlr = RichFileHandler(console=Console(file=io.StringIO())) logger.addHandler(hdlr) logger.log_file = str(log_file.resolve()) - if os.path.exists(log_file): - os.remove(log_file) + if log_file.exists(): + log_file.unlink() return - valid_cfg = [ - file - for file in pathlib.Path("./config").glob("*.json") - if not file.name.endswith((".maa.json", ".fpy.json", "template.json")) - ] - if len(valid_cfg) > 0: + config_file = next((f for f in pathlib.Path("./config").glob("*.json")), None) + if config_file: try: - with open(str(valid_cfg[0]),"r") as f: - data_dict = json.load(f) - cnt = data_dict.get("General", {}).get("Log", {}).get("LogKeepCount") - count = cnt if cnt is not None and isinstance(cnt, int) and cnt >= 0 else 7 - bkm = data_dict.get("General", {}).get("Log", {}).get("LogBackUpMethod") - bak_method = bkm if bkm is not None and bkm in ["none", "delete", "zip"] else "none" - method = data_dict.get("General", {}).get("Log", {}).get("ZipMethod") - zip_method = method if method is not None and method in ["gzip", "bz2"] else "gzip" + with open(config_file, "r") as f: + config = json.load(f) + log_config = config.get("General", {}).get("Log", {}) + count = log_config.get("LogKeepCount", 7) + bak_method = log_config.get("LogBackUpMethod", "none") + zip_method = log_config.get("ZipMethod", "gzip") except Exception as e: + logging.exception(e) count = 7 bak_method = "none" zip_method = "gzip" - logger.exception(e) else: count = 7 - zip_method = "none" - bak_method = "gzip" + bak_method = "none" + zip_method = "gzip" hdlr = RichTimedRotatingHandler( bak=bak_method, filename=str(log_file), compression=zip_method, when="midnight", - # when="S", interval=1, backupCount=count, encoding="utf-8", @@ -432,8 +425,8 @@ def set_file_logger(name=pyw_name): logger.log_file = hdlr.log_file # Delete the default log file after initialize the handler - if os.path.exists(log_file): - os.remove(log_file) + if log_file.exists(): + log_file.unlink() def set_func_logger(func): From 08eb2c7f4dbc733fca856f0d7345d8483445e845 Mon Sep 17 00:00:00 2001 From: izum1 Date: Sat, 4 May 2024 21:24:19 +0800 Subject: [PATCH 06/16] style: fix style in logger.py --- module/logger.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/module/logger.py b/module/logger.py index cf02d4b138..5df20fbf40 100644 --- a/module/logger.py +++ b/module/logger.py @@ -381,8 +381,6 @@ def set_file_logger(name=pyw_name): hdlr = RichFileHandler(console=Console(file=io.StringIO())) logger.addHandler(hdlr) logger.log_file = str(log_file.resolve()) - if log_file.exists(): - log_file.unlink() return config_file = next((f for f in pathlib.Path("./config").glob("*.json")), None) @@ -414,20 +412,11 @@ def set_file_logger(name=pyw_name): encoding="utf-8", ) - logger.handlers = [ - h - for h in logger.handlers - if not isinstance( - h, (logging.FileHandler, RichTimedRotatingHandler, RichFileHandler) - ) - ] + logger.handlers = [ h for h in logger.handlers if not isinstance( + h, (logging.FileHandler, RichTimedRotatingHandler, RichFileHandler))] logger.addHandler(hdlr) logger.log_file = hdlr.log_file - # Delete the default log file after initialize the handler - if log_file.exists(): - log_file.unlink() - def set_func_logger(func): console = HTMLConsole( From 0678c00fd9ab03fe3ba147e825b2f4f5d34fffaa Mon Sep 17 00:00:00 2001 From: izum1 Date: Sat, 4 May 2024 21:26:46 +0800 Subject: [PATCH 07/16] fix: fix some bugs of file logger --- module/logger.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/module/logger.py b/module/logger.py index 5df20fbf40..f6c4a6e234 100644 --- a/module/logger.py +++ b/module/logger.py @@ -381,6 +381,8 @@ def set_file_logger(name=pyw_name): hdlr = RichFileHandler(console=Console(file=io.StringIO())) logger.addHandler(hdlr) logger.log_file = str(log_file.resolve()) + if log_file.exists(): + log_file.unlink() return config_file = next((f for f in pathlib.Path("./config").glob("*.json")), None) @@ -416,6 +418,8 @@ def set_file_logger(name=pyw_name): h, (logging.FileHandler, RichTimedRotatingHandler, RichFileHandler))] logger.addHandler(hdlr) logger.log_file = hdlr.log_file + if log_file.exists(): + log_file.unlink() def set_func_logger(func): From d01442c2555d5773a67672a918cdeba25c04d653 Mon Sep 17 00:00:00 2001 From: izum1 Date: Sat, 4 May 2024 22:25:51 +0800 Subject: [PATCH 08/16] feat: add backup limit of error log --- alas.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/alas.py b/alas.py index 46a91e4f65..04d3c8708e 100644 --- a/alas.py +++ b/alas.py @@ -1,5 +1,6 @@ import os import re +import shutil import threading import time from datetime import datetime, timedelta @@ -131,20 +132,31 @@ def run(self, command): ) exit(1) + def keep_last_errlog(self, folder_path, n): + folders = [ + os.path.join(folder_path, f) + for f in os.listdir(folder_path) + if os.path.isdir(os.path.join(folder_path, f)) + ] + for folder in folders[:-n]: + shutil.rmtree(folder) + def save_error_log(self): """ Save last 60 screenshots in ./log/error/ Save logs to ./log/error//log.txt """ + import pathlib from module.base.utils import save_image from module.handler.sensitive_info import (handle_sensitive_image, handle_sensitive_logs) if self.config.Error_SaveError: - if not os.path.exists('./log/error'): - os.mkdir('./log/error') - folder = f'./log/error/{int(time.time() * 1000)}' + config_name = self.config_name + config_folder = pathlib.Path(f"./log/error/{config_name}") + config_folder.mkdir(parents=True, exist_ok=True) + folder = config_folder.joinpath(int(time.time() * 1000)) logger.warning(f'Saving error: {folder}') - os.mkdir(folder) + for data in self.device.screenshot_deque: image_time = datetime.strftime(data['time'], '%Y-%m-%d_%H-%M-%S-%f') image = handle_sensitive_image(data['image']) @@ -160,6 +172,7 @@ def save_error_log(self): lines = handle_sensitive_logs(lines) with open(f'{folder}/log.txt', 'w', encoding='utf-8') as f: f.writelines(lines) + self.keep_last_log(config_folder, 6) def restart(self): from module.handler.login import LoginHandler From e3250567cc74ccc60202896434386f2ffd3f19be Mon Sep 17 00:00:00 2001 From: izum1 Date: Sun, 5 May 2024 14:16:56 +0800 Subject: [PATCH 09/16] feat: add limit to the number of Error files --- alas.py | 26 +++++++++++++------ config/template.json | 1 + module/config/argument/args.json | 4 +++ module/config/argument/argument.yaml | 1 + module/config/config_generated.py | 1 + module/config/i18n/en-US.json | 4 +++ module/config/i18n/ja-JP.json | 4 +++ module/config/i18n/zh-CN.json | 6 ++++- module/config/i18n/zh-TW.json | 6 ++++- module/logger.py | 39 +++++++++++++++------------- 10 files changed, 64 insertions(+), 28 deletions(-) diff --git a/alas.py b/alas.py index 04d3c8708e..152eec68eb 100644 --- a/alas.py +++ b/alas.py @@ -132,7 +132,18 @@ def run(self, command): ) exit(1) - def keep_last_errlog(self, folder_path, n): + def keep_last_errlog(self, folder_path, n: int = 30): + """ + + Keep last n folders in folder_path, delete others. + If n is negative or 0, do nothing.(Keep all errlog folders) + + Args: + folder_path (str): Path to folder.\n + n (int): Number of folders to keep. + """ + if n <= 0: + return folders = [ os.path.join(folder_path, f) for f in os.listdir(folder_path) @@ -143,18 +154,17 @@ def keep_last_errlog(self, folder_path, n): def save_error_log(self): """ - Save last 60 screenshots in ./log/error/ - Save logs to ./log/error//log.txt + Save last 60 screenshots in ./log/error// + Save logs to ./log/error///log.txt """ import pathlib from module.base.utils import save_image from module.handler.sensitive_info import (handle_sensitive_image, handle_sensitive_logs) if self.config.Error_SaveError: - config_name = self.config_name - config_folder = pathlib.Path(f"./log/error/{config_name}") - config_folder.mkdir(parents=True, exist_ok=True) - folder = config_folder.joinpath(int(time.time() * 1000)) + config_folder = pathlib.Path(f"./log/error/{self.config_name}") + folder = config_folder.joinpath(str(int(time.time() * 1000))) + folder.mkdir(parents=True, exist_ok=True) logger.warning(f'Saving error: {folder}') for data in self.device.screenshot_deque: @@ -172,7 +182,7 @@ def save_error_log(self): lines = handle_sensitive_logs(lines) with open(f'{folder}/log.txt', 'w', encoding='utf-8') as f: f.writelines(lines) - self.keep_last_log(config_folder, 6) + self.keep_last_errlog(config_folder, self.config.Error_SaveErrorCount) def restart(self): from module.handler.login import LoginHandler diff --git a/config/template.json b/config/template.json index 52603643c4..1cfccc1112 100644 --- a/config/template.json +++ b/config/template.json @@ -17,6 +17,7 @@ "Error": { "HandleError": true, "SaveError": true, + "SaveErrorCount": 30, "OnePushConfig": "provider: null", "ScreenshotLength": 1 }, diff --git a/module/config/argument/args.json b/module/config/argument/args.json index e6b82dc06d..b79e8b021f 100644 --- a/module/config/argument/args.json +++ b/module/config/argument/args.json @@ -179,6 +179,10 @@ "type": "checkbox", "value": true }, + "SaveErrorCount": { + "type": "input", + "value": 30 + }, "OnePushConfig": { "type": "textarea", "value": "provider: null", diff --git a/module/config/argument/argument.yaml b/module/config/argument/argument.yaml index f2609053f7..79c042ac87 100644 --- a/module/config/argument/argument.yaml +++ b/module/config/argument/argument.yaml @@ -83,6 +83,7 @@ EmulatorInfo: Error: HandleError: true SaveError: true + SaveErrorCount: 30 OnePushConfig: type: textarea mode: yaml diff --git a/module/config/config_generated.py b/module/config/config_generated.py index 28ff2e6f4d..86472be8f0 100644 --- a/module/config/config_generated.py +++ b/module/config/config_generated.py @@ -34,6 +34,7 @@ class GeneratedConfig: # Group `Error` Error_HandleError = True Error_SaveError = True + Error_SaveErrorCount = 30 Error_OnePushConfig = 'provider: null' Error_ScreenshotLength = 1 diff --git a/module/config/i18n/en-US.json b/module/config/i18n/en-US.json index 2dbe3aac31..bd5bdab1cf 100644 --- a/module/config/i18n/en-US.json +++ b/module/config/i18n/en-US.json @@ -475,6 +475,10 @@ "name": "Record Exception", "help": "Records exception and log into directory for review or sharing" }, + "SaveErrorCount": { + "name": "Error storage limit", + "help": "Error logs that exceed will be deleted, unlimited if it is negative" + }, "OnePushConfig": { "name": "Error notify config", "help": "When Alas cannot handle exception, send a message through Onepush. Configuration document: \nhttps://github.com/LmeSzinc/AzurLaneAutoScript/wiki/Onepush-configuration-%5BEN%5D" diff --git a/module/config/i18n/ja-JP.json b/module/config/i18n/ja-JP.json index ad853525b9..1d88c433ec 100644 --- a/module/config/i18n/ja-JP.json +++ b/module/config/i18n/ja-JP.json @@ -475,6 +475,10 @@ "name": "Error.SaveError.name", "help": "Error.SaveError.help" }, + "SaveErrorCount": { + "name": "Error.SaveErrorCount.name", + "help": "Error.SaveErrorCount.help" + }, "OnePushConfig": { "name": "Error.OnePushConfig.name", "help": "Error.OnePushConfig.help" diff --git a/module/config/i18n/zh-CN.json b/module/config/i18n/zh-CN.json index 4031784c63..1c8d3fa26b 100644 --- a/module/config/i18n/zh-CN.json +++ b/module/config/i18n/zh-CN.json @@ -475,6 +475,10 @@ "name": "出错时，保存 Log 和截图", "help": "" }, + "SaveErrorCount": { + "name": "Error Log 保存上限", + "help": "超出上限的错误日志将被删除，若为0或负值则没有限制" + }, "OnePushConfig": { "name": "错误推送设置", "help": "发生无法处理的异常后，使用 Onepush 推送一条错误信息。配置方法见文档：https://github.com/LmeSzinc/AzurLaneAutoScript/wiki/Onepush-configuration-%5BCN%5D" @@ -583,7 +587,7 @@ "help": "Log过期时间(天)" }, "LogBackUpMethod": { - "name": "Log处理方式", + "name": "过期Log处理方式", "help": "备份目录为./log/bak", "delete": "删除", "zip": "压缩备份", diff --git a/module/config/i18n/zh-TW.json b/module/config/i18n/zh-TW.json index 652ccf224e..b4c04e062f 100644 --- a/module/config/i18n/zh-TW.json +++ b/module/config/i18n/zh-TW.json @@ -475,6 +475,10 @@ "name": "出錯時，保存 Log 和截圖", "help": "" }, + "SaveErrorCount": { + "name": "Error Log 保存上限", + "help": "超出上限的錯誤日志將被刪除, 若爲0或負數則沒有限制" + }, "OnePushConfig": { "name": "錯誤推送設定", "help": "發生無法處理的異常後，使用 Onepush 推送错误消息。設定參考文檔：https://github.com/LmeSzinc/AzurLaneAutoScript/wiki/Onepush-configuration-%5BCN%5D" @@ -576,7 +580,7 @@ "Log": { "_info": { "name": "Log", - "help": "通用設置，調度器重啟後生效" + "help": "通用設定，調度器重啟後生效" }, "LogKeepCount": { "name": "Log保留數量", diff --git a/module/logger.py b/module/logger.py index f6c4a6e234..3571b1d237 100644 --- a/module/logger.py +++ b/module/logger.py @@ -254,7 +254,6 @@ def _compress(self, file, compression="gzip", bak="none") -> None: def print(self, *objects: ConsoleRenderable, **kwargs) -> None: Console.print(self.console, *objects, **kwargs) - # @override def emit(self, record: logging.LogRecord) -> None: try: if self.shouldRollover(record): @@ -385,6 +384,26 @@ def set_file_logger(name=pyw_name): log_file.unlink() return + count, bak_method, zip_method = _read_file_logger_config(json) + + hdlr = RichTimedRotatingHandler( + bak=bak_method, + filename=str(log_file), + compression=zip_method, + when="midnight", + interval=1, + backupCount=count, + encoding="utf-8", + ) + + logger.handlers = [ h for h in logger.handlers if not isinstance( + h, (logging.FileHandler, RichTimedRotatingHandler, RichFileHandler))] + logger.addHandler(hdlr) + logger.log_file = hdlr.log_file + if log_file.exists(): + log_file.unlink() + +def _read_file_logger_config(json): config_file = next((f for f in pathlib.Path("./config").glob("*.json")), None) if config_file: try: @@ -403,23 +422,7 @@ def set_file_logger(name=pyw_name): count = 7 bak_method = "none" zip_method = "gzip" - - hdlr = RichTimedRotatingHandler( - bak=bak_method, - filename=str(log_file), - compression=zip_method, - when="midnight", - interval=1, - backupCount=count, - encoding="utf-8", - ) - - logger.handlers = [ h for h in logger.handlers if not isinstance( - h, (logging.FileHandler, RichTimedRotatingHandler, RichFileHandler))] - logger.addHandler(hdlr) - logger.log_file = hdlr.log_file - if log_file.exists(): - log_file.unlink() + return count,bak_method,zip_method def set_func_logger(func): From 51cbfd7268c8729380c038b1f7e9f15f246775ad Mon Sep 17 00:00:00 2001 From: izum1 Date: Sun, 5 May 2024 14:23:59 +0800 Subject: [PATCH 10/16] fix: fix bugs in logger.py --- module/logger.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/module/logger.py b/module/logger.py index 3571b1d237..86c57cdcdb 100644 --- a/module/logger.py +++ b/module/logger.py @@ -238,13 +238,12 @@ def _compress(self, file, compression="gzip", bak="none") -> None: parent = logFile.parent cmpFunc, ext = self.ZIPMAP.get(compression, (gzip.open, "gz")) zipFile = parent.joinpath("bak").joinpath(logFile.name).with_suffix("." + ext) - + (parent / "bak").mkdir(exist_ok=True) if bak == "none": shutil.copy2(logFile, zipFile.with_name(logFile.name)) return elif bak == "zip": if not zipFile.exists(): - (parent / "bak").mkdir(exist_ok=True) with logFile.open("rb") as f_in: with cmpFunc(zipFile, "wb") as f_out: shutil.copyfileobj(f_in, f_out) From 814b580af054a114f9e40e12cdf17539b7fa9273 Mon Sep 17 00:00:00 2001 From: izum1 Date: Sun, 5 May 2024 17:34:11 +0800 Subject: [PATCH 11/16] refactor: refactor file logger compress --- module/config/argument/args.json | 6 +- module/config/argument/argument.yaml | 4 +- module/config/config_generated.py | 4 +- module/config/i18n/en-US.json | 6 +- module/config/i18n/ja-JP.json | 6 +- module/config/i18n/zh-CN.json | 8 +- module/config/i18n/zh-TW.json | 8 +- module/logger.py | 119 ++++++++++++++------------- 8 files changed, 90 insertions(+), 71 deletions(-) diff --git a/module/config/argument/args.json b/module/config/argument/args.json index b79e8b021f..8345746379 100644 --- a/module/config/argument/args.json +++ b/module/config/argument/args.json @@ -311,7 +311,7 @@ "option": [ "delete", "zip", - "none" + "copy" ] }, "ZipMethod": { @@ -319,7 +319,9 @@ "value": "bz2", "option": [ "bz2", - "gzip" + "gzip", + "xz", + "zip" ] } }, diff --git a/module/config/argument/argument.yaml b/module/config/argument/argument.yaml index 79c042ac87..f03712bded 100644 --- a/module/config/argument/argument.yaml +++ b/module/config/argument/argument.yaml @@ -124,10 +124,10 @@ Log: LogKeepCount: 7 LogBackUpMethod: value: delete - option: [ delete, zip, none ] + option: [ delete, zip, copy ] ZipMethod: value: bz2 - option: [ bz2, gzip ] + option: [ bz2, gzip, xz, zip ] Retirement: RetireMode: value: one_click_retire diff --git a/module/config/config_generated.py b/module/config/config_generated.py index 86472be8f0..835aad071a 100644 --- a/module/config/config_generated.py +++ b/module/config/config_generated.py @@ -57,8 +57,8 @@ class GeneratedConfig: # Group `Log` Log_LogKeepCount = 7 - Log_LogBackUpMethod = 'delete' # delete, zip, none - Log_ZipMethod = 'bz2' # bz2, gzip + Log_LogBackUpMethod = 'delete' # delete, zip, copy + Log_ZipMethod = 'bz2' # bz2, gzip, xz, zip # Group `Retirement` Retirement_RetireMode = 'one_click_retire' # one_click_retire, enhance, old_retire diff --git a/module/config/i18n/en-US.json b/module/config/i18n/en-US.json index bd5bdab1cf..c3b1997068 100644 --- a/module/config/i18n/en-US.json +++ b/module/config/i18n/en-US.json @@ -591,13 +591,15 @@ "help": "Back up logs or delete expired logs directly", "delete": "delete", "zip": "create zip in ./log/bak", - "none": "copy logs to ./log/bak" + "copy": "copy to ./log/bak" }, "ZipMethod": { "name": "Zip Method", "help": "Log.ZipMethod.help", "bz2": "bz2", - "gzip": "gzip" + "gzip": "gzip", + "xz": "xz", + "zip": "zip" } }, "Retirement": { diff --git a/module/config/i18n/ja-JP.json b/module/config/i18n/ja-JP.json index 1d88c433ec..53940a43d6 100644 --- a/module/config/i18n/ja-JP.json +++ b/module/config/i18n/ja-JP.json @@ -591,13 +591,15 @@ "help": "Log.LogBackUpMethod.help", "delete": "delete", "zip": "zip", - "none": "none" + "copy": "copy" }, "ZipMethod": { "name": "Log.ZipMethod.name", "help": "Log.ZipMethod.help", "bz2": "bz2", - "gzip": "gzip" + "gzip": "gzip", + "xz": "xz", + "zip": "zip" } }, "Retirement": { diff --git a/module/config/i18n/zh-CN.json b/module/config/i18n/zh-CN.json index 1c8d3fa26b..6951fb17e4 100644 --- a/module/config/i18n/zh-CN.json +++ b/module/config/i18n/zh-CN.json @@ -580,7 +580,7 @@ "Log": { "_info": { "name": "日志", - "help": "通用设置，调度器重启后生效" + "help": "调度器重启后生效" }, "LogKeepCount": { "name": "Log保留数量", @@ -591,13 +591,15 @@ "help": "备份目录为./log/bak", "delete": "删除", "zip": "压缩备份", - "none": "拷贝备份" + "copy": "拷贝备份" }, "ZipMethod": { "name": "压缩格式", "help": "", "bz2": "bz2", - "gzip": "gzip" + "gzip": "gzip", + "xz": "xz", + "zip": "zip" } }, "Retirement": { diff --git a/module/config/i18n/zh-TW.json b/module/config/i18n/zh-TW.json index b4c04e062f..753196e3a4 100644 --- a/module/config/i18n/zh-TW.json +++ b/module/config/i18n/zh-TW.json @@ -580,7 +580,7 @@ "Log": { "_info": { "name": "Log", - "help": "通用設定，調度器重啟後生效" + "help": "調度器重啟後生效" }, "LogKeepCount": { "name": "Log保留數量", @@ -591,13 +591,15 @@ "help": "備份目錄為./log/bak", "delete": "刪除", "zip": "壓縮備份", - "none": "拷貝備份" + "copy": "copy" }, "ZipMethod": { "name": "壓縮格式", "help": "", "bz2": "bz2", - "gzip": "gzip" + "gzip": "gzip", + "xz": "xz", + "zip": "zip" } }, "Retirement": { diff --git a/module/logger.py b/module/logger.py index 86c57cdcdb..869289f7fb 100644 --- a/module/logger.py +++ b/module/logger.py @@ -1,16 +1,16 @@ -import bz2 import datetime -import gzip import io import logging import multiprocessing import os -import pathlib import shutil import sys +import tarfile import threading import time +import zipfile from logging.handlers import TimedRotatingFileHandler +from pathlib import Path from typing import Callable, List from rich.console import Console, ConsoleOptions, ConsoleRenderable, NewLine @@ -97,10 +97,13 @@ def handle(self, record: logging.LogRecord) -> bool: class RichTimedRotatingHandler(TimedRotatingFileHandler): ZIPMAP = { - "gzip": (gzip.open, "gz"), - "bz2" : (bz2.open, "bz2") + "gzip": "gz", + "gz" : "gz", + "bz2" : "bz2", + "xz": "xz", + "zip": "zip", } - def __init__(self, bak="none", compression="bz2", *args, **kwargs) -> None: + def __init__(self, bak="copy", compression="bz2", *args, **kwargs) -> None: TimedRotatingFileHandler.__init__(self, *args, **kwargs) self.console = Console(file=io.StringIO(), no_color=True, highlight=False, width=119) self.richd = RichHandler( @@ -124,6 +127,7 @@ def __init__(self, bak="none", compression="bz2", *args, **kwargs) -> None: self.console = self.richd.console # To handle the API of alas.save_error_log() self.log_file = None + # For expire method self.bak = bak.lower() self.compression = compression.lower() @@ -135,7 +139,7 @@ def __init__(self, bak="none", compression="bz2", *args, **kwargs) -> None: self.stream.close() self.stream = None - def getFilesToDelete(self) -> list: + def getFilesToDelete(self) -> List[Path]: """ Determine the files to delete when rolling over.\n Override the original method to use RichHandler @@ -149,7 +153,7 @@ def getFilesToDelete(self) -> list: if fileName[-plen:] == suffix: prefix = fileName[:-plen] if self.extMatch.match(prefix): - result.append(os.path.join(dirName, fileName)) + result.append(Path(dirName).joinpath(fileName).resolve()) if len(result) < self.backupCount: result = [] else: @@ -181,7 +185,7 @@ def doRollover(self) -> None: addend = -3600 timeTuple = time.localtime(t + addend) - path = pathlib.Path(self.baseFilename) + path = Path(self.baseFilename) # 2021-08-01 + _ + alas.txt -> "2021-08-01_alas.txt" newPath = path.with_name( time.strftime(self.suffix, timeTuple) + "_" + path.name @@ -191,8 +195,9 @@ def doRollover(self) -> None: ) if self.backupCount > 0: - for s in self.getFilesToDelete(): - os.remove(s) + files = self.getFilesToDelete() + if files: + threading.Thread(target=self.expire, args=(files,), daemon=True).start() newRolloverAt = self.computeRollover(currentTime) while newRolloverAt <= currentTime: @@ -210,43 +215,46 @@ def doRollover(self) -> None: newRolloverAt += addend self.rolloverAt = newRolloverAt - thread = threading.Thread( - target=self._compress, - args=(self.log_file, self.compression, self.bak,) - ) - thread.daemon = True - thread.start() self.log_file = str(newPath.resolve()) - def _compress(self, file, compression="gzip", bak="none") -> None: + def expire(self, files: List[Path]) -> None: """ - Compress a file with gzip\n - If file is None (In the initialization), compress the last log file\n - Template: ./log/2021-08-01_alas.txt to ./log/bak/2021-08-01_alas.gz + Remove or backup the expired log files """ - if bak == "delete": + basePath = Path(self.baseFilename) + bakPath = basePath.parent / "bak" + bakPath.mkdir(parents=True, exist_ok=True) + if self.bak == "delete": + for file in files: + file.unlink() + return + elif self.bak == "copy": + for file in files: + dst = bakPath.joinpath(file.name) + if not dst.exists(): + shutil.copy2(file, dst) + file.unlink() return - basePath = pathlib.Path(self.baseFilename) try: - if file is None: - logFiles = [file for file in basePath.parent.glob("*_" + basePath.name)] - if len(logFiles) < 2: - return - file = sorted(logFiles, key=lambda x: str(x))[-2] - - logFile = pathlib.Path(file) - parent = logFile.parent - cmpFunc, ext = self.ZIPMAP.get(compression, (gzip.open, "gz")) - zipFile = parent.joinpath("bak").joinpath(logFile.name).with_suffix("." + ext) - (parent / "bak").mkdir(exist_ok=True) - if bak == "none": - shutil.copy2(logFile, zipFile.with_name(logFile.name)) - return - elif bak == "zip": - if not zipFile.exists(): - with logFile.open("rb") as f_in: - with cmpFunc(zipFile, "wb") as f_out: - shutil.copyfileobj(f_in, f_out) + dates = [file.stem.split("_")[0] for file in files] + name = ( + min(dates) + "~" + max(dates) + "_" + basePath.name + if len(dates) > 1 + else files[0].name + ) + ext = self.ZIPMAP[self.compression] + if ext == "zip": + zipFile = bakPath.joinpath(name).with_suffix(".zip") + with zipfile.ZipFile(zipFile, "w", zipfile.ZIP_DEFLATED) as zipf: + for file in files: + zipf.write(file, arcname=file.name) + file.unlink() + else: + zipFile = bakPath.joinpath(name).with_suffix(".tar." + ext) + with tarfile.open(zipFile, "w:" + ext) as tar: + for file in files: + tar.add(file, arcname=file.name) + file.unlink() except Exception as e: logger.exception(e) @@ -362,14 +370,13 @@ def _set_file_logger(name=pyw_name): def set_file_logger(name=pyw_name): - import json if "_" in name: name = name.split("_", 1)[0] # Handler Windows : Windows have "SyncManager-N:N", "MainProcess", "Process-N", "gui" 4 Processes # There have no process named "gui", only "MainProcess" in Linux pname = multiprocessing.current_process().name.replace(":", "_") if os.name == "nt" else name - log_dir = pathlib.Path("./log") + log_dir = Path("./log") log_file = log_dir.joinpath(f"{pname}.txt" if name == "gui" else f"{name}.txt") log_dir.mkdir(parents=True, exist_ok=True) @@ -383,7 +390,7 @@ def set_file_logger(name=pyw_name): log_file.unlink() return - count, bak_method, zip_method = _read_file_logger_config(json) + count, bak_method, zip_method = _read_file_logger_config(pname) hdlr = RichTimedRotatingHandler( bak=bak_method, @@ -402,25 +409,27 @@ def set_file_logger(name=pyw_name): if log_file.exists(): log_file.unlink() -def _read_file_logger_config(json): - config_file = next((f for f in pathlib.Path("./config").glob("*.json")), None) - if config_file: +def _read_file_logger_config(process_name): + import json + cfg_name = "alas" if process_name == "gui" else process_name + config_file = Path("./config").joinpath(f"{cfg_name}.json") + if config_file.exists(): try: - with open(config_file, "r") as f: + with config_file.open("r") as f: config = json.load(f) log_config = config.get("General", {}).get("Log", {}) count = log_config.get("LogKeepCount", 7) - bak_method = log_config.get("LogBackUpMethod", "none") - zip_method = log_config.get("ZipMethod", "gzip") + bak_method = log_config.get("LogBackUpMethod", "copy") + zip_method = log_config.get("ZipMethod", "bz2") except Exception as e: logging.exception(e) count = 7 - bak_method = "none" - zip_method = "gzip" + bak_method = "copy" + zip_method = "bz2" else: count = 7 - bak_method = "none" - zip_method = "gzip" + bak_method = "zip" if process_name == "gui" else "copy" + zip_method = "bz2" return count,bak_method,zip_method From d35f873730200c50931bd4be647a900f27b716a1 Mon Sep 17 00:00:00 2001 From: izum1 Date: Sun, 5 May 2024 19:10:14 +0800 Subject: [PATCH 12/16] refactor: _read_file_logger_config --- module/logger.py | 66 +++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 35 deletions(-) diff --git a/module/logger.py b/module/logger.py index 869289f7fb..bee4fd9f67 100644 --- a/module/logger.py +++ b/module/logger.py @@ -1,5 +1,6 @@ import datetime import io +import json import logging import multiprocessing import os @@ -103,8 +104,9 @@ class RichTimedRotatingHandler(TimedRotatingFileHandler): "xz": "xz", "zip": "zip", } - def __init__(self, bak="copy", compression="bz2", *args, **kwargs) -> None: - TimedRotatingFileHandler.__init__(self, *args, **kwargs) + def __init__(self, pname:str, *args, **kwargs) -> None: + count, bak_method, zip_method = self._read_file_logger_config(pname) + TimedRotatingFileHandler.__init__(self, backupCount=count,* args, **kwargs) self.console = Console(file=io.StringIO(), no_color=True, highlight=False, width=119) self.richd = RichHandler( console=self.console, @@ -123,13 +125,11 @@ def __init__(self, bak="copy", compression="bz2", *args, **kwargs) -> None: datefmt="%Y-%m-%d %H:%M:%S", ) ) - # To handle the API of logger.print() - self.console = self.richd.console # To handle the API of alas.save_error_log() self.log_file = None # For expire method - self.bak = bak.lower() - self.compression = compression.lower() + self.bak = bak_method.lower() + self.compression = zip_method.lower() # Override the initial rolloverAt self.rolloverAt = time.time() @@ -138,7 +138,29 @@ def __init__(self, bak="copy", compression="bz2", *args, **kwargs) -> None: # Close unnecessary stream self.stream.close() self.stream = None - + + def _read_file_logger_config(self, process_name): + cfg_name = "alas" if process_name == "gui" else process_name + config_file = Path("./config").joinpath(f"{cfg_name}.json") + if config_file.exists(): + try: + with config_file.open("r") as f: + config = json.load(f) + log_config = config.get("General", {}).get("Log", {}) + count = log_config.get("LogKeepCount", 7) + bak_method = log_config.get("LogBackUpMethod", "copy") + zip_method = log_config.get("ZipMethod", "bz2") + except Exception as e: + logging.exception(e) + count = 7 + bak_method = "copy" + zip_method = "bz2" + else: + count = 7 + bak_method = "zip" if process_name == "gui" else "copy" + zip_method = "bz2" + return count, bak_method, zip_method + def getFilesToDelete(self) -> List[Path]: """ Determine the files to delete when rolling over.\n @@ -390,47 +412,21 @@ def set_file_logger(name=pyw_name): log_file.unlink() return - count, bak_method, zip_method = _read_file_logger_config(pname) - hdlr = RichTimedRotatingHandler( - bak=bak_method, + pname=name, filename=str(log_file), - compression=zip_method, when="midnight", interval=1, - backupCount=count, encoding="utf-8", ) - logger.handlers = [ h for h in logger.handlers if not isinstance( + logger.handlers = [h for h in logger.handlers if not isinstance( h, (logging.FileHandler, RichTimedRotatingHandler, RichFileHandler))] logger.addHandler(hdlr) logger.log_file = hdlr.log_file if log_file.exists(): log_file.unlink() -def _read_file_logger_config(process_name): - import json - cfg_name = "alas" if process_name == "gui" else process_name - config_file = Path("./config").joinpath(f"{cfg_name}.json") - if config_file.exists(): - try: - with config_file.open("r") as f: - config = json.load(f) - log_config = config.get("General", {}).get("Log", {}) - count = log_config.get("LogKeepCount", 7) - bak_method = log_config.get("LogBackUpMethod", "copy") - zip_method = log_config.get("ZipMethod", "bz2") - except Exception as e: - logging.exception(e) - count = 7 - bak_method = "copy" - zip_method = "bz2" - else: - count = 7 - bak_method = "zip" if process_name == "gui" else "copy" - zip_method = "bz2" - return count,bak_method,zip_method def set_func_logger(func): From 1b6c7e0c06c85dbf4077d752ad1d6241127e5150 Mon Sep 17 00:00:00 2001 From: izum1 Date: Sun, 5 May 2024 21:07:54 +0800 Subject: [PATCH 13/16] fix: some bugs in logger.py --- module/logger.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/module/logger.py b/module/logger.py index bee4fd9f67..98b1e71203 100644 --- a/module/logger.py +++ b/module/logger.py @@ -220,6 +220,7 @@ def doRollover(self) -> None: files = self.getFilesToDelete() if files: threading.Thread(target=self.expire, args=(files,), daemon=True).start() + # self.expire(files) newRolloverAt = self.computeRollover(currentTime) while newRolloverAt <= currentTime: @@ -394,22 +395,21 @@ def _set_file_logger(name=pyw_name): def set_file_logger(name=pyw_name): if "_" in name: name = name.split("_", 1)[0] + # Handler Windows : Windows have "SyncManager-N:N", "MainProcess", "Process-N", "gui" 4 Processes - # There have no process named "gui", only "MainProcess" in Linux + # There have no process named "gui", only "MainProcess" on Linux pname = multiprocessing.current_process().name.replace(":", "_") if os.name == "nt" else name - + + # Each process should only call once when alas start. + if any(isinstance(obj, RichTimedRotatingHandler) for obj in logger.handlers): + return log_dir = Path("./log") log_file = log_dir.joinpath(f"{pname}.txt" if name == "gui" else f"{name}.txt") log_dir.mkdir(parents=True, exist_ok=True) - # These process needn't to save log file in Windows + # These process needn't to save log file on Windows process = ["SyncManager-", "MainProcess", "Process-"] if os.name == "nt" else [] if any(p in log_file.name for p in process): - hdlr = RichFileHandler(console=Console(file=io.StringIO())) - logger.addHandler(hdlr) - logger.log_file = str(log_file.resolve()) - if log_file.exists(): - log_file.unlink() return hdlr = RichTimedRotatingHandler( @@ -420,8 +420,6 @@ def set_file_logger(name=pyw_name): encoding="utf-8", ) - logger.handlers = [h for h in logger.handlers if not isinstance( - h, (logging.FileHandler, RichTimedRotatingHandler, RichFileHandler))] logger.addHandler(hdlr) logger.log_file = hdlr.log_file if log_file.exists(): From 660b65988d9002d4b0bf1ddf233b22a4311d99dd Mon Sep 17 00:00:00 2001 From: izum1 Date: Sun, 5 May 2024 22:24:14 +0800 Subject: [PATCH 14/16] fix: fix logger to handle linux os --- module/logger.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/module/logger.py b/module/logger.py index 98b1e71203..5c646b8956 100644 --- a/module/logger.py +++ b/module/logger.py @@ -128,6 +128,7 @@ def __init__(self, pname:str, *args, **kwargs) -> None: # To handle the API of alas.save_error_log() self.log_file = None # For expire method + self.pname = pname self.bak = bak_method.lower() self.compression = zip_method.lower() @@ -398,18 +399,29 @@ def set_file_logger(name=pyw_name): # Handler Windows : Windows have "SyncManager-N:N", "MainProcess", "Process-N", "gui" 4 Processes # There have no process named "gui", only "MainProcess" on Linux - pname = multiprocessing.current_process().name.replace(":", "_") if os.name == "nt" else name - # Each process should only call once when alas start. - if any(isinstance(obj, RichTimedRotatingHandler) for obj in logger.handlers): - return + if os.name == "nt": + pname = multiprocessing.current_process().name.replace(":", "_") + # These process needn't to save log file on Windows + processes = ["SyncManager-", "MainProcess", "Process-"] + if any(isinstance(hdlr, RichTimedRotatingHandler) for hdlr in logger.handlers): + return + else: + pname = name + processes = [] + for hdlr in logger.handlers: + if isinstance(hdlr, RichTimedRotatingHandler): + if hdlr.pname == pname: + return + else: + logger.handlers = [h for h in logger.handlers if not isinstance( + h, (logging.FileHandler, RichTimedRotatingHandler, RichFileHandler))] + log_dir = Path("./log") log_file = log_dir.joinpath(f"{pname}.txt" if name == "gui" else f"{name}.txt") log_dir.mkdir(parents=True, exist_ok=True) - # These process needn't to save log file on Windows - process = ["SyncManager-", "MainProcess", "Process-"] if os.name == "nt" else [] - if any(p in log_file.name for p in process): + if any(p in log_file.name for p in processes): return hdlr = RichTimedRotatingHandler( From 4a949b743b74ec442a5a5555b8c6013a425394fa Mon Sep 17 00:00:00 2001 From: izum1 Date: Sun, 5 May 2024 23:22:25 +0800 Subject: [PATCH 15/16] fix: fix logger to handle windows os --- module/logger.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/module/logger.py b/module/logger.py index 5c646b8956..8a8eacf5ef 100644 --- a/module/logger.py +++ b/module/logger.py @@ -213,9 +213,7 @@ def doRollover(self) -> None: newPath = path.with_name( time.strftime(self.suffix, timeTuple) + "_" + path.name ) - self.richd.console.file = open( - newPath, "a" if os.path.exists(newPath) else "w", encoding="utf-8" - ) + self.richd.console.file = open(newPath, "a", encoding="utf-8") if self.backupCount > 0: files = self.getFilesToDelete() @@ -396,31 +394,30 @@ def _set_file_logger(name=pyw_name): def set_file_logger(name=pyw_name): if "_" in name: name = name.split("_", 1)[0] - # Handler Windows : Windows have "SyncManager-N:N", "MainProcess", "Process-N", "gui" 4 Processes # There have no process named "gui", only "MainProcess" on Linux - # Each process should only call once when alas start. if os.name == "nt": - pname = multiprocessing.current_process().name.replace(":", "_") # These process needn't to save log file on Windows processes = ["SyncManager-", "MainProcess", "Process-"] + pname = multiprocessing.current_process().name.replace(":", "_") + # Each process should only call once when alas start. if any(isinstance(hdlr, RichTimedRotatingHandler) for hdlr in logger.handlers): return else: - pname = name processes = [] + pname = name for hdlr in logger.handlers: if isinstance(hdlr, RichTimedRotatingHandler): - if hdlr.pname == pname: + # Each process should only call once when alas start. + if hdlr.pname == name: return else: logger.handlers = [h for h in logger.handlers if not isinstance( h, (logging.FileHandler, RichTimedRotatingHandler, RichFileHandler))] - + log_dir = Path("./log") - log_file = log_dir.joinpath(f"{pname}.txt" if name == "gui" else f"{name}.txt") log_dir.mkdir(parents=True, exist_ok=True) - + log_file = log_dir.joinpath(f"{pname}.txt" if name == "gui" else f"{name}.txt") if any(p in log_file.name for p in processes): return @@ -434,8 +431,11 @@ def set_file_logger(name=pyw_name): logger.addHandler(hdlr) logger.log_file = hdlr.log_file - if log_file.exists(): - log_file.unlink() + try: + if log_file.exists(): + log_file.unlink() + except Exception: + pass From 9364019679c6aa9eb529ef4d06fabe2c81bb232f Mon Sep 17 00:00:00 2001 From: izum1 Date: Sun, 5 May 2024 23:44:16 +0800 Subject: [PATCH 16/16] docs: Modify comments in logger.py --- module/logger.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/module/logger.py b/module/logger.py index 8a8eacf5ef..c1b00323e4 100644 --- a/module/logger.py +++ b/module/logger.py @@ -132,7 +132,7 @@ def __init__(self, pname:str, *args, **kwargs) -> None: self.bak = bak_method.lower() self.compression = zip_method.lower() - # Override the initial rolloverAt + # Override the initial rolloverAt and rich.console.file self.rolloverAt = time.time() self.doRollover() @@ -165,7 +165,7 @@ def _read_file_logger_config(self, process_name): def getFilesToDelete(self) -> List[Path]: """ Determine the files to delete when rolling over.\n - Override the original method to use RichHandler + Override the original method to use RichHandler and keep the same format """ dirName, baseName = os.path.split(self.baseFilename) fileNames = os.listdir(dirName) @@ -241,7 +241,12 @@ def doRollover(self) -> None: def expire(self, files: List[Path]) -> None: """ - Remove or backup the expired log files + Remove or backup the expired log files\n + + Template: + 2021-08-01_alas.txt...2021-08-07_alas.txt -> bak/2021-08-01~2021-08-07_alas.tar.bz2 \n + 2021-08-01_gui.txt -> bak/2021-08-01_gui.zip \n + 2021-08-01_gui.txt(copy) -> bak/2021-08-01_gui.txt(copy) \n """ basePath = Path(self.baseFilename) bakPath = basePath.parent / "bak" @@ -395,7 +400,7 @@ def set_file_logger(name=pyw_name): if "_" in name: name = name.split("_", 1)[0] # Handler Windows : Windows have "SyncManager-N:N", "MainProcess", "Process-N", "gui" 4 Processes - # There have no process named "gui", only "MainProcess" on Linux + # There have no process named "SyncManager", only "MainProcess" on Linux if os.name == "nt": # These process needn't to save log file on Windows processes = ["SyncManager-", "MainProcess", "Process-"]