From a490767f913b8f1ebc068bfe5439fe28f4dda25f Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 24 Jun 2025 16:11:22 -0600 Subject: [PATCH] Use other logging redirect class --- frigate/data_processing/common/face/model.py | 4 +- frigate/detectors/plugins/cpu_tfl.py | 2 + frigate/embeddings/onnx/face_embedding.py | 4 +- frigate/events/audio.py | 4 +- frigate/log.py | 64 +++++++++++++++++--- frigate/util/classification.py | 16 +---- 6 files changed, 67 insertions(+), 27 deletions(-) diff --git a/frigate/data_processing/common/face/model.py b/frigate/data_processing/common/face/model.py index 29bb67592..7736298bb 100644 --- a/frigate/data_processing/common/face/model.py +++ b/frigate/data_processing/common/face/model.py @@ -11,7 +11,7 @@ from scipy import stats from frigate.config import FrigateConfig from frigate.const import MODEL_CACHE_DIR from frigate.embeddings.onnx.face_embedding import ArcfaceEmbedding, FaceNetEmbedding -from frigate.log import redirect_stdout_to_logpipe +from frigate.log import redirect_stdout_to_logger logger = logging.getLogger(__name__) @@ -38,7 +38,7 @@ class FaceRecognizer(ABC): def classify(self, face_image: np.ndarray) -> tuple[str, float] | None: pass - @redirect_stdout_to_logpipe(__name__, logging.DEBUG) + @redirect_stdout_to_logger(__name__, logging.DEBUG) def init_landmark_detector(self) -> None: landmark_model = os.path.join(MODEL_CACHE_DIR, "facedet/landmarkdet.yaml") diff --git a/frigate/detectors/plugins/cpu_tfl.py b/frigate/detectors/plugins/cpu_tfl.py index fc8db0f4b..128ce45a5 100644 --- a/frigate/detectors/plugins/cpu_tfl.py +++ b/frigate/detectors/plugins/cpu_tfl.py @@ -5,6 +5,7 @@ from typing_extensions import Literal from frigate.detectors.detection_api import DetectionApi from frigate.detectors.detector_config import BaseDetectorConfig +from frigate.log import redirect_stdout_to_logger from ..detector_utils import tflite_detect_raw, tflite_init @@ -27,6 +28,7 @@ class CpuDetectorConfig(BaseDetectorConfig): class CpuTfl(DetectionApi): type_key = DETECTOR_KEY + @redirect_stdout_to_logger(__name__, logging.DEBUG) def __init__(self, detector_config: CpuDetectorConfig): interpreter = Interpreter( model_path=detector_config.model.path, diff --git a/frigate/embeddings/onnx/face_embedding.py b/frigate/embeddings/onnx/face_embedding.py index 37b3d9797..d9b62451f 100644 --- a/frigate/embeddings/onnx/face_embedding.py +++ b/frigate/embeddings/onnx/face_embedding.py @@ -6,7 +6,7 @@ import os import numpy as np from frigate.const import MODEL_CACHE_DIR -from frigate.log import redirect_stdout_to_logpipe +from frigate.log import redirect_stdout_to_logger from frigate.util.downloader import ModelDownloader from .base_embedding import BaseEmbedding @@ -54,7 +54,7 @@ class FaceNetEmbedding(BaseEmbedding): self._load_model_and_utils() logger.debug(f"models are already downloaded for {self.model_name}") - @redirect_stdout_to_logpipe(__name__, logging.DEBUG) + @redirect_stdout_to_logger(__name__, logging.DEBUG) def _load_model_and_utils(self): if self.runner is None: if self.downloader: diff --git a/frigate/events/audio.py b/frigate/events/audio.py index fe67102bd..acdbbbd09 100644 --- a/frigate/events/audio.py +++ b/frigate/events/audio.py @@ -37,7 +37,7 @@ from frigate.data_processing.real_time.audio_transcription import ( AudioTranscriptionRealTimeProcessor, ) from frigate.ffmpeg_presets import parse_preset_input -from frigate.log import LogPipe, redirect_stdout_to_logpipe +from frigate.log import LogPipe, redirect_stdout_to_logger from frigate.object_detection.base import load_labels from frigate.util.builtin import get_ffmpeg_arg_list from frigate.util.process import FrigateProcess @@ -423,7 +423,7 @@ class AudioEventMaintainer(threading.Thread): class AudioTfl: - @redirect_stdout_to_logpipe(__name__, logging.DEBUG) + @redirect_stdout_to_logger(__name__, logging.DEBUG) def __init__(self, stop_event: threading.Event, num_threads=2): self.stop_event = stop_event self.num_threads = num_threads diff --git a/frigate/log.py b/frigate/log.py index cc081b5ff..d479c7f93 100644 --- a/frigate/log.py +++ b/frigate/log.py @@ -1,5 +1,6 @@ # In log.py import atexit +import io import logging import os import sys @@ -10,7 +11,7 @@ from functools import wraps from logging.handlers import QueueHandler, QueueListener from multiprocessing.managers import SyncManager from queue import Queue -from typing import Deque, Optional +from typing import Any, Callable, Deque, Optional from frigate.util.builtin import clean_camera_user_pass @@ -138,12 +139,61 @@ class LogPipe(threading.Thread): os.close(self.fdWrite) -def redirect_stdout_to_logpipe(log_name: str, level: int): - def decorator(func): +class LogRedirect(io.StringIO): + """ + A custom file-like object to capture stdout and process it. + It extends io.StringIO to capture output and then processes it + line by line. + """ + + def __init__(self, logger_instance: logging.Logger, level: int): + super().__init__() + self.logger = logger_instance + self.log_level = level + self.buffer = [] + + def write(self, s): + if not isinstance(s, str): + s = str(s) + + self.buffer.append(s) + + # Process output line by line if a newline is present + if "\n" in s: + full_output = "".join(self.buffer) + lines = full_output.splitlines(keepends=True) + self.buffer = [] + + for line in lines: + if line.endswith("\n"): + self._process_line(line.rstrip("\n")) + else: + self.buffer.append(line) + + def _process_line(self, line): + self.logger.log(self.log_level, line) + + def flush(self): + if self.buffer: + full_output = "".join(self.buffer) + self.buffer = [] + if full_output: # Only process if there's content + self._process_line(full_output) + + def __enter__(self): + """Context manager entry point.""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Context manager exit point. Ensures buffered content is flushed.""" + self.flush() + + +def redirect_stdout_to_logger(log_name: str, level: int) -> Any: + def decorator(func: Callable): @wraps(func) def wrapper(*args, **kwargs): - current_log_pipe = LogPipe(log_name, logging.ERROR) - current_log_pipe.run() + current_log_pipe = LogRedirect(log_name, logging.ERROR) old_stdout = sys.stdout old_stderr = sys.stderr @@ -151,13 +201,11 @@ def redirect_stdout_to_logpipe(log_name: str, level: int): sys.stderr = current_log_pipe try: - print() result = func(*args, **kwargs) finally: sys.stdout = old_stdout sys.stderr = old_stderr - current_log_pipe.dump() - current_log_pipe.close() + current_log_pipe.flush() return result diff --git a/frigate/util/classification.py b/frigate/util/classification.py index a2ba1bf26..b7339aec3 100644 --- a/frigate/util/classification.py +++ b/frigate/util/classification.py @@ -1,7 +1,7 @@ """Util for classification models.""" +import logging import os -import sys import cv2 import numpy as np @@ -9,6 +9,7 @@ import numpy as np from frigate.comms.embeddings_updater import EmbeddingsRequestEnum, EmbeddingsRequestor from frigate.comms.inter_process import InterProcessRequestor from frigate.const import CLIPS_DIR, MODEL_CACHE_DIR, UPDATE_MODEL_STATE +from frigate.log import redirect_stdout_to_logger from frigate.types import ModelStatusTypesEnum from frigate.util.process import FrigateProcess @@ -36,6 +37,7 @@ def __generate_representative_dataset_factory(dataset_dir: str): return generate_representative_dataset +@redirect_stdout_to_logger(__name__, logging.DEBUG) def __train_classification_model(model_name: str) -> bool: """Train a classification model.""" @@ -55,14 +57,6 @@ def __train_classification_model(model_name: str) -> bool: ] ) - # TF and Keras are very loud with logging - # we want to avoid these logs so we - # temporarily redirect stdout / stderr - original_stdout = sys.stdout - original_stderr = sys.stderr - sys.stdout = open(os.devnull, "w") - sys.stderr = open(os.devnull, "w") - # Start with imagenet base model with 35% of channels in each layer base_model = MobileNetV2( input_shape=(224, 224, 3), @@ -124,10 +118,6 @@ def __train_classification_model(model_name: str) -> bool: with open(os.path.join(model_dir, "model.tflite"), "wb") as f: f.write(tflite_model) - # restore original stdout / stderr - sys.stdout = original_stdout - sys.stderr = original_stderr - @staticmethod def kickoff_model_training(