From f5937d83706330af7e8059bbdf08d014f3962bdc Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Wed, 25 Mar 2026 08:10:16 -0500 Subject: [PATCH 01/72] Update PR template and add check workflow (#22628) --- .github/pull_request_template.md | 2 + .github/workflows/pr_template_check.yml | 120 ++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 .github/workflows/pr_template_check.yml diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 05a75ca5f..81d448f25 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,6 +1,7 @@ _Please read the [contributing guidelines](https://github.com/blakeblackshear/frigate/blob/dev/CONTRIBUTING.md) before submitting a PR._ ## Proposed change + \s*)?([\s\S]*?)(?=\n## )/ + ); + const proposedContent = proposedChangeMatch + ? proposedChangeMatch[1].trim() + : ''; + if (!proposedContent) { + errors.push( + 'The **Proposed change** section is empty. Please describe what this PR does.' + ); + } + + // Check that at least one "Type of change" checkbox is checked + const typeSection = body.match( + /## Type of change\s*([\s\S]*?)(?=\n## )/ + ); + if (typeSection && !/- \[x\]/i.test(typeSection[1])) { + errors.push( + 'No **Type of change** selected. Please check at least one option.' + ); + } + + // Check that at least one AI disclosure checkbox is checked + const aiSection = body.match( + /## AI disclosure\s*([\s\S]*?)(?=\n## )/ + ); + if (aiSection && !/- \[x\]/i.test(aiSection[1])) { + errors.push( + 'No **AI disclosure** option selected. Please indicate whether AI tools were used.' + ); + } + + // Check that at least one checklist item is checked + const checklistSection = body.match( + /## Checklist\s*([\s\S]*?)$/ + ); + if (checklistSection && !/- \[x\]/i.test(checklistSection[1])) { + errors.push( + 'No **Checklist** items checked. Please review and check the items that apply.' + ); + } + + if (errors.length === 0) { + console.log('PR description passes template validation.'); + return; + } + + const prNumber = context.payload.pull_request.number; + const message = [ + '## PR template validation failed', + '', + 'This PR was automatically closed because the description does not follow the [pull request template](https://github.com/blakeblackshear/frigate/blob/dev/.github/pull_request_template.md).', + '', + '**Issues found:**', + ...errors.map((e) => `- ${e}`), + '', + 'Please update your PR description to include all required sections from the template, then reopen this PR.', + '', + '> If you used an AI tool to generate this PR, please see our [contributing guidelines](https://github.com/blakeblackshear/frigate/blob/dev/CONTRIBUTING.md) for details.', + ].join('\n'); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: message, + }); + + await github.rest.pulls.update({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + state: 'closed', + }); + + core.setFailed('PR description does not follow the template.'); From 3f6d5bcf2218b12b93d414a12f93ce6316ad482b Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Wed, 25 Mar 2026 08:57:47 -0500 Subject: [PATCH 02/72] ONVIF refactor (#22629) * add profile support and decouple relative move from autotracking * add drag to zoom * docs * add profile selection to UI * dynamically update onvif config * ui tweak * docs * docs tweak --- docs/docs/configuration/autotracking.md | 4 + docs/docs/configuration/cameras.md | 2 + docs/docs/configuration/reference.md | 6 +- frigate/config/camera/onvif.py | 5 + frigate/config/camera/updater.py | 3 + frigate/ptz/onvif.py | 381 ++++++++++++------ web/public/locales/en/config/cameras.json | 4 + web/public/locales/en/config/global.json | 4 + web/public/locales/en/views/live.json | 1 + web/public/locales/en/views/settings.json | 4 + .../config-form/section-configs/onvif.ts | 14 +- .../config-form/theme/frigateTheme.ts | 2 + .../theme/widgets/OnvifProfileWidget.tsx | 84 ++++ .../components/overlay/PtzControlPanel.tsx | 4 +- web/src/types/ptz.ts | 6 + web/src/views/live/LiveCameraView.tsx | 163 ++++++-- 16 files changed, 519 insertions(+), 168 deletions(-) create mode 100644 web/src/components/config-form/theme/widgets/OnvifProfileWidget.tsx diff --git a/docs/docs/configuration/autotracking.md b/docs/docs/configuration/autotracking.md index 86179a264..6dddef458 100644 --- a/docs/docs/configuration/autotracking.md +++ b/docs/docs/configuration/autotracking.md @@ -52,6 +52,10 @@ cameras: password: admin # Optional: Skip TLS verification from the ONVIF server (default: shown below) tls_insecure: False + # Optional: ONVIF media profile to use for PTZ control, matched by token or name. (default: shown below) + # If not set, the first profile with valid PTZ configuration is selected automatically. + # Use this when your camera has multiple ONVIF profiles and you need to select a specific one. + profile: None # Optional: PTZ camera object autotracking. Keeps a moving object in # the center of the frame by automatically moving the PTZ camera. autotracking: diff --git a/docs/docs/configuration/cameras.md b/docs/docs/configuration/cameras.md index eed430b52..84cf318e4 100644 --- a/docs/docs/configuration/cameras.md +++ b/docs/docs/configuration/cameras.md @@ -91,6 +91,8 @@ If your ONVIF camera does not require authentication credentials, you may still ::: +If your camera has multiple ONVIF profiles, you can specify which one to use for PTZ control with the `profile` option, matched by token or name. When not set, Frigate selects the first profile with a valid PTZ configuration. Check the Frigate debug logs (`frigate.ptz.onvif: debug`) to see available profile names and tokens for your camera. + An ONVIF-capable camera that supports relative movement within the field of view (FOV) can also be configured to automatically track moving objects and keep them in the center of the frame. For autotracking setup, see the [autotracking](autotracking.md) docs. ## ONVIF PTZ camera recommendations diff --git a/docs/docs/configuration/reference.md b/docs/docs/configuration/reference.md index c6ac207aa..e5eb16138 100644 --- a/docs/docs/configuration/reference.md +++ b/docs/docs/configuration/reference.md @@ -951,7 +951,7 @@ cameras: onvif: # Required: host of the camera being connected to. # NOTE: HTTP is assumed by default; HTTPS is supported if you specify the scheme, ex: "https://0.0.0.0". - # NOTE: ONVIF user, and password can be specified with environment variables or docker secrets + # NOTE: ONVIF host, user, and password can be specified with environment variables or docker secrets # that must begin with 'FRIGATE_'. e.g. host: '{FRIGATE_ONVIF_USERNAME}' host: 0.0.0.0 # Optional: ONVIF port for device (default: shown below). @@ -966,6 +966,10 @@ cameras: # Optional: Ignores time synchronization mismatches between the camera and the server during authentication. # Using NTP on both ends is recommended and this should only be set to True in a "safe" environment due to the security risk it represents. ignore_time_mismatch: False + # Optional: ONVIF media profile to use for PTZ control, matched by token or name. (default: shown below) + # If not set, the first profile with valid PTZ configuration is selected automatically. + # Use this when your camera has multiple ONVIF profiles and you need to select a specific one. + profile: None # Optional: PTZ camera object autotracking. Keeps a moving object in # the center of the frame by automatically moving the PTZ camera. autotracking: diff --git a/frigate/config/camera/onvif.py b/frigate/config/camera/onvif.py index eb21e24bd..836dec6aa 100644 --- a/frigate/config/camera/onvif.py +++ b/frigate/config/camera/onvif.py @@ -117,6 +117,11 @@ class OnvifConfig(FrigateBaseModel): title="Disable TLS verify", description="Skip TLS verification and disable digest auth for ONVIF (unsafe; use in safe networks only).", ) + profile: Optional[str] = Field( + default=None, + title="ONVIF profile", + description="Specific ONVIF media profile to use for PTZ control, matched by token or name. If not set, the first profile with valid PTZ configuration is selected automatically.", + ) autotracking: PtzAutotrackConfig = Field( default_factory=PtzAutotrackConfig, title="Autotracking", diff --git a/frigate/config/camera/updater.py b/frigate/config/camera/updater.py index a55f355fb..6474edf43 100644 --- a/frigate/config/camera/updater.py +++ b/frigate/config/camera/updater.py @@ -23,6 +23,7 @@ class CameraConfigUpdateEnum(str, Enum): notifications = "notifications" objects = "objects" object_genai = "object_genai" + onvif = "onvif" record = "record" remove = "remove" # for removing a camera review = "review" @@ -130,6 +131,8 @@ class CameraConfigUpdateSubscriber: config.lpr = updated_config elif update_type == CameraConfigUpdateEnum.snapshots: config.snapshots = updated_config + elif update_type == CameraConfigUpdateEnum.onvif: + config.onvif = updated_config elif update_type == CameraConfigUpdateEnum.zones: config.zones = updated_config diff --git a/frigate/ptz/onvif.py b/frigate/ptz/onvif.py index 488dbd278..79b771cb2 100644 --- a/frigate/ptz/onvif.py +++ b/frigate/ptz/onvif.py @@ -15,6 +15,10 @@ from zeep.exceptions import Fault, TransportError from frigate.camera import PTZMetrics from frigate.config import FrigateConfig, ZoomingModeEnum +from frigate.config.camera.updater import ( + CameraConfigUpdateEnum, + CameraConfigUpdateSubscriber, +) from frigate.util.builtin import find_by_key logger = logging.getLogger(__name__) @@ -65,7 +69,14 @@ class OnvifController: self.camera_configs[cam_name] = cam self.status_locks[cam_name] = asyncio.Lock() + self.config_subscriber = CameraConfigUpdateSubscriber( + self.config, + self.config.cameras, + [CameraConfigUpdateEnum.onvif], + ) + asyncio.run_coroutine_threadsafe(self._init_cameras(), self.loop) + asyncio.run_coroutine_threadsafe(self._poll_config_updates(), self.loop) def _run_event_loop(self) -> None: """Run the event loop in a separate thread.""" @@ -80,6 +91,52 @@ class OnvifController: for cam_name in self.camera_configs: await self._init_single_camera(cam_name) + async def _poll_config_updates(self) -> None: + """Poll for ONVIF config updates and re-initialize cameras as needed.""" + while True: + await asyncio.sleep(1) + try: + updates = self.config_subscriber.check_for_updates() + for update_type, cameras in updates.items(): + if update_type == CameraConfigUpdateEnum.onvif.name: + for cam_name in cameras: + await self._reinit_camera(cam_name) + except Exception: + logger.error("Error checking for ONVIF config updates") + + async def _close_camera(self, cam_name: str) -> None: + """Close the ONVIF client session for a camera.""" + cam_state = self.cams.get(cam_name) + if cam_state and "onvif" in cam_state: + try: + await cam_state["onvif"].close() + except Exception: + logger.debug(f"Error closing ONVIF session for {cam_name}") + + async def _reinit_camera(self, cam_name: str) -> None: + """Re-initialize a camera after config change.""" + logger.info(f"Re-initializing ONVIF for {cam_name} due to config change") + + # close existing session before re-init + await self._close_camera(cam_name) + + cam = self.config.cameras.get(cam_name) + if not cam or not cam.onvif.host: + # ONVIF removed from config, clean up + self.cams.pop(cam_name, None) + self.camera_configs.pop(cam_name, None) + self.failed_cams.pop(cam_name, None) + return + + # update stored config and reset state + self.camera_configs[cam_name] = cam + if cam_name not in self.status_locks: + self.status_locks[cam_name] = asyncio.Lock() + self.cams.pop(cam_name, None) + self.failed_cams.pop(cam_name, None) + + await self._init_single_camera(cam_name) + async def _init_single_camera(self, cam_name: str) -> bool: """Initialize a single camera by name. @@ -118,6 +175,7 @@ class OnvifController: "active": False, "features": [], "presets": {}, + "profiles": [], } return True except (Fault, ONVIFError, TransportError, Exception) as e: @@ -161,22 +219,60 @@ class OnvifController: ) return False + # build list of valid PTZ profiles + valid_profiles = [ + p + for p in profiles + if p.VideoEncoderConfiguration + and p.PTZConfiguration + and ( + p.PTZConfiguration.DefaultContinuousPanTiltVelocitySpace is not None + or p.PTZConfiguration.DefaultContinuousZoomVelocitySpace is not None + ) + ] + + # store available profiles for API response and log for debugging + self.cams[camera_name]["profiles"] = [ + {"name": getattr(p, "Name", None) or p.token, "token": p.token} + for p in valid_profiles + ] + for p in valid_profiles: + logger.debug( + "Onvif profile for %s: name='%s', token='%s'", + camera_name, + getattr(p, "Name", None), + p.token, + ) + + configured_profile = self.config.cameras[camera_name].onvif.profile profile = None - for _, onvif_profile in enumerate(profiles): - if ( - onvif_profile.VideoEncoderConfiguration - and onvif_profile.PTZConfiguration - and ( - onvif_profile.PTZConfiguration.DefaultContinuousPanTiltVelocitySpace - is not None - or onvif_profile.PTZConfiguration.DefaultContinuousZoomVelocitySpace - is not None + + if configured_profile is not None: + # match by exact token first, then by name + for p in valid_profiles: + if p.token == configured_profile: + profile = p + break + if profile is None: + for p in valid_profiles: + if getattr(p, "Name", None) == configured_profile: + profile = p + break + if profile is None: + available = [ + f"name='{getattr(p, 'Name', None)}', token='{p.token}'" + for p in valid_profiles + ] + logger.error( + "Onvif profile '%s' not found for camera %s. Available profiles: %s", + configured_profile, + camera_name, + available, ) - ): - # use the first profile that has a valid ptz configuration - profile = onvif_profile - logger.debug(f"Selected Onvif profile for {camera_name}: {profile}") - break + return False + else: + # use the first profile that has a valid ptz configuration + profile = valid_profiles[0] if valid_profiles else None if profile is None: logger.error( @@ -184,6 +280,8 @@ class OnvifController: ) return False + logger.debug(f"Selected Onvif profile for {camera_name}: {profile}") + # get the PTZ config for the profile try: configs = profile.PTZConfiguration @@ -218,48 +316,92 @@ class OnvifController: move_request.ProfileToken = profile.token self.cams[camera_name]["move_request"] = move_request - # extra setup for autotracking cameras - if ( - self.config.cameras[camera_name].onvif.autotracking.enabled_in_config - and self.config.cameras[camera_name].onvif.autotracking.enabled - ): + # get PTZ configuration options for feature detection and relative movement + ptz_config = None + fov_space_id = None + + try: request = ptz.create_type("GetConfigurationOptions") request.ConfigurationToken = profile.PTZConfiguration.token ptz_config = await ptz.GetConfigurationOptions(request) - logger.debug(f"Onvif config for {camera_name}: {ptz_config}") + logger.debug( + f"Onvif PTZ configuration options for {camera_name}: {ptz_config}" + ) + except (Fault, ONVIFError, TransportError, Exception) as e: + logger.debug( + f"Unable to get PTZ configuration options for {camera_name}: {e}" + ) + + # detect FOV translation space for relative movement + if ptz_config is not None: + try: + fov_space_id = next( + ( + i + for i, space in enumerate( + ptz_config.Spaces.RelativePanTiltTranslationSpace + ) + if "TranslationSpaceFov" in space["URI"] + ), + None, + ) + except (AttributeError, TypeError): + fov_space_id = None + + autotracking_config = self.config.cameras[camera_name].onvif.autotracking + autotracking_enabled = ( + autotracking_config.enabled_in_config and autotracking_config.enabled + ) + + # autotracking-only: status request and service capabilities + if autotracking_enabled: + status_request = ptz.create_type("GetStatus") + status_request.ProfileToken = profile.token + self.cams[camera_name]["status_request"] = status_request service_capabilities_request = ptz.create_type("GetServiceCapabilities") self.cams[camera_name]["service_capabilities_request"] = ( service_capabilities_request ) - fov_space_id = next( - ( - i - for i, space in enumerate( - ptz_config.Spaces.RelativePanTiltTranslationSpace - ) - if "TranslationSpaceFov" in space["URI"] - ), - None, - ) - - # status request for autotracking and filling ptz-parameters - status_request = ptz.create_type("GetStatus") - status_request.ProfileToken = profile.token - self.cams[camera_name]["status_request"] = status_request + # setup relative move request when FOV relative movement is supported + if ( + fov_space_id is not None + and configs.DefaultRelativePanTiltTranslationSpace is not None + ): + # one-off GetStatus to seed Translation field + status = None try: - status = await ptz.GetStatus(status_request) - logger.debug(f"Onvif status config for {camera_name}: {status}") + one_off_status_request = ptz.create_type("GetStatus") + one_off_status_request.ProfileToken = profile.token + status = await ptz.GetStatus(one_off_status_request) + logger.debug(f"Onvif status for {camera_name}: {status}") except Exception as e: - logger.warning(f"Unable to get status from camera: {camera_name}: {e}") - status = None + logger.warning(f"Unable to get status from camera {camera_name}: {e}") - # autotracking relative panning/tilting needs a relative zoom value set to 0 - # if camera supports relative movement + rel_move_request = ptz.create_type("RelativeMove") + rel_move_request.ProfileToken = profile.token + logger.debug(f"{camera_name}: Relative move request: {rel_move_request}") + + fov_uri = ptz_config["Spaces"]["RelativePanTiltTranslationSpace"][ + fov_space_id + ]["URI"] + + if rel_move_request.Translation is None: + if status is not None: + # seed from current position + rel_move_request.Translation = status.Position + rel_move_request.Translation.PanTilt.space = fov_uri + else: + # fallback: construct Translation explicitly + rel_move_request.Translation = { + "PanTilt": {"x": 0, "y": 0, "space": fov_uri} + } + + # configure zoom on relative move request if ( - self.config.cameras[camera_name].onvif.autotracking.zooming - != ZoomingModeEnum.disabled + autotracking_enabled + and autotracking_config.zooming != ZoomingModeEnum.disabled ): zoom_space_id = next( ( @@ -271,60 +413,43 @@ class OnvifController: ), None, ) - - # setup relative moving request for autotracking - move_request = ptz.create_type("RelativeMove") - move_request.ProfileToken = profile.token - logger.debug(f"{camera_name}: Relative move request: {move_request}") - if move_request.Translation is None and fov_space_id is not None: - move_request.Translation = status.Position - move_request.Translation.PanTilt.space = ptz_config["Spaces"][ - "RelativePanTiltTranslationSpace" - ][fov_space_id]["URI"] - - # try setting relative zoom translation space - try: - if ( - self.config.cameras[camera_name].onvif.autotracking.zooming - != ZoomingModeEnum.disabled - ): + try: if zoom_space_id is not None: - move_request.Translation.Zoom.space = ptz_config["Spaces"][ + rel_move_request.Translation.Zoom.space = ptz_config["Spaces"][ "RelativeZoomTranslationSpace" ][zoom_space_id]["URI"] - else: - if ( - move_request["Translation"] is not None - and "Zoom" in move_request["Translation"] - ): - del move_request["Translation"]["Zoom"] - if ( - move_request["Speed"] is not None - and "Zoom" in move_request["Speed"] - ): - del move_request["Speed"]["Zoom"] - logger.debug( - f"{camera_name}: Relative move request after deleting zoom: {move_request}" + except Exception as e: + autotracking_config.zooming = ZoomingModeEnum.disabled + logger.warning( + f"Disabling autotracking zooming for {camera_name}: Relative zoom not supported. Exception: {e}" ) - except Exception as e: - self.config.cameras[ - camera_name - ].onvif.autotracking.zooming = ZoomingModeEnum.disabled - logger.warning( - f"Disabling autotracking zooming for {camera_name}: Relative zoom not supported. Exception: {e}" + else: + # remove zoom fields from relative move request + if ( + rel_move_request["Translation"] is not None + and "Zoom" in rel_move_request["Translation"] + ): + del rel_move_request["Translation"]["Zoom"] + if ( + rel_move_request["Speed"] is not None + and "Zoom" in rel_move_request["Speed"] + ): + del rel_move_request["Speed"]["Zoom"] + logger.debug( + f"{camera_name}: Relative move request after deleting zoom: {rel_move_request}" ) - if move_request.Speed is None: - move_request.Speed = configs.DefaultPTZSpeed if configs else None + if rel_move_request.Speed is None: + rel_move_request.Speed = configs.DefaultPTZSpeed if configs else None logger.debug( - f"{camera_name}: Relative move request after setup: {move_request}" + f"{camera_name}: Relative move request after setup: {rel_move_request}" ) - self.cams[camera_name]["relative_move_request"] = move_request + self.cams[camera_name]["relative_move_request"] = rel_move_request - # setup absolute moving request for autotracking zooming - move_request = ptz.create_type("AbsoluteMove") - move_request.ProfileToken = profile.token - self.cams[camera_name]["absolute_move_request"] = move_request + # setup absolute move request + abs_move_request = ptz.create_type("AbsoluteMove") + abs_move_request.ProfileToken = profile.token + self.cams[camera_name]["absolute_move_request"] = abs_move_request # setup existing presets try: @@ -358,48 +483,48 @@ class OnvifController: if configs.DefaultRelativeZoomTranslationSpace: supported_features.append("zoom-r") - if ( - self.config.cameras[camera_name].onvif.autotracking.enabled_in_config - and self.config.cameras[camera_name].onvif.autotracking.enabled - ): + if ptz_config is not None: try: - # get camera's zoom limits from onvif config self.cams[camera_name]["relative_zoom_range"] = ( ptz_config.Spaces.RelativeZoomTranslationSpace[0] ) except Exception as e: - if ( - self.config.cameras[camera_name].onvif.autotracking.zooming - == ZoomingModeEnum.relative - ): - self.config.cameras[ - camera_name - ].onvif.autotracking.zooming = ZoomingModeEnum.disabled + if autotracking_config.zooming == ZoomingModeEnum.relative: + autotracking_config.zooming = ZoomingModeEnum.disabled logger.warning( f"Disabling autotracking zooming for {camera_name}: Relative zoom not supported. Exception: {e}" ) if configs.DefaultAbsoluteZoomPositionSpace: supported_features.append("zoom-a") - if ( - self.config.cameras[camera_name].onvif.autotracking.enabled_in_config - and self.config.cameras[camera_name].onvif.autotracking.enabled - ): + if ptz_config is not None: try: - # get camera's zoom limits from onvif config self.cams[camera_name]["absolute_zoom_range"] = ( ptz_config.Spaces.AbsoluteZoomPositionSpace[0] ) self.cams[camera_name]["zoom_limits"] = configs.ZoomLimits except Exception as e: - if self.config.cameras[camera_name].onvif.autotracking.zooming: - self.config.cameras[ - camera_name - ].onvif.autotracking.zooming = ZoomingModeEnum.disabled + if autotracking_config.zooming != ZoomingModeEnum.disabled: + autotracking_config.zooming = ZoomingModeEnum.disabled logger.warning( f"Disabling autotracking zooming for {camera_name}: Absolute zoom not supported. Exception: {e}" ) + # disable autotracking zoom if required ranges are unavailable + if autotracking_config.zooming != ZoomingModeEnum.disabled: + if autotracking_config.zooming == ZoomingModeEnum.relative: + if "relative_zoom_range" not in self.cams[camera_name]: + autotracking_config.zooming = ZoomingModeEnum.disabled + logger.warning( + f"Disabling autotracking zooming for {camera_name}: Relative zoom range unavailable" + ) + if autotracking_config.zooming == ZoomingModeEnum.absolute: + if "absolute_zoom_range" not in self.cams[camera_name]: + autotracking_config.zooming = ZoomingModeEnum.disabled + logger.warning( + f"Disabling autotracking zooming for {camera_name}: Absolute zoom range unavailable" + ) + if ( self.cams[camera_name]["video_source_token"] is not None and imaging is not None @@ -416,10 +541,9 @@ class OnvifController: except (Fault, ONVIFError, TransportError, Exception) as e: logger.debug(f"Focus not supported for {camera_name}: {e}") + # detect FOV relative movement support if ( - self.config.cameras[camera_name].onvif.autotracking.enabled_in_config - and self.config.cameras[camera_name].onvif.autotracking.enabled - and fov_space_id is not None + fov_space_id is not None and configs.DefaultRelativePanTiltTranslationSpace is not None ): supported_features.append("pt-r-fov") @@ -548,11 +672,8 @@ class OnvifController: move_request.Translation.PanTilt.x = pan move_request.Translation.PanTilt.y = tilt - if ( - "zoom-r" in self.cams[camera_name]["features"] - and self.config.cameras[camera_name].onvif.autotracking.zooming - == ZoomingModeEnum.relative - ): + # include zoom if requested and camera supports relative zoom + if zoom != 0 and "zoom-r" in self.cams[camera_name]["features"]: move_request.Speed = { "PanTilt": { "x": speed, @@ -560,7 +681,7 @@ class OnvifController: }, "Zoom": {"x": speed}, } - move_request.Translation.Zoom.x = zoom + move_request["Translation"]["Zoom"] = {"x": zoom} await self.cams[camera_name]["ptz"].RelativeMove(move_request) @@ -568,12 +689,8 @@ class OnvifController: move_request.Translation.PanTilt.x = 0 move_request.Translation.PanTilt.y = 0 - if ( - "zoom-r" in self.cams[camera_name]["features"] - and self.config.cameras[camera_name].onvif.autotracking.zooming - == ZoomingModeEnum.relative - ): - move_request.Translation.Zoom.x = 0 + if zoom != 0 and "zoom-r" in self.cams[camera_name]["features"]: + del move_request["Translation"]["Zoom"] self.cams[camera_name]["active"] = False @@ -717,8 +834,18 @@ class OnvifController: elif command == OnvifCommandEnum.preset: await self._move_to_preset(camera_name, param) elif command == OnvifCommandEnum.move_relative: - _, pan, tilt = param.split("_") - await self._move_relative(camera_name, float(pan), float(tilt), 0, 1) + parts = param.split("_") + if len(parts) == 3: + _, pan, tilt = parts + zoom = 0.0 + elif len(parts) == 4: + _, pan, tilt, zoom = parts + else: + logger.error(f"Invalid move_relative params: {param}") + return + await self._move_relative( + camera_name, float(pan), float(tilt), float(zoom), 1 + ) elif command in (OnvifCommandEnum.zoom_in, OnvifCommandEnum.zoom_out): await self._zoom(camera_name, command) elif command in (OnvifCommandEnum.focus_in, OnvifCommandEnum.focus_out): @@ -773,6 +900,7 @@ class OnvifController: "name": camera_name, "features": self.cams[camera_name]["features"], "presets": list(self.cams[camera_name]["presets"].keys()), + "profiles": self.cams[camera_name].get("profiles", []), } if camera_name not in self.cams.keys() and camera_name in self.config.cameras: @@ -970,6 +1098,7 @@ class OnvifController: return logger.info("Exiting ONVIF controller...") + self.config_subscriber.stop() def stop_and_cleanup(): try: diff --git a/web/public/locales/en/config/cameras.json b/web/public/locales/en/config/cameras.json index ebe775504..470af687e 100644 --- a/web/public/locales/en/config/cameras.json +++ b/web/public/locales/en/config/cameras.json @@ -787,6 +787,10 @@ "label": "Disable TLS verify", "description": "Skip TLS verification and disable digest auth for ONVIF (unsafe; use in safe networks only)." }, + "profile": { + "label": "ONVIF profile", + "description": "Specific ONVIF media profile to use for PTZ control, matched by token or name. If not set, the first profile with valid PTZ configuration is selected automatically." + }, "autotracking": { "label": "Autotracking", "description": "Automatically track moving objects and keep them centered in the frame using PTZ camera movements.", diff --git a/web/public/locales/en/config/global.json b/web/public/locales/en/config/global.json index 8587ec263..e653818fa 100644 --- a/web/public/locales/en/config/global.json +++ b/web/public/locales/en/config/global.json @@ -1536,6 +1536,10 @@ "label": "Disable TLS verify", "description": "Skip TLS verification and disable digest auth for ONVIF (unsafe; use in safe networks only)." }, + "profile": { + "label": "ONVIF profile", + "description": "Specific ONVIF media profile to use for PTZ control, matched by token or name. If not set, the first profile with valid PTZ configuration is selected automatically." + }, "autotracking": { "label": "Autotracking", "description": "Automatically track moving objects and keep them centered in the frame using PTZ camera movements.", diff --git a/web/public/locales/en/views/live.json b/web/public/locales/en/views/live.json index 878470187..37e6b15db 100644 --- a/web/public/locales/en/views/live.json +++ b/web/public/locales/en/views/live.json @@ -17,6 +17,7 @@ "clickMove": { "label": "Click in the frame to center the camera", "enable": "Enable click to move", + "enableWithZoom": "Enable click to move / drag to zoom", "disable": "Disable click to move" }, "left": { diff --git a/web/public/locales/en/views/settings.json b/web/public/locales/en/views/settings.json index 42de28d52..4109b4821 100644 --- a/web/public/locales/en/views/settings.json +++ b/web/public/locales/en/views/settings.json @@ -1573,5 +1573,9 @@ "hardwareNone": "No hardware acceleration", "hardwareAuto": "Automatic hardware acceleration" } + }, + "onvif": { + "profileAuto": "Auto", + "profileLoading": "Loading profiles..." } } diff --git a/web/src/components/config-form/section-configs/onvif.ts b/web/src/components/config-form/section-configs/onvif.ts index b8be693d6..c08cd7a58 100644 --- a/web/src/components/config-form/section-configs/onvif.ts +++ b/web/src/components/config-form/section-configs/onvif.ts @@ -3,20 +3,12 @@ import type { SectionConfigOverrides } from "./types"; const onvif: SectionConfigOverrides = { base: { sectionDocs: "/configuration/cameras#setting-up-camera-ptz-controls", - restartRequired: [ - "host", - "port", - "user", - "password", - "tls_insecure", - "ignore_time_mismatch", - "autotracking.calibrate_on_startup", - ], fieldOrder: [ "host", "port", "user", "password", + "profile", "tls_insecure", "ignore_time_mismatch", "autotracking", @@ -27,10 +19,14 @@ const onvif: SectionConfigOverrides = { ], advancedFields: ["tls_insecure", "ignore_time_mismatch"], overrideFields: [], + restartRequired: ["autotracking.calibrate_on_startup"], uiSchema: { host: { "ui:options": { size: "sm" }, }, + profile: { + "ui:widget": "onvifProfile", + }, autotracking: { required_zones: { "ui:widget": "zoneNames", diff --git a/web/src/components/config-form/theme/frigateTheme.ts b/web/src/components/config-form/theme/frigateTheme.ts index 5df8564f2..5497e35b7 100644 --- a/web/src/components/config-form/theme/frigateTheme.ts +++ b/web/src/components/config-form/theme/frigateTheme.ts @@ -29,6 +29,7 @@ import { TimezoneSelectWidget } from "./widgets/TimezoneSelectWidget"; import { CameraPathWidget } from "./widgets/CameraPathWidget"; import { OptionalFieldWidget } from "./widgets/OptionalFieldWidget"; import { SemanticSearchModelWidget } from "./widgets/SemanticSearchModelWidget"; +import { OnvifProfileWidget } from "./widgets/OnvifProfileWidget"; import { FieldTemplate } from "./templates/FieldTemplate"; import { ObjectFieldTemplate } from "./templates/ObjectFieldTemplate"; @@ -79,6 +80,7 @@ export const frigateTheme: FrigateTheme = { timezoneSelect: TimezoneSelectWidget, optionalField: OptionalFieldWidget, semanticSearchModel: SemanticSearchModelWidget, + onvifProfile: OnvifProfileWidget, }, templates: { FieldTemplate: FieldTemplate as React.ComponentType, diff --git a/web/src/components/config-form/theme/widgets/OnvifProfileWidget.tsx b/web/src/components/config-form/theme/widgets/OnvifProfileWidget.tsx new file mode 100644 index 000000000..6743b5589 --- /dev/null +++ b/web/src/components/config-form/theme/widgets/OnvifProfileWidget.tsx @@ -0,0 +1,84 @@ +import type { WidgetProps } from "@rjsf/utils"; +import useSWR from "swr"; +import { useTranslation } from "react-i18next"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import type { ConfigFormContext } from "@/types/configForm"; +import type { CameraPtzInfo } from "@/types/ptz"; +import { getSizedFieldClassName } from "../utils"; +import ActivityIndicator from "@/components/indicators/activity-indicator"; +import { cn } from "@/lib/utils"; + +const AUTO_VALUE = "__auto__"; + +export function OnvifProfileWidget(props: WidgetProps) { + const { id, value, disabled, readonly, onChange, schema, options } = props; + const { t } = useTranslation(["views/settings"]); + + const formContext = props.registry?.formContext as + | ConfigFormContext + | undefined; + const cameraName = formContext?.cameraName; + const isCameraLevel = formContext?.level === "camera"; + const hasOnvifHost = !!formContext?.fullCameraConfig?.onvif?.host; + + const { data: ptzInfo } = useSWR( + isCameraLevel && cameraName && hasOnvifHost + ? `${cameraName}/ptz/info` + : null, + { + // ONVIF may not be initialized yet when the settings page loads, + // so retry until profiles become available + refreshInterval: (data) => + data?.profiles && data.profiles.length > 0 ? 0 : 5000, + }, + ); + + const profiles = ptzInfo?.profiles ?? []; + const fieldClassName = getSizedFieldClassName(options, "md"); + const hasProfiles = profiles.length > 0; + const waiting = isCameraLevel && !!cameraName && hasOnvifHost && !hasProfiles; + + const selected = value ?? AUTO_VALUE; + + if (waiting) { + return ( +
+ + + {t("onvif.profileLoading")} + +
+ ); + } + + return ( + + ); +} diff --git a/web/src/components/overlay/PtzControlPanel.tsx b/web/src/components/overlay/PtzControlPanel.tsx index 5deb62fd3..32e2c26f1 100644 --- a/web/src/components/overlay/PtzControlPanel.tsx +++ b/web/src/components/overlay/PtzControlPanel.tsx @@ -284,7 +284,9 @@ export default function PtzControlPanel({

{clickOverlay ? t("ptz.move.clickMove.disable") - : t("ptz.move.clickMove.enable")} + : ptz?.features?.includes("zoom-r") + ? t("ptz.move.clickMove.enableWithZoom") + : t("ptz.move.clickMove.enable")}

diff --git a/web/src/types/ptz.ts b/web/src/types/ptz.ts index 21a300b3d..02e55ae81 100644 --- a/web/src/types/ptz.ts +++ b/web/src/types/ptz.ts @@ -7,8 +7,14 @@ type PtzFeature = | "pt-r-fov" | "focus"; +export type OnvifProfile = { + name: string; + token: string; +}; + export type CameraPtzInfo = { name: string; features: PtzFeature[]; presets: string[]; + profiles: OnvifProfile[]; }; diff --git a/web/src/views/live/LiveCameraView.tsx b/web/src/views/live/LiveCameraView.tsx index 418c74068..f8a36eb7b 100644 --- a/web/src/views/live/LiveCameraView.tsx +++ b/web/src/views/live/LiveCameraView.tsx @@ -122,6 +122,11 @@ import { SnapshotResult, } from "@/utils/snapshotUtil"; import ActivityIndicator from "@/components/indicators/activity-indicator"; +import { Stage, Layer, Rect } from "react-konva"; +import type { KonvaEventObject } from "konva/lib/Node"; + +/** Pixel threshold to distinguish drag from click. */ +const DRAG_MIN_PX = 15; type LiveCameraViewProps = { config?: FrigateConfig; @@ -213,45 +218,112 @@ export default function LiveCameraView({ }; }, [audioTranscriptionState, sendTranscription]); - // click overlay for ptzs + // click-to-move / drag-to-zoom overlay for PTZ cameras const [clickOverlay, setClickOverlay] = useState(false); const clickOverlayRef = useRef(null); const { send: sendPtz } = usePtzCommand(camera.name); - const handleOverlayClick = useCallback( - ( - e: React.MouseEvent | React.TouchEvent, - ) => { - if (!clickOverlay) { - return; - } + // drag rectangle state in stage-local coordinates + const [ptzRect, setPtzRect] = useState<{ + x: number; + y: number; + width: number; + height: number; + } | null>(null); + const [isPtzDrawing, setIsPtzDrawing] = useState(false); + // raw origin to determine drag direction (not min/max corrected) + const ptzOriginRef = useRef<{ x: number; y: number } | null>(null); - let clientX; - let clientY; - if ("TouchEvent" in window && e.nativeEvent instanceof TouchEvent) { - clientX = e.nativeEvent.touches[0].clientX; - clientY = e.nativeEvent.touches[0].clientY; - } else if (e.nativeEvent instanceof MouseEvent) { - clientX = e.nativeEvent.clientX; - clientY = e.nativeEvent.clientY; - } + const [overlaySize] = useResizeObserver(clickOverlayRef); - if (clickOverlayRef.current && clientX && clientY) { - const rect = clickOverlayRef.current.getBoundingClientRect(); - - const normalizedX = (clientX - rect.left) / rect.width; - const normalizedY = (clientY - rect.top) / rect.height; - - const pan = (normalizedX - 0.5) * 2; - const tilt = (0.5 - normalizedY) * 2; - - sendPtz(`move_relative_${pan}_${tilt}`); + const onPtzStageDown = useCallback( + (e: KonvaEventObject | KonvaEventObject) => { + const pos = e.target.getStage()?.getPointerPosition(); + if (pos) { + setIsPtzDrawing(true); + ptzOriginRef.current = { x: pos.x, y: pos.y }; + setPtzRect({ x: pos.x, y: pos.y, width: 0, height: 0 }); } }, - [clickOverlayRef, clickOverlay, sendPtz], + [], ); + const onPtzStageMove = useCallback( + (e: KonvaEventObject | KonvaEventObject) => { + if (!isPtzDrawing || !ptzRect) return; + const pos = e.target.getStage()?.getPointerPosition(); + if (pos) { + setPtzRect({ + ...ptzRect, + width: pos.x - ptzRect.x, + height: pos.y - ptzRect.y, + }); + } + }, + [isPtzDrawing, ptzRect], + ); + + const onPtzStageUp = useCallback(() => { + setIsPtzDrawing(false); + + if (!ptzRect || !ptzOriginRef.current || overlaySize.width === 0) { + setPtzRect(null); + ptzOriginRef.current = null; + return; + } + + const endX = ptzRect.x + ptzRect.width; + const endY = ptzRect.y + ptzRect.height; + const distX = Math.abs(ptzRect.width); + const distY = Math.abs(ptzRect.height); + + if (distX < DRAG_MIN_PX && distY < DRAG_MIN_PX) { + // click — pan/tilt to point without zoom + const normX = endX / overlaySize.width; + const normY = endY / overlaySize.height; + const pan = (normX - 0.5) * 2; + const tilt = (0.5 - normY) * 2; + sendPtz(`move_relative_${pan}_${tilt}`); + } else { + // drag — pan/tilt to box center, zoom based on box size + const origin = ptzOriginRef.current; + + const n0x = Math.min(origin.x, endX) / overlaySize.width; + const n0y = Math.min(origin.y, endY) / overlaySize.height; + const n1x = Math.max(origin.x, endX) / overlaySize.width; + const n1y = Math.max(origin.y, endY) / overlaySize.height; + + let boxW = n1x - n0x; + let boxH = n1y - n0y; + + // correct box to match camera aspect ratio so zoom is uniform + const frameAR = overlaySize.width / overlaySize.height; + const boxAR = boxW / boxH; + if (boxAR > frameAR) { + boxH = boxW / frameAR; + } else { + boxW = boxH * frameAR; + } + + const centerX = (n0x + n1x) / 2; + const centerY = (n0y + n1y) / 2; + const pan = (centerX - 0.5) * 2; + const tilt = (0.5 - centerY) * 2; + + // zoom magnitude from box size (small box = more zoom) + let zoom = Math.max(0.01, Math.min(1, Math.max(boxW, boxH))); + // drag direction: top-left → bottom-right = zoom in, reverse = zoom out + const zoomIn = endX > origin.x && endY > origin.y; + if (!zoomIn) zoom = -zoom; + + sendPtz(`move_relative_${pan}_${tilt}_${zoom}`); + } + + setPtzRect(null); + ptzOriginRef.current = null; + }, [ptzRect, overlaySize, sendPtz]); + // pip state useEffect(() => { @@ -440,7 +512,8 @@ export default function LiveCameraView({
+ {clickOverlay && overlaySize.width > 0 && ( +
+ + + {ptzRect && ( + + )} + + +
+ )} Date: Wed, 25 Mar 2026 08:44:12 -0600 Subject: [PATCH 03/72] Split apart video.py (#22631) --- frigate/events/audio.py | 2 +- frigate/util/ffmpeg.py | 48 +++ frigate/video/__init__.py | 2 + frigate/video/detect.py | 563 ++++++++++++++++++++++++ frigate/{video.py => video/ffmpeg.py} | 587 +------------------------- 5 files changed, 620 insertions(+), 582 deletions(-) create mode 100644 frigate/util/ffmpeg.py create mode 100644 frigate/video/__init__.py create mode 100644 frigate/video/detect.py rename frigate/{video.py => video/ffmpeg.py} (55%) mode change 100755 => 100644 diff --git a/frigate/events/audio.py b/frigate/events/audio.py index 6afa1c237..505874469 100644 --- a/frigate/events/audio.py +++ b/frigate/events/audio.py @@ -37,8 +37,8 @@ from frigate.ffmpeg_presets import parse_preset_input from frigate.log import LogPipe, suppress_stderr_during from frigate.object_detection.base import load_labels from frigate.util.builtin import get_ffmpeg_arg_list +from frigate.util.ffmpeg import start_or_restart_ffmpeg, stop_ffmpeg from frigate.util.process import FrigateProcess -from frigate.video import start_or_restart_ffmpeg, stop_ffmpeg try: from tflite_runtime.interpreter import Interpreter diff --git a/frigate/util/ffmpeg.py b/frigate/util/ffmpeg.py new file mode 100644 index 000000000..9abacd4ed --- /dev/null +++ b/frigate/util/ffmpeg.py @@ -0,0 +1,48 @@ +"""FFmpeg utility functions for managing ffmpeg processes.""" + +import logging +import subprocess as sp +from typing import Any + +from frigate.log import LogPipe + + +def stop_ffmpeg(ffmpeg_process: sp.Popen[Any], logger: logging.Logger): + logger.info("Terminating the existing ffmpeg process...") + ffmpeg_process.terminate() + try: + logger.info("Waiting for ffmpeg to exit gracefully...") + ffmpeg_process.communicate(timeout=30) + logger.info("FFmpeg has exited") + except sp.TimeoutExpired: + logger.info("FFmpeg didn't exit. Force killing...") + ffmpeg_process.kill() + ffmpeg_process.communicate() + logger.info("FFmpeg has been killed") + ffmpeg_process = None + + +def start_or_restart_ffmpeg( + ffmpeg_cmd, logger, logpipe: LogPipe, frame_size=None, ffmpeg_process=None +) -> sp.Popen[Any]: + if ffmpeg_process is not None: + stop_ffmpeg(ffmpeg_process, logger) + + if frame_size is None: + process = sp.Popen( + ffmpeg_cmd, + stdout=sp.DEVNULL, + stderr=logpipe, + stdin=sp.DEVNULL, + start_new_session=True, + ) + else: + process = sp.Popen( + ffmpeg_cmd, + stdout=sp.PIPE, + stderr=logpipe, + stdin=sp.DEVNULL, + bufsize=frame_size * 10, + start_new_session=True, + ) + return process diff --git a/frigate/video/__init__.py b/frigate/video/__init__.py new file mode 100644 index 000000000..24589835c --- /dev/null +++ b/frigate/video/__init__.py @@ -0,0 +1,2 @@ +from .detect import * # noqa: F403 +from .ffmpeg import * # noqa: F403 diff --git a/frigate/video/detect.py b/frigate/video/detect.py new file mode 100644 index 000000000..339b11e53 --- /dev/null +++ b/frigate/video/detect.py @@ -0,0 +1,563 @@ +"""Manages camera object detection processes.""" + +import logging +import queue +import time +from datetime import datetime, timezone +from multiprocessing import Queue +from multiprocessing.synchronize import Event as MpEvent +from typing import Any + +import cv2 + +from frigate.camera import CameraMetrics, PTZMetrics +from frigate.comms.inter_process import InterProcessRequestor +from frigate.config import CameraConfig, DetectConfig, LoggerConfig, ModelConfig +from frigate.config.camera.camera import CameraTypeEnum +from frigate.config.camera.updater import ( + CameraConfigUpdateEnum, + CameraConfigUpdateSubscriber, +) +from frigate.const import ( + PROCESS_PRIORITY_HIGH, + REQUEST_REGION_GRID, +) +from frigate.motion import MotionDetector +from frigate.motion.improved_motion import ImprovedMotionDetector +from frigate.object_detection.base import RemoteObjectDetector +from frigate.ptz.autotrack import ptz_moving_at_frame_time +from frigate.track import ObjectTracker +from frigate.track.norfair_tracker import NorfairTracker +from frigate.track.tracked_object import TrackedObjectAttribute +from frigate.util.builtin import EventsPerSecond +from frigate.util.image import ( + FrameManager, + SharedMemoryFrameManager, + draw_box_with_label, +) +from frigate.util.object import ( + create_tensor_input, + get_cluster_candidates, + get_cluster_region, + get_cluster_region_from_grid, + get_min_region_size, + get_startup_regions, + inside_any, + intersects_any, + is_object_filtered, + reduce_detections, +) +from frigate.util.process import FrigateProcess +from frigate.util.time import get_tomorrow_at_time + +logger = logging.getLogger(__name__) + + +class CameraTracker(FrigateProcess): + def __init__( + self, + config: CameraConfig, + model_config: ModelConfig, + labelmap: dict[int, str], + detection_queue: Queue, + detected_objects_queue, + camera_metrics: CameraMetrics, + ptz_metrics: PTZMetrics, + region_grid: list[list[dict[str, Any]]], + stop_event: MpEvent, + log_config: LoggerConfig | None = None, + ) -> None: + super().__init__( + stop_event, + PROCESS_PRIORITY_HIGH, + name=f"frigate.process:{config.name}", + daemon=True, + ) + self.config = config + self.model_config = model_config + self.labelmap = labelmap + self.detection_queue = detection_queue + self.detected_objects_queue = detected_objects_queue + self.camera_metrics = camera_metrics + self.ptz_metrics = ptz_metrics + self.region_grid = region_grid + self.log_config = log_config + + def run(self) -> None: + self.pre_run_setup(self.log_config) + frame_queue = self.camera_metrics.frame_queue + frame_shape = self.config.frame_shape + + motion_detector = ImprovedMotionDetector( + frame_shape, + self.config.motion, + self.config.detect.fps, + name=self.config.name, + ptz_metrics=self.ptz_metrics, + ) + object_detector = RemoteObjectDetector( + self.config.name, + self.labelmap, + self.detection_queue, + self.model_config, + self.stop_event, + ) + + object_tracker = NorfairTracker(self.config, self.ptz_metrics) + + frame_manager = SharedMemoryFrameManager() + + # create communication for region grid updates + requestor = InterProcessRequestor() + + process_frames( + requestor, + frame_queue, + frame_shape, + self.model_config, + self.config, + frame_manager, + motion_detector, + object_detector, + object_tracker, + self.detected_objects_queue, + self.camera_metrics, + self.stop_event, + self.ptz_metrics, + self.region_grid, + ) + + # empty the frame queue + logger.info(f"{self.config.name}: emptying frame queue") + while not frame_queue.empty(): + (frame_name, _) = frame_queue.get(False) + frame_manager.delete(frame_name) + + logger.info(f"{self.config.name}: exiting subprocess") + + +def detect( + detect_config: DetectConfig, + object_detector, + frame, + model_config: ModelConfig, + region, + objects_to_track, + object_filters, +): + tensor_input = create_tensor_input(frame, model_config, region) + + detections = [] + region_detections = object_detector.detect(tensor_input) + for d in region_detections: + box = d[2] + size = region[2] - region[0] + x_min = int(max(0, (box[1] * size) + region[0])) + y_min = int(max(0, (box[0] * size) + region[1])) + x_max = int(min(detect_config.width - 1, (box[3] * size) + region[0])) + y_max = int(min(detect_config.height - 1, (box[2] * size) + region[1])) + + # ignore objects that were detected outside the frame + if (x_min >= detect_config.width - 1) or (y_min >= detect_config.height - 1): + continue + + width = x_max - x_min + height = y_max - y_min + area = width * height + ratio = width / max(1, height) + det = (d[0], d[1], (x_min, y_min, x_max, y_max), area, ratio, region) + # apply object filters + if is_object_filtered(det, objects_to_track, object_filters): + continue + detections.append(det) + return detections + + +def process_frames( + requestor: InterProcessRequestor, + frame_queue: Queue, + frame_shape: tuple[int, int], + model_config: ModelConfig, + camera_config: CameraConfig, + frame_manager: FrameManager, + motion_detector: MotionDetector, + object_detector: RemoteObjectDetector, + object_tracker: ObjectTracker, + detected_objects_queue: Queue, + camera_metrics: CameraMetrics, + stop_event: MpEvent, + ptz_metrics: PTZMetrics, + region_grid: list[list[dict[str, Any]]], + exit_on_empty: bool = False, +): + next_region_update = get_tomorrow_at_time(2) + config_subscriber = CameraConfigUpdateSubscriber( + None, + {camera_config.name: camera_config}, + [ + CameraConfigUpdateEnum.detect, + CameraConfigUpdateEnum.enabled, + CameraConfigUpdateEnum.motion, + CameraConfigUpdateEnum.objects, + ], + ) + + fps_tracker = EventsPerSecond() + fps_tracker.start() + + startup_scan = True + stationary_frame_counter = 0 + camera_enabled = True + + region_min_size = get_min_region_size(model_config) + + attributes_map = model_config.attributes_map + all_attributes = model_config.all_attributes + + # remove license_plate from attributes if this camera is a dedicated LPR cam + if camera_config.type == CameraTypeEnum.lpr: + modified_attributes_map = model_config.attributes_map.copy() + + if ( + "car" in modified_attributes_map + and "license_plate" in modified_attributes_map["car"] + ): + modified_attributes_map["car"] = [ + attr + for attr in modified_attributes_map["car"] + if attr != "license_plate" + ] + + attributes_map = modified_attributes_map + + all_attributes = [ + attr for attr in model_config.all_attributes if attr != "license_plate" + ] + + while not stop_event.is_set(): + updated_configs = config_subscriber.check_for_updates() + + if "enabled" in updated_configs: + prev_enabled = camera_enabled + camera_enabled = camera_config.enabled + + if "motion" in updated_configs: + motion_detector.config = camera_config.motion + motion_detector.update_mask() + + if ( + not camera_enabled + and prev_enabled != camera_enabled + and camera_metrics.frame_queue.empty() + ): + logger.debug( + f"Camera {camera_config.name} disabled, clearing tracked objects" + ) + prev_enabled = camera_enabled + + # Clear norfair's dictionaries + object_tracker.tracked_objects.clear() + object_tracker.disappeared.clear() + object_tracker.stationary_box_history.clear() + object_tracker.positions.clear() + object_tracker.track_id_map.clear() + + # Clear internal norfair states + for trackers_by_type in object_tracker.trackers.values(): + for tracker in trackers_by_type.values(): + tracker.tracked_objects = [] + for tracker in object_tracker.default_tracker.values(): + tracker.tracked_objects = [] + + if not camera_enabled: + time.sleep(0.1) + continue + + if datetime.now().astimezone(timezone.utc) > next_region_update: + region_grid = requestor.send_data(REQUEST_REGION_GRID, camera_config.name) + next_region_update = get_tomorrow_at_time(2) + + try: + if exit_on_empty: + frame_name, frame_time = frame_queue.get(False) + else: + frame_name, frame_time = frame_queue.get(True, 1) + except queue.Empty: + if exit_on_empty: + logger.info("Exiting track_objects...") + break + continue + + camera_metrics.detection_frame.value = frame_time + ptz_metrics.frame_time.value = frame_time + + frame = frame_manager.get(frame_name, (frame_shape[0] * 3 // 2, frame_shape[1])) + + if frame is None: + logger.debug( + f"{camera_config.name}: frame {frame_time} is not in memory store." + ) + continue + + # look for motion if enabled + motion_boxes = motion_detector.detect(frame) + + regions = [] + consolidated_detections = [] + + # if detection is disabled + if not camera_config.detect.enabled: + object_tracker.match_and_update(frame_name, frame_time, []) + else: + # get stationary object ids + # check every Nth frame for stationary objects + # disappeared objects are not stationary + # also check for overlapping motion boxes + if stationary_frame_counter == camera_config.detect.stationary.interval: + stationary_frame_counter = 0 + stationary_object_ids = [] + else: + stationary_frame_counter += 1 + stationary_object_ids = [ + obj["id"] + for obj in object_tracker.tracked_objects.values() + # if it has exceeded the stationary threshold + if obj["motionless_count"] + >= camera_config.detect.stationary.threshold + # and it hasn't disappeared + and object_tracker.disappeared[obj["id"]] == 0 + # and it doesn't overlap with any current motion boxes when not calibrating + and not intersects_any( + obj["box"], + [] if motion_detector.is_calibrating() else motion_boxes, + ) + ] + + # get tracked object boxes that aren't stationary + tracked_object_boxes = [ + ( + # use existing object box for stationary objects + obj["estimate"] + if obj["motionless_count"] + < camera_config.detect.stationary.threshold + else obj["box"] + ) + for obj in object_tracker.tracked_objects.values() + if obj["id"] not in stationary_object_ids + ] + object_boxes = tracked_object_boxes + object_tracker.untracked_object_boxes + + # get consolidated regions for tracked objects + regions = [ + get_cluster_region( + frame_shape, region_min_size, candidate, object_boxes + ) + for candidate in get_cluster_candidates( + frame_shape, region_min_size, object_boxes + ) + ] + + # only add in the motion boxes when not calibrating and a ptz is not moving via autotracking + # ptz_moving_at_frame_time() always returns False for non-autotracking cameras + if not motion_detector.is_calibrating() and not ptz_moving_at_frame_time( + frame_time, + ptz_metrics.start_time.value, + ptz_metrics.stop_time.value, + ): + # find motion boxes that are not inside tracked object regions + standalone_motion_boxes = [ + b for b in motion_boxes if not inside_any(b, regions) + ] + + if standalone_motion_boxes: + motion_clusters = get_cluster_candidates( + frame_shape, + region_min_size, + standalone_motion_boxes, + ) + motion_regions = [ + get_cluster_region_from_grid( + frame_shape, + region_min_size, + candidate, + standalone_motion_boxes, + region_grid, + ) + for candidate in motion_clusters + ] + regions += motion_regions + + # if starting up, get the next startup scan region + if startup_scan: + for region in get_startup_regions( + frame_shape, region_min_size, region_grid + ): + regions.append(region) + startup_scan = False + + # resize regions and detect + # seed with stationary objects + detections = [ + ( + obj["label"], + obj["score"], + obj["box"], + obj["area"], + obj["ratio"], + obj["region"], + ) + for obj in object_tracker.tracked_objects.values() + if obj["id"] in stationary_object_ids + ] + + for region in regions: + detections.extend( + detect( + camera_config.detect, + object_detector, + frame, + model_config, + region, + camera_config.objects.track, + camera_config.objects.filters, + ) + ) + + consolidated_detections = reduce_detections(frame_shape, detections) + + # if detection was run on this frame, consolidate + if len(regions) > 0: + tracked_detections = [ + d for d in consolidated_detections if d[0] not in all_attributes + ] + # now that we have refined our detections, we need to track objects + object_tracker.match_and_update( + frame_name, frame_time, tracked_detections + ) + # else, just update the frame times for the stationary objects + else: + object_tracker.update_frame_times(frame_name, frame_time) + + # group the attribute detections based on what label they apply to + attribute_detections: dict[str, list[TrackedObjectAttribute]] = {} + for label, attribute_labels in attributes_map.items(): + attribute_detections[label] = [ + TrackedObjectAttribute(d) + for d in consolidated_detections + if d[0] in attribute_labels + ] + + # build detections + detections = {} + for obj in object_tracker.tracked_objects.values(): + detections[obj["id"]] = {**obj, "attributes": []} + + # find the best object for each attribute to be assigned to + all_objects: list[dict[str, Any]] = object_tracker.tracked_objects.values() + for attributes in attribute_detections.values(): + for attribute in attributes: + filtered_objects = filter( + lambda o: attribute.label in attributes_map.get(o["label"], []), + all_objects, + ) + selected_object_id = attribute.find_best_object(filtered_objects) + + if selected_object_id is not None: + detections[selected_object_id]["attributes"].append( + attribute.get_tracking_data() + ) + + # debug object tracking + if False: + bgr_frame = cv2.cvtColor( + frame, + cv2.COLOR_YUV2BGR_I420, + ) + object_tracker.debug_draw(bgr_frame, frame_time) + cv2.imwrite( + f"debug/frames/track-{'{:.6f}'.format(frame_time)}.jpg", bgr_frame + ) + # debug + if False: + bgr_frame = cv2.cvtColor( + frame, + cv2.COLOR_YUV2BGR_I420, + ) + + for m_box in motion_boxes: + cv2.rectangle( + bgr_frame, + (m_box[0], m_box[1]), + (m_box[2], m_box[3]), + (0, 0, 255), + 2, + ) + + for b in tracked_object_boxes: + cv2.rectangle( + bgr_frame, + (b[0], b[1]), + (b[2], b[3]), + (255, 0, 0), + 2, + ) + + for obj in object_tracker.tracked_objects.values(): + if obj["frame_time"] == frame_time: + thickness = 2 + color = model_config.colormap.get(obj["label"], (255, 255, 255)) + else: + thickness = 1 + color = (255, 0, 0) + + # draw the bounding boxes on the frame + box = obj["box"] + + draw_box_with_label( + bgr_frame, + box[0], + box[1], + box[2], + box[3], + obj["label"], + obj["id"], + thickness=thickness, + color=color, + ) + + for region in regions: + cv2.rectangle( + bgr_frame, + (region[0], region[1]), + (region[2], region[3]), + (0, 255, 0), + 2, + ) + + cv2.imwrite( + f"debug/frames/{camera_config.name}-{'{:.6f}'.format(frame_time)}.jpg", + bgr_frame, + ) + # add to the queue if not full + if detected_objects_queue.full(): + frame_manager.close(frame_name) + continue + else: + fps_tracker.update() + camera_metrics.process_fps.value = fps_tracker.eps() + detected_objects_queue.put( + ( + camera_config.name, + frame_name, + frame_time, + detections, + motion_boxes, + regions, + ) + ) + camera_metrics.detection_fps.value = object_detector.fps.eps() + frame_manager.close(frame_name) + + motion_detector.stop() + requestor.stop() + config_subscriber.stop() diff --git a/frigate/video.py b/frigate/video/ffmpeg.py old mode 100755 new mode 100644 similarity index 55% rename from frigate/video.py rename to frigate/video/ffmpeg.py index 289027bb4..852ea4a16 --- a/frigate/video.py +++ b/frigate/video/ffmpeg.py @@ -1,3 +1,5 @@ +"""Manages ffmpeg processes for camera frame capture.""" + import logging import queue import subprocess as sp @@ -9,97 +11,30 @@ from multiprocessing import Queue, Value from multiprocessing.synchronize import Event as MpEvent from typing import Any -import cv2 - -from frigate.camera import CameraMetrics, PTZMetrics +from frigate.camera import CameraMetrics from frigate.comms.inter_process import InterProcessRequestor from frigate.comms.recordings_updater import ( RecordingsDataSubscriber, RecordingsDataTypeEnum, ) -from frigate.config import CameraConfig, DetectConfig, LoggerConfig, ModelConfig -from frigate.config.camera.camera import CameraTypeEnum +from frigate.config import CameraConfig, LoggerConfig from frigate.config.camera.updater import ( CameraConfigUpdateEnum, CameraConfigUpdateSubscriber, ) -from frigate.const import ( - PROCESS_PRIORITY_HIGH, - REQUEST_REGION_GRID, -) +from frigate.const import PROCESS_PRIORITY_HIGH from frigate.log import LogPipe -from frigate.motion import MotionDetector -from frigate.motion.improved_motion import ImprovedMotionDetector -from frigate.object_detection.base import RemoteObjectDetector -from frigate.ptz.autotrack import ptz_moving_at_frame_time -from frigate.track import ObjectTracker -from frigate.track.norfair_tracker import NorfairTracker -from frigate.track.tracked_object import TrackedObjectAttribute from frigate.util.builtin import EventsPerSecond +from frigate.util.ffmpeg import start_or_restart_ffmpeg, stop_ffmpeg from frigate.util.image import ( FrameManager, SharedMemoryFrameManager, - draw_box_with_label, -) -from frigate.util.object import ( - create_tensor_input, - get_cluster_candidates, - get_cluster_region, - get_cluster_region_from_grid, - get_min_region_size, - get_startup_regions, - inside_any, - intersects_any, - is_object_filtered, - reduce_detections, ) from frigate.util.process import FrigateProcess -from frigate.util.time import get_tomorrow_at_time logger = logging.getLogger(__name__) -def stop_ffmpeg(ffmpeg_process: sp.Popen[Any], logger: logging.Logger): - logger.info("Terminating the existing ffmpeg process...") - ffmpeg_process.terminate() - try: - logger.info("Waiting for ffmpeg to exit gracefully...") - ffmpeg_process.communicate(timeout=30) - logger.info("FFmpeg has exited") - except sp.TimeoutExpired: - logger.info("FFmpeg didn't exit. Force killing...") - ffmpeg_process.kill() - ffmpeg_process.communicate() - logger.info("FFmpeg has been killed") - ffmpeg_process = None - - -def start_or_restart_ffmpeg( - ffmpeg_cmd, logger, logpipe: LogPipe, frame_size=None, ffmpeg_process=None -) -> sp.Popen[Any]: - if ffmpeg_process is not None: - stop_ffmpeg(ffmpeg_process, logger) - - if frame_size is None: - process = sp.Popen( - ffmpeg_cmd, - stdout=sp.DEVNULL, - stderr=logpipe, - stdin=sp.DEVNULL, - start_new_session=True, - ) - else: - process = sp.Popen( - ffmpeg_cmd, - stdout=sp.PIPE, - stderr=logpipe, - stdin=sp.DEVNULL, - bufsize=frame_size * 10, - start_new_session=True, - ) - return process - - def capture_frames( ffmpeg_process: sp.Popen[Any], config: CameraConfig, @@ -708,513 +643,3 @@ class CameraCapture(FrigateProcess): ) camera_watchdog.start() camera_watchdog.join() - - -class CameraTracker(FrigateProcess): - def __init__( - self, - config: CameraConfig, - model_config: ModelConfig, - labelmap: dict[int, str], - detection_queue: Queue, - detected_objects_queue, - camera_metrics: CameraMetrics, - ptz_metrics: PTZMetrics, - region_grid: list[list[dict[str, Any]]], - stop_event: MpEvent, - log_config: LoggerConfig | None = None, - ) -> None: - super().__init__( - stop_event, - PROCESS_PRIORITY_HIGH, - name=f"frigate.process:{config.name}", - daemon=True, - ) - self.config = config - self.model_config = model_config - self.labelmap = labelmap - self.detection_queue = detection_queue - self.detected_objects_queue = detected_objects_queue - self.camera_metrics = camera_metrics - self.ptz_metrics = ptz_metrics - self.region_grid = region_grid - self.log_config = log_config - - def run(self) -> None: - self.pre_run_setup(self.log_config) - frame_queue = self.camera_metrics.frame_queue - frame_shape = self.config.frame_shape - - motion_detector = ImprovedMotionDetector( - frame_shape, - self.config.motion, - self.config.detect.fps, - name=self.config.name, - ptz_metrics=self.ptz_metrics, - ) - object_detector = RemoteObjectDetector( - self.config.name, - self.labelmap, - self.detection_queue, - self.model_config, - self.stop_event, - ) - - object_tracker = NorfairTracker(self.config, self.ptz_metrics) - - frame_manager = SharedMemoryFrameManager() - - # create communication for region grid updates - requestor = InterProcessRequestor() - - process_frames( - requestor, - frame_queue, - frame_shape, - self.model_config, - self.config, - frame_manager, - motion_detector, - object_detector, - object_tracker, - self.detected_objects_queue, - self.camera_metrics, - self.stop_event, - self.ptz_metrics, - self.region_grid, - ) - - # empty the frame queue - logger.info(f"{self.config.name}: emptying frame queue") - while not frame_queue.empty(): - (frame_name, _) = frame_queue.get(False) - frame_manager.delete(frame_name) - - logger.info(f"{self.config.name}: exiting subprocess") - - -def detect( - detect_config: DetectConfig, - object_detector, - frame, - model_config: ModelConfig, - region, - objects_to_track, - object_filters, -): - tensor_input = create_tensor_input(frame, model_config, region) - - detections = [] - region_detections = object_detector.detect(tensor_input) - for d in region_detections: - box = d[2] - size = region[2] - region[0] - x_min = int(max(0, (box[1] * size) + region[0])) - y_min = int(max(0, (box[0] * size) + region[1])) - x_max = int(min(detect_config.width - 1, (box[3] * size) + region[0])) - y_max = int(min(detect_config.height - 1, (box[2] * size) + region[1])) - - # ignore objects that were detected outside the frame - if (x_min >= detect_config.width - 1) or (y_min >= detect_config.height - 1): - continue - - width = x_max - x_min - height = y_max - y_min - area = width * height - ratio = width / max(1, height) - det = (d[0], d[1], (x_min, y_min, x_max, y_max), area, ratio, region) - # apply object filters - if is_object_filtered(det, objects_to_track, object_filters): - continue - detections.append(det) - return detections - - -def process_frames( - requestor: InterProcessRequestor, - frame_queue: Queue, - frame_shape: tuple[int, int], - model_config: ModelConfig, - camera_config: CameraConfig, - frame_manager: FrameManager, - motion_detector: MotionDetector, - object_detector: RemoteObjectDetector, - object_tracker: ObjectTracker, - detected_objects_queue: Queue, - camera_metrics: CameraMetrics, - stop_event: MpEvent, - ptz_metrics: PTZMetrics, - region_grid: list[list[dict[str, Any]]], - exit_on_empty: bool = False, -): - next_region_update = get_tomorrow_at_time(2) - config_subscriber = CameraConfigUpdateSubscriber( - None, - {camera_config.name: camera_config}, - [ - CameraConfigUpdateEnum.detect, - CameraConfigUpdateEnum.enabled, - CameraConfigUpdateEnum.motion, - CameraConfigUpdateEnum.objects, - ], - ) - - fps_tracker = EventsPerSecond() - fps_tracker.start() - - startup_scan = True - stationary_frame_counter = 0 - camera_enabled = True - - region_min_size = get_min_region_size(model_config) - - attributes_map = model_config.attributes_map - all_attributes = model_config.all_attributes - - # remove license_plate from attributes if this camera is a dedicated LPR cam - if camera_config.type == CameraTypeEnum.lpr: - modified_attributes_map = model_config.attributes_map.copy() - - if ( - "car" in modified_attributes_map - and "license_plate" in modified_attributes_map["car"] - ): - modified_attributes_map["car"] = [ - attr - for attr in modified_attributes_map["car"] - if attr != "license_plate" - ] - - attributes_map = modified_attributes_map - - all_attributes = [ - attr for attr in model_config.all_attributes if attr != "license_plate" - ] - - while not stop_event.is_set(): - updated_configs = config_subscriber.check_for_updates() - - if "enabled" in updated_configs: - prev_enabled = camera_enabled - camera_enabled = camera_config.enabled - - if "motion" in updated_configs: - motion_detector.config = camera_config.motion - motion_detector.update_mask() - - if ( - not camera_enabled - and prev_enabled != camera_enabled - and camera_metrics.frame_queue.empty() - ): - logger.debug( - f"Camera {camera_config.name} disabled, clearing tracked objects" - ) - prev_enabled = camera_enabled - - # Clear norfair's dictionaries - object_tracker.tracked_objects.clear() - object_tracker.disappeared.clear() - object_tracker.stationary_box_history.clear() - object_tracker.positions.clear() - object_tracker.track_id_map.clear() - - # Clear internal norfair states - for trackers_by_type in object_tracker.trackers.values(): - for tracker in trackers_by_type.values(): - tracker.tracked_objects = [] - for tracker in object_tracker.default_tracker.values(): - tracker.tracked_objects = [] - - if not camera_enabled: - time.sleep(0.1) - continue - - if datetime.now().astimezone(timezone.utc) > next_region_update: - region_grid = requestor.send_data(REQUEST_REGION_GRID, camera_config.name) - next_region_update = get_tomorrow_at_time(2) - - try: - if exit_on_empty: - frame_name, frame_time = frame_queue.get(False) - else: - frame_name, frame_time = frame_queue.get(True, 1) - except queue.Empty: - if exit_on_empty: - logger.info("Exiting track_objects...") - break - continue - - camera_metrics.detection_frame.value = frame_time - ptz_metrics.frame_time.value = frame_time - - frame = frame_manager.get(frame_name, (frame_shape[0] * 3 // 2, frame_shape[1])) - - if frame is None: - logger.debug( - f"{camera_config.name}: frame {frame_time} is not in memory store." - ) - continue - - # look for motion if enabled - motion_boxes = motion_detector.detect(frame) - - regions = [] - consolidated_detections = [] - - # if detection is disabled - if not camera_config.detect.enabled: - object_tracker.match_and_update(frame_name, frame_time, []) - else: - # get stationary object ids - # check every Nth frame for stationary objects - # disappeared objects are not stationary - # also check for overlapping motion boxes - if stationary_frame_counter == camera_config.detect.stationary.interval: - stationary_frame_counter = 0 - stationary_object_ids = [] - else: - stationary_frame_counter += 1 - stationary_object_ids = [ - obj["id"] - for obj in object_tracker.tracked_objects.values() - # if it has exceeded the stationary threshold - if obj["motionless_count"] - >= camera_config.detect.stationary.threshold - # and it hasn't disappeared - and object_tracker.disappeared[obj["id"]] == 0 - # and it doesn't overlap with any current motion boxes when not calibrating - and not intersects_any( - obj["box"], - [] if motion_detector.is_calibrating() else motion_boxes, - ) - ] - - # get tracked object boxes that aren't stationary - tracked_object_boxes = [ - ( - # use existing object box for stationary objects - obj["estimate"] - if obj["motionless_count"] - < camera_config.detect.stationary.threshold - else obj["box"] - ) - for obj in object_tracker.tracked_objects.values() - if obj["id"] not in stationary_object_ids - ] - object_boxes = tracked_object_boxes + object_tracker.untracked_object_boxes - - # get consolidated regions for tracked objects - regions = [ - get_cluster_region( - frame_shape, region_min_size, candidate, object_boxes - ) - for candidate in get_cluster_candidates( - frame_shape, region_min_size, object_boxes - ) - ] - - # only add in the motion boxes when not calibrating and a ptz is not moving via autotracking - # ptz_moving_at_frame_time() always returns False for non-autotracking cameras - if not motion_detector.is_calibrating() and not ptz_moving_at_frame_time( - frame_time, - ptz_metrics.start_time.value, - ptz_metrics.stop_time.value, - ): - # find motion boxes that are not inside tracked object regions - standalone_motion_boxes = [ - b for b in motion_boxes if not inside_any(b, regions) - ] - - if standalone_motion_boxes: - motion_clusters = get_cluster_candidates( - frame_shape, - region_min_size, - standalone_motion_boxes, - ) - motion_regions = [ - get_cluster_region_from_grid( - frame_shape, - region_min_size, - candidate, - standalone_motion_boxes, - region_grid, - ) - for candidate in motion_clusters - ] - regions += motion_regions - - # if starting up, get the next startup scan region - if startup_scan: - for region in get_startup_regions( - frame_shape, region_min_size, region_grid - ): - regions.append(region) - startup_scan = False - - # resize regions and detect - # seed with stationary objects - detections = [ - ( - obj["label"], - obj["score"], - obj["box"], - obj["area"], - obj["ratio"], - obj["region"], - ) - for obj in object_tracker.tracked_objects.values() - if obj["id"] in stationary_object_ids - ] - - for region in regions: - detections.extend( - detect( - camera_config.detect, - object_detector, - frame, - model_config, - region, - camera_config.objects.track, - camera_config.objects.filters, - ) - ) - - consolidated_detections = reduce_detections(frame_shape, detections) - - # if detection was run on this frame, consolidate - if len(regions) > 0: - tracked_detections = [ - d for d in consolidated_detections if d[0] not in all_attributes - ] - # now that we have refined our detections, we need to track objects - object_tracker.match_and_update( - frame_name, frame_time, tracked_detections - ) - # else, just update the frame times for the stationary objects - else: - object_tracker.update_frame_times(frame_name, frame_time) - - # group the attribute detections based on what label they apply to - attribute_detections: dict[str, list[TrackedObjectAttribute]] = {} - for label, attribute_labels in attributes_map.items(): - attribute_detections[label] = [ - TrackedObjectAttribute(d) - for d in consolidated_detections - if d[0] in attribute_labels - ] - - # build detections - detections = {} - for obj in object_tracker.tracked_objects.values(): - detections[obj["id"]] = {**obj, "attributes": []} - - # find the best object for each attribute to be assigned to - all_objects: list[dict[str, Any]] = object_tracker.tracked_objects.values() - for attributes in attribute_detections.values(): - for attribute in attributes: - filtered_objects = filter( - lambda o: attribute.label in attributes_map.get(o["label"], []), - all_objects, - ) - selected_object_id = attribute.find_best_object(filtered_objects) - - if selected_object_id is not None: - detections[selected_object_id]["attributes"].append( - attribute.get_tracking_data() - ) - - # debug object tracking - if False: - bgr_frame = cv2.cvtColor( - frame, - cv2.COLOR_YUV2BGR_I420, - ) - object_tracker.debug_draw(bgr_frame, frame_time) - cv2.imwrite( - f"debug/frames/track-{'{:.6f}'.format(frame_time)}.jpg", bgr_frame - ) - # debug - if False: - bgr_frame = cv2.cvtColor( - frame, - cv2.COLOR_YUV2BGR_I420, - ) - - for m_box in motion_boxes: - cv2.rectangle( - bgr_frame, - (m_box[0], m_box[1]), - (m_box[2], m_box[3]), - (0, 0, 255), - 2, - ) - - for b in tracked_object_boxes: - cv2.rectangle( - bgr_frame, - (b[0], b[1]), - (b[2], b[3]), - (255, 0, 0), - 2, - ) - - for obj in object_tracker.tracked_objects.values(): - if obj["frame_time"] == frame_time: - thickness = 2 - color = model_config.colormap.get(obj["label"], (255, 255, 255)) - else: - thickness = 1 - color = (255, 0, 0) - - # draw the bounding boxes on the frame - box = obj["box"] - - draw_box_with_label( - bgr_frame, - box[0], - box[1], - box[2], - box[3], - obj["label"], - obj["id"], - thickness=thickness, - color=color, - ) - - for region in regions: - cv2.rectangle( - bgr_frame, - (region[0], region[1]), - (region[2], region[3]), - (0, 255, 0), - 2, - ) - - cv2.imwrite( - f"debug/frames/{camera_config.name}-{'{:.6f}'.format(frame_time)}.jpg", - bgr_frame, - ) - # add to the queue if not full - if detected_objects_queue.full(): - frame_manager.close(frame_name) - continue - else: - fps_tracker.update() - camera_metrics.process_fps.value = fps_tracker.eps() - detected_objects_queue.put( - ( - camera_config.name, - frame_name, - frame_time, - detections, - motion_boxes, - regions, - ) - ) - camera_metrics.detection_fps.value = object_detector.fps.eps() - frame_manager.close(frame_name) - - motion_detector.stop() - requestor.stop() - config_subscriber.stop() From 80c4ce2b5d225e0ffbbc0e60c418a581ad3577fa Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Wed, 25 Mar 2026 09:28:48 -0600 Subject: [PATCH 04/72] Increase mypy coverage and fixes (#22632) --- frigate/genai/__init__.py | 12 +++---- frigate/genai/azure-openai.py | 14 ++++---- frigate/genai/gemini.py | 67 +++++++++++++++++++++++------------ frigate/genai/llama_cpp.py | 16 ++++----- frigate/genai/ollama.py | 12 +++---- frigate/genai/openai.py | 12 +++---- frigate/jobs/media_sync.py | 8 ++--- frigate/jobs/motion_search.py | 39 ++++++++++++-------- frigate/jobs/vlm_watch.py | 17 +++++---- frigate/motion/__init__.py | 12 +++---- frigate/mypy.ini | 18 ++++++++++ 11 files changed, 140 insertions(+), 87 deletions(-) diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index f799931ec..96e956242 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -5,7 +5,7 @@ import importlib import logging import os import re -from typing import Any, Optional +from typing import Any, Callable, Optional import numpy as np from playhouse.shortcuts import model_to_dict @@ -31,10 +31,10 @@ __all__ = [ PROVIDERS = {} -def register_genai_provider(key: GenAIProviderEnum): +def register_genai_provider(key: GenAIProviderEnum) -> Callable: """Register a GenAI provider.""" - def decorator(cls): + def decorator(cls: type) -> type: PROVIDERS[key] = cls return cls @@ -297,7 +297,7 @@ Guidelines: """Generate a description for the frame.""" try: prompt = camera_config.objects.genai.object_prompts.get( - event.label, + str(event.label), camera_config.objects.genai.prompt, ).format(**model_to_dict(event)) except KeyError as e: @@ -307,7 +307,7 @@ Guidelines: logger.debug(f"Sending images to genai provider with prompt: {prompt}") return self._send(prompt, thumbnails) - def _init_provider(self): + def _init_provider(self) -> Any: """Initialize the client.""" return None @@ -402,7 +402,7 @@ Guidelines: } -def load_providers(): +def load_providers() -> None: package_dir = os.path.dirname(__file__) for filename in os.listdir(package_dir): if filename.endswith(".py") and filename != "__init__.py": diff --git a/frigate/genai/azure-openai.py b/frigate/genai/azure-openai.py index f424f7610..7cd8894a4 100644 --- a/frigate/genai/azure-openai.py +++ b/frigate/genai/azure-openai.py @@ -3,7 +3,7 @@ import base64 import json import logging -from typing import Any, Optional +from typing import Any, AsyncGenerator, Optional from urllib.parse import parse_qs, urlparse from openai import AzureOpenAI @@ -20,10 +20,10 @@ class OpenAIClient(GenAIClient): provider: AzureOpenAI - def _init_provider(self): + def _init_provider(self) -> AzureOpenAI | None: """Initialize the client.""" try: - parsed_url = urlparse(self.genai_config.base_url) + parsed_url = urlparse(self.genai_config.base_url or "") query_params = parse_qs(parsed_url.query) api_version = query_params.get("api-version", [None])[0] azure_endpoint = f"{parsed_url.scheme}://{parsed_url.netloc}/" @@ -79,7 +79,7 @@ class OpenAIClient(GenAIClient): logger.warning("Azure OpenAI returned an error: %s", str(e)) return None if len(result.choices) > 0: - return result.choices[0].message.content.strip() + return str(result.choices[0].message.content.strip()) return None def get_context_size(self) -> int: @@ -113,7 +113,7 @@ class OpenAIClient(GenAIClient): if openai_tool_choice is not None: request_params["tool_choice"] = openai_tool_choice - result = self.provider.chat.completions.create(**request_params) + result = self.provider.chat.completions.create(**request_params) # type: ignore[call-overload] if ( result is None @@ -181,7 +181,7 @@ class OpenAIClient(GenAIClient): messages: list[dict[str, Any]], tools: Optional[list[dict[str, Any]]] = None, tool_choice: Optional[str] = "auto", - ): + ) -> AsyncGenerator[tuple[str, Any], None]: """ Stream chat with tools; yields content deltas then final message. @@ -214,7 +214,7 @@ class OpenAIClient(GenAIClient): tool_calls_by_index: dict[int, dict[str, Any]] = {} finish_reason = "stop" - stream = self.provider.chat.completions.create(**request_params) + stream = self.provider.chat.completions.create(**request_params) # type: ignore[call-overload] for chunk in stream: if not chunk or not chunk.choices: diff --git a/frigate/genai/gemini.py b/frigate/genai/gemini.py index f32d37e80..e0c0c7698 100644 --- a/frigate/genai/gemini.py +++ b/frigate/genai/gemini.py @@ -2,10 +2,11 @@ import json import logging -from typing import Any, Optional +from typing import Any, AsyncGenerator, Optional from google import genai from google.genai import errors, types +from google.genai.types import FunctionCallingConfigMode from frigate.config import GenAIProviderEnum from frigate.genai import GenAIClient, register_genai_provider @@ -19,10 +20,10 @@ class GeminiClient(GenAIClient): provider: genai.Client - def _init_provider(self): + def _init_provider(self) -> genai.Client: """Initialize the client.""" # Merge provider_options into HttpOptions - http_options_dict = { + http_options_dict: dict[str, Any] = { "timeout": int(self.timeout * 1000), # requires milliseconds "retry_options": types.HttpRetryOptions( attempts=3, @@ -54,7 +55,7 @@ class GeminiClient(GenAIClient): ] + [prompt] try: # Merge runtime_options into generation_config if provided - generation_config_dict = {"candidate_count": 1} + generation_config_dict: dict[str, Any] = {"candidate_count": 1} generation_config_dict.update(self.genai_config.runtime_options) if response_format and response_format.get("type") == "json_schema": @@ -65,7 +66,7 @@ class GeminiClient(GenAIClient): response = self.provider.models.generate_content( model=self.genai_config.model, - contents=contents, + contents=contents, # type: ignore[arg-type] config=types.GenerateContentConfig( **generation_config_dict, ), @@ -78,6 +79,8 @@ class GeminiClient(GenAIClient): return None try: + if response.text is None: + return None description = response.text.strip() except (ValueError, AttributeError): # No description was generated @@ -102,7 +105,7 @@ class GeminiClient(GenAIClient): """ try: # Convert messages to Gemini format - gemini_messages = [] + gemini_messages: list[types.Content] = [] for msg in messages: role = msg.get("role", "user") content = msg.get("content", "") @@ -110,7 +113,11 @@ class GeminiClient(GenAIClient): # Map roles to Gemini format if role == "system": # Gemini doesn't have system role, prepend to first user message - if gemini_messages and gemini_messages[0].role == "user": + if ( + gemini_messages + and gemini_messages[0].role == "user" + and gemini_messages[0].parts + ): gemini_messages[0].parts[ 0 ].text = f"{content}\n\n{gemini_messages[0].parts[0].text}" @@ -136,7 +143,7 @@ class GeminiClient(GenAIClient): types.Content( role="function", parts=[ - types.Part.from_function_response(function_response) + types.Part.from_function_response(function_response) # type: ignore[misc,call-arg,arg-type] ], ) ) @@ -171,19 +178,25 @@ class GeminiClient(GenAIClient): if tool_choice: if tool_choice == "none": tool_config = types.ToolConfig( - function_calling_config=types.FunctionCallingConfig(mode="NONE") + function_calling_config=types.FunctionCallingConfig( + mode=FunctionCallingConfigMode.NONE + ) ) elif tool_choice == "auto": tool_config = types.ToolConfig( - function_calling_config=types.FunctionCallingConfig(mode="AUTO") + function_calling_config=types.FunctionCallingConfig( + mode=FunctionCallingConfigMode.AUTO + ) ) elif tool_choice == "required": tool_config = types.ToolConfig( - function_calling_config=types.FunctionCallingConfig(mode="ANY") + function_calling_config=types.FunctionCallingConfig( + mode=FunctionCallingConfigMode.ANY + ) ) # Build request config - config_params = {"candidate_count": 1} + config_params: dict[str, Any] = {"candidate_count": 1} if gemini_tools: config_params["tools"] = gemini_tools @@ -197,7 +210,7 @@ class GeminiClient(GenAIClient): response = self.provider.models.generate_content( model=self.genai_config.model, - contents=gemini_messages, + contents=gemini_messages, # type: ignore[arg-type] config=types.GenerateContentConfig(**config_params), ) @@ -291,7 +304,7 @@ class GeminiClient(GenAIClient): messages: list[dict[str, Any]], tools: Optional[list[dict[str, Any]]] = None, tool_choice: Optional[str] = "auto", - ): + ) -> AsyncGenerator[tuple[str, Any], None]: """ Stream chat with tools; yields content deltas then final message. @@ -299,7 +312,7 @@ class GeminiClient(GenAIClient): """ try: # Convert messages to Gemini format - gemini_messages = [] + gemini_messages: list[types.Content] = [] for msg in messages: role = msg.get("role", "user") content = msg.get("content", "") @@ -307,7 +320,11 @@ class GeminiClient(GenAIClient): # Map roles to Gemini format if role == "system": # Gemini doesn't have system role, prepend to first user message - if gemini_messages and gemini_messages[0].role == "user": + if ( + gemini_messages + and gemini_messages[0].role == "user" + and gemini_messages[0].parts + ): gemini_messages[0].parts[ 0 ].text = f"{content}\n\n{gemini_messages[0].parts[0].text}" @@ -333,7 +350,7 @@ class GeminiClient(GenAIClient): types.Content( role="function", parts=[ - types.Part.from_function_response(function_response) + types.Part.from_function_response(function_response) # type: ignore[misc,call-arg,arg-type] ], ) ) @@ -368,19 +385,25 @@ class GeminiClient(GenAIClient): if tool_choice: if tool_choice == "none": tool_config = types.ToolConfig( - function_calling_config=types.FunctionCallingConfig(mode="NONE") + function_calling_config=types.FunctionCallingConfig( + mode=FunctionCallingConfigMode.NONE + ) ) elif tool_choice == "auto": tool_config = types.ToolConfig( - function_calling_config=types.FunctionCallingConfig(mode="AUTO") + function_calling_config=types.FunctionCallingConfig( + mode=FunctionCallingConfigMode.AUTO + ) ) elif tool_choice == "required": tool_config = types.ToolConfig( - function_calling_config=types.FunctionCallingConfig(mode="ANY") + function_calling_config=types.FunctionCallingConfig( + mode=FunctionCallingConfigMode.ANY + ) ) # Build request config - config_params = {"candidate_count": 1} + config_params: dict[str, Any] = {"candidate_count": 1} if gemini_tools: config_params["tools"] = gemini_tools @@ -399,7 +422,7 @@ class GeminiClient(GenAIClient): stream = await self.provider.aio.models.generate_content_stream( model=self.genai_config.model, - contents=gemini_messages, + contents=gemini_messages, # type: ignore[arg-type] config=types.GenerateContentConfig(**config_params), ) diff --git a/frigate/genai/llama_cpp.py b/frigate/genai/llama_cpp.py index 48ea9747c..fbb1b45df 100644 --- a/frigate/genai/llama_cpp.py +++ b/frigate/genai/llama_cpp.py @@ -4,7 +4,7 @@ import base64 import io import json import logging -from typing import Any, Optional +from typing import Any, AsyncGenerator, Optional import httpx import numpy as np @@ -23,7 +23,7 @@ def _to_jpeg(img_bytes: bytes) -> bytes | None: try: img = Image.open(io.BytesIO(img_bytes)) if img.mode != "RGB": - img = img.convert("RGB") + img = img.convert("RGB") # type: ignore[assignment] buf = io.BytesIO() img.save(buf, format="JPEG", quality=85) return buf.getvalue() @@ -36,10 +36,10 @@ def _to_jpeg(img_bytes: bytes) -> bytes | None: class LlamaCppClient(GenAIClient): """Generative AI client for Frigate using llama.cpp server.""" - provider: str # base_url + provider: str | None # base_url provider_options: dict[str, Any] - def _init_provider(self): + def _init_provider(self) -> str | None: """Initialize the client.""" self.provider_options = { **self.genai_config.provider_options, @@ -75,7 +75,7 @@ class LlamaCppClient(GenAIClient): content.append( { "type": "image_url", - "image_url": { + "image_url": { # type: ignore[dict-item] "url": f"data:image/jpeg;base64,{encoded_image}", }, } @@ -111,7 +111,7 @@ class LlamaCppClient(GenAIClient): ): choice = result["choices"][0] if "message" in choice and "content" in choice["message"]: - return choice["message"]["content"].strip() + return str(choice["message"]["content"].strip()) return None except Exception as e: logger.warning("llama.cpp returned an error: %s", str(e)) @@ -229,7 +229,7 @@ class LlamaCppClient(GenAIClient): content.append( { "prompt_string": "<__media__>\n", - "multimodal_data": [encoded], + "multimodal_data": [encoded], # type: ignore[dict-item] } ) @@ -367,7 +367,7 @@ class LlamaCppClient(GenAIClient): messages: list[dict[str, Any]], tools: Optional[list[dict[str, Any]]] = None, tool_choice: Optional[str] = "auto", - ): + ) -> AsyncGenerator[tuple[str, Any], None]: """Stream chat with tools via OpenAI-compatible streaming API.""" if self.provider is None: logger.warning( diff --git a/frigate/genai/ollama.py b/frigate/genai/ollama.py index 0bfb95000..2af1a6350 100644 --- a/frigate/genai/ollama.py +++ b/frigate/genai/ollama.py @@ -2,7 +2,7 @@ import json import logging -from typing import Any, Optional +from typing import Any, AsyncGenerator, Optional from httpx import RemoteProtocolError, TimeoutException from ollama import AsyncClient as OllamaAsyncClient @@ -28,10 +28,10 @@ class OllamaClient(GenAIClient): }, } - provider: ApiClient + provider: ApiClient | None provider_options: dict[str, Any] - def _init_provider(self): + def _init_provider(self) -> ApiClient | None: """Initialize the client.""" self.provider_options = { **self.LOCAL_OPTIMIZED_OPTIONS, @@ -73,7 +73,7 @@ class OllamaClient(GenAIClient): "exclusiveMinimum", "exclusiveMaximum", } - result = {} + result: dict[str, Any] = {} for key, value in schema.items(): if not _is_properties and key in STRIP_KEYS: continue @@ -122,7 +122,7 @@ class OllamaClient(GenAIClient): logger.debug( f"Ollama tokens used: eval_count={result.get('eval_count')}, prompt_eval_count={result.get('prompt_eval_count')}" ) - return result["response"].strip() + return str(result["response"]).strip() except ( TimeoutException, ResponseError, @@ -263,7 +263,7 @@ class OllamaClient(GenAIClient): messages: list[dict[str, Any]], tools: Optional[list[dict[str, Any]]] = None, tool_choice: Optional[str] = "auto", - ): + ) -> AsyncGenerator[tuple[str, Any], None]: """Stream chat with tools; yields content deltas then final message. When tools are provided, Ollama streaming does not include tool_calls diff --git a/frigate/genai/openai.py b/frigate/genai/openai.py index 7d8700579..02ad301fa 100644 --- a/frigate/genai/openai.py +++ b/frigate/genai/openai.py @@ -3,7 +3,7 @@ import base64 import json import logging -from typing import Any, Optional +from typing import Any, AsyncGenerator, Optional from httpx import TimeoutException from openai import OpenAI @@ -21,7 +21,7 @@ class OpenAIClient(GenAIClient): provider: OpenAI context_size: Optional[int] = None - def _init_provider(self): + def _init_provider(self) -> OpenAI: """Initialize the client.""" # Extract context_size from provider_options as it's not a valid OpenAI client parameter # It will be used in get_context_size() instead @@ -81,7 +81,7 @@ class OpenAIClient(GenAIClient): and hasattr(result, "choices") and len(result.choices) > 0 ): - return result.choices[0].message.content.strip() + return str(result.choices[0].message.content.strip()) return None except (TimeoutException, Exception) as e: logger.warning("OpenAI returned an error: %s", str(e)) @@ -171,7 +171,7 @@ class OpenAIClient(GenAIClient): } request_params.update(provider_opts) - result = self.provider.chat.completions.create(**request_params) + result = self.provider.chat.completions.create(**request_params) # type: ignore[call-overload] if ( result is None @@ -245,7 +245,7 @@ class OpenAIClient(GenAIClient): messages: list[dict[str, Any]], tools: Optional[list[dict[str, Any]]] = None, tool_choice: Optional[str] = "auto", - ): + ) -> AsyncGenerator[tuple[str, Any], None]: """ Stream chat with tools; yields content deltas then final message. @@ -287,7 +287,7 @@ class OpenAIClient(GenAIClient): tool_calls_by_index: dict[int, dict[str, Any]] = {} finish_reason = "stop" - stream = self.provider.chat.completions.create(**request_params) + stream = self.provider.chat.completions.create(**request_params) # type: ignore[call-overload] for chunk in stream: if not chunk or not chunk.choices: diff --git a/frigate/jobs/media_sync.py b/frigate/jobs/media_sync.py index 803a80a9d..4a3fdc355 100644 --- a/frigate/jobs/media_sync.py +++ b/frigate/jobs/media_sync.py @@ -5,7 +5,7 @@ import os import threading from dataclasses import dataclass, field from datetime import datetime -from typing import Optional +from typing import Optional, cast from frigate.comms.inter_process import InterProcessRequestor from frigate.const import CONFIG_DIR, UPDATE_JOB_STATE @@ -122,7 +122,7 @@ def start_media_sync_job( if job_is_running("media_sync"): current = get_current_job("media_sync") logger.warning( - f"Media sync job {current.id} is already running. Rejecting new request." + f"Media sync job {current.id if current else 'unknown'} is already running. Rejecting new request." ) return None @@ -146,9 +146,9 @@ def start_media_sync_job( def get_current_media_sync_job() -> Optional[MediaSyncJob]: """Get the current running/queued media sync job, if any.""" - return get_current_job("media_sync") + return cast(Optional[MediaSyncJob], get_current_job("media_sync")) def get_media_sync_job_by_id(job_id: str) -> Optional[MediaSyncJob]: """Get media sync job by ID. Currently only tracks the current job.""" - return get_job_by_id("media_sync", job_id) + return cast(Optional[MediaSyncJob], get_job_by_id("media_sync", job_id)) diff --git a/frigate/jobs/motion_search.py b/frigate/jobs/motion_search.py index d7c8f8fbc..1a90f0bb9 100644 --- a/frigate/jobs/motion_search.py +++ b/frigate/jobs/motion_search.py @@ -6,7 +6,7 @@ import threading from concurrent.futures import Future, ThreadPoolExecutor, as_completed from dataclasses import asdict, dataclass, field from datetime import datetime -from typing import Any, Optional +from typing import Any, Optional, cast import cv2 import numpy as np @@ -96,7 +96,7 @@ def create_polygon_mask( dtype=np.int32, ) mask = np.zeros((frame_height, frame_width), dtype=np.uint8) - cv2.fillPoly(mask, [motion_points], 255) + cv2.fillPoly(mask, [motion_points], (255,)) return mask @@ -116,7 +116,7 @@ def compute_roi_bbox_normalized( def heatmap_overlaps_roi( - heatmap: dict[str, int], roi_bbox: tuple[float, float, float, float] + heatmap: object, roi_bbox: tuple[float, float, float, float] ) -> bool: """Check if a sparse motion heatmap has any overlap with the ROI bounding box. @@ -155,9 +155,9 @@ def segment_passes_activity_gate(recording: Recordings) -> bool: Returns True if any of motion, objects, or regions is non-zero/non-null. Returns True if all are null (old segments without data). """ - motion = recording.motion - objects = recording.objects - regions = recording.regions + motion: Any = recording.motion + objects: Any = recording.objects + regions: Any = recording.regions # Old segments without metadata - pass through (conservative) if motion is None and objects is None and regions is None: @@ -278,6 +278,9 @@ class MotionSearchRunner(threading.Thread): frame_width = camera_config.detect.width frame_height = camera_config.detect.height + if frame_width is None or frame_height is None: + raise ValueError(f"Camera {camera_name} detect dimensions not configured") + # Create polygon mask polygon_mask = create_polygon_mask( self.job.polygon_points, frame_width, frame_height @@ -415,11 +418,13 @@ class MotionSearchRunner(threading.Thread): if self._should_stop(): break + rec_start: float = recording.start_time # type: ignore[assignment] + rec_end: float = recording.end_time # type: ignore[assignment] future = executor.submit( self._process_recording_for_motion, - recording.path, - recording.start_time, - recording.end_time, + str(recording.path), + rec_start, + rec_end, self.job.start_time_range, self.job.end_time_range, polygon_mask, @@ -524,10 +529,12 @@ class MotionSearchRunner(threading.Thread): break try: + rec_start: float = recording.start_time # type: ignore[assignment] + rec_end: float = recording.end_time # type: ignore[assignment] results, frames = self._process_recording_for_motion( - recording.path, - recording.start_time, - recording.end_time, + str(recording.path), + rec_start, + rec_end, self.job.start_time_range, self.job.end_time_range, polygon_mask, @@ -672,7 +679,9 @@ class MotionSearchRunner(threading.Thread): # Handle frame dimension changes if gray.shape != polygon_mask.shape: resized_mask = cv2.resize( - polygon_mask, (gray.shape[1], gray.shape[0]), cv2.INTER_NEAREST + polygon_mask, + (gray.shape[1], gray.shape[0]), + interpolation=cv2.INTER_NEAREST, ) current_bbox = cv2.boundingRect(resized_mask) else: @@ -698,7 +707,7 @@ class MotionSearchRunner(threading.Thread): ) if prev_frame_gray is not None: - diff = cv2.absdiff(prev_frame_gray, masked_gray) + diff = cv2.absdiff(prev_frame_gray, masked_gray) # type: ignore[unreachable] diff_blurred = cv2.GaussianBlur(diff, (3, 3), 0) _, thresh = cv2.threshold( diff_blurred, threshold, 255, cv2.THRESH_BINARY @@ -825,7 +834,7 @@ def get_motion_search_job(job_id: str) -> Optional[MotionSearchJob]: if job_entry: return job_entry[0] # Check completed jobs via manager - return get_job_by_id("motion_search", job_id) + return cast(Optional[MotionSearchJob], get_job_by_id("motion_search", job_id)) def cancel_motion_search_job(job_id: str) -> bool: diff --git a/frigate/jobs/vlm_watch.py b/frigate/jobs/vlm_watch.py index dae5e5f41..a66f60dfc 100644 --- a/frigate/jobs/vlm_watch.py +++ b/frigate/jobs/vlm_watch.py @@ -54,9 +54,9 @@ class VLMWatchRunner(threading.Thread): job: VLMWatchJob, config: FrigateConfig, cancel_event: threading.Event, - frame_processor, - genai_manager, - dispatcher, + frame_processor: Any, + genai_manager: Any, + dispatcher: Any, ) -> None: super().__init__(daemon=True, name=f"vlm_watch_{job.id}") self.job = job @@ -226,9 +226,12 @@ class VLMWatchRunner(threading.Thread): remaining = deadline - time.time() if remaining <= 0: break - topic, payload = self.detection_subscriber.check_for_update( + result = self.detection_subscriber.check_for_update( timeout=min(1.0, remaining) ) + if result is None: + continue + topic, payload = result if topic is None or payload is None: continue # payload = (camera, frame_name, frame_time, tracked_objects, motion_boxes, regions) @@ -328,9 +331,9 @@ def start_vlm_watch_job( condition: str, max_duration_minutes: int, config: FrigateConfig, - frame_processor, - genai_manager, - dispatcher, + frame_processor: Any, + genai_manager: Any, + dispatcher: Any, labels: list[str] | None = None, zones: list[str] | None = None, ) -> str: diff --git a/frigate/motion/__init__.py b/frigate/motion/__init__.py index 1f6785d5d..58f781f46 100644 --- a/frigate/motion/__init__.py +++ b/frigate/motion/__init__.py @@ -13,10 +13,10 @@ class MotionDetector(ABC): frame_shape: Tuple[int, int, int], config: MotionConfig, fps: int, - improve_contrast, - threshold, - contour_area, - ): + improve_contrast: bool, + threshold: int, + contour_area: int | None, + ) -> None: pass @abstractmethod @@ -25,7 +25,7 @@ class MotionDetector(ABC): pass @abstractmethod - def is_calibrating(self): + def is_calibrating(self) -> bool: """Return if motion is recalibrating.""" pass @@ -35,6 +35,6 @@ class MotionDetector(ABC): pass @abstractmethod - def stop(self): + def stop(self) -> None: """Stop any ongoing work and processes.""" pass diff --git a/frigate/mypy.ini b/frigate/mypy.ini index 5bad10f49..e1da675be 100644 --- a/frigate/mypy.ini +++ b/frigate/mypy.ini @@ -41,6 +41,24 @@ ignore_errors = false [mypy-frigate.events] ignore_errors = false +[mypy-frigate.genai.*] +ignore_errors = false + +[mypy-frigate.jobs.*] +ignore_errors = false + +[mypy-frigate.motion] +ignore_errors = false + +[mypy-frigate.object_detection] +ignore_errors = false + +[mypy-frigate.output] +ignore_errors = false + +[mypy-frigate.ptz] +ignore_errors = false + [mypy-frigate.log] ignore_errors = false From b1c410bc3eb47d935ee96069d0ae6c610f93412a Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Wed, 25 Mar 2026 12:53:19 -0600 Subject: [PATCH 05/72] Optimize more mypy classes (#22637) * Cleanup motion mypy * Cleanup object detection mypy * Update output mypy * Cleanup --- frigate/detectors/plugins/memryx.py | 2 +- frigate/motion/frigate_motion.py | 29 +++++----- frigate/motion/improved_motion.py | 44 +++++++------- frigate/mypy.ini | 6 +- frigate/object_detection/base.py | 90 ++++++++++++++++------------- frigate/object_detection/util.py | 18 +++--- frigate/output/birdseye.py | 88 ++++++++++++++++------------ frigate/output/camera.py | 30 +++++----- frigate/output/output.py | 43 +++++++++----- frigate/output/preview.py | 57 +++++++++--------- 10 files changed, 228 insertions(+), 179 deletions(-) diff --git a/frigate/detectors/plugins/memryx.py b/frigate/detectors/plugins/memryx.py index e0ad401cb..2c03d14a4 100644 --- a/frigate/detectors/plugins/memryx.py +++ b/frigate/detectors/plugins/memryx.py @@ -317,7 +317,7 @@ class MemryXDetector(DetectionApi): f"Failed to remove downloaded zip {zip_path}: {e}" ) - def send_input(self, connection_id, tensor_input: np.ndarray): + def send_input(self, connection_id, tensor_input: np.ndarray) -> None: """Pre-process (if needed) and send frame to MemryX input queue""" if tensor_input is None: raise ValueError("[send_input] No image data provided for inference") diff --git a/frigate/motion/frigate_motion.py b/frigate/motion/frigate_motion.py index d49b0e861..8a067e1da 100644 --- a/frigate/motion/frigate_motion.py +++ b/frigate/motion/frigate_motion.py @@ -1,7 +1,9 @@ +from typing import Any + import cv2 import numpy as np -from frigate.config import MotionConfig +from frigate.config.config import RuntimeMotionConfig from frigate.motion import MotionDetector from frigate.util.image import grab_cv2_contours @@ -9,19 +11,20 @@ from frigate.util.image import grab_cv2_contours class FrigateMotionDetector(MotionDetector): def __init__( self, - frame_shape, - config: MotionConfig, + frame_shape: tuple[int, ...], + config: RuntimeMotionConfig, fps: int, - improve_contrast, - threshold, - contour_area, - ): + improve_contrast: Any, + threshold: Any, + contour_area: Any, + ) -> None: self.config = config self.frame_shape = frame_shape - self.resize_factor = frame_shape[0] / config.frame_height + frame_height = config.frame_height or frame_shape[0] + self.resize_factor = frame_shape[0] / frame_height self.motion_frame_size = ( - config.frame_height, - config.frame_height * frame_shape[1] // frame_shape[0], + frame_height, + frame_height * frame_shape[1] // frame_shape[0], ) self.avg_frame = np.zeros(self.motion_frame_size, np.float32) self.avg_delta = np.zeros(self.motion_frame_size, np.float32) @@ -38,10 +41,10 @@ class FrigateMotionDetector(MotionDetector): self.threshold = threshold self.contour_area = contour_area - def is_calibrating(self): + def is_calibrating(self) -> bool: return False - def detect(self, frame): + def detect(self, frame: np.ndarray) -> list: motion_boxes = [] gray = frame[0 : self.frame_shape[0], 0 : self.frame_shape[1]] @@ -99,7 +102,7 @@ class FrigateMotionDetector(MotionDetector): # dilate the thresholded image to fill in holes, then find contours # on thresholded image - thresh_dilated = cv2.dilate(thresh, None, iterations=2) + thresh_dilated = cv2.dilate(thresh, None, iterations=2) # type: ignore[call-overload] contours = cv2.findContours( thresh_dilated, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE ) diff --git a/frigate/motion/improved_motion.py b/frigate/motion/improved_motion.py index b821e9532..6694dafff 100644 --- a/frigate/motion/improved_motion.py +++ b/frigate/motion/improved_motion.py @@ -1,11 +1,12 @@ import logging +from typing import Optional import cv2 import numpy as np from scipy.ndimage import gaussian_filter from frigate.camera import PTZMetrics -from frigate.config import MotionConfig +from frigate.config.config import RuntimeMotionConfig from frigate.motion import MotionDetector from frigate.util.image import grab_cv2_contours @@ -15,22 +16,23 @@ logger = logging.getLogger(__name__) class ImprovedMotionDetector(MotionDetector): def __init__( self, - frame_shape, - config: MotionConfig, + frame_shape: tuple[int, ...], + config: RuntimeMotionConfig, fps: int, - ptz_metrics: PTZMetrics = None, - name="improved", - blur_radius=1, - interpolation=cv2.INTER_NEAREST, - contrast_frame_history=50, - ): + ptz_metrics: Optional[PTZMetrics] = None, + name: str = "improved", + blur_radius: int = 1, + interpolation: int = cv2.INTER_NEAREST, + contrast_frame_history: int = 50, + ) -> None: self.name = name self.config = config self.frame_shape = frame_shape - self.resize_factor = frame_shape[0] / config.frame_height + frame_height = config.frame_height or frame_shape[0] + self.resize_factor = frame_shape[0] / frame_height self.motion_frame_size = ( - config.frame_height, - config.frame_height * frame_shape[1] // frame_shape[0], + frame_height, + frame_height * frame_shape[1] // frame_shape[0], ) self.avg_frame = np.zeros(self.motion_frame_size, np.float32) self.motion_frame_count = 0 @@ -44,20 +46,20 @@ class ImprovedMotionDetector(MotionDetector): self.contrast_values[:, 1:2] = 255 self.contrast_values_index = 0 self.ptz_metrics = ptz_metrics - self.last_stop_time = None + self.last_stop_time: float | None = None - def is_calibrating(self): + def is_calibrating(self) -> bool: return self.calibrating - def detect(self, frame): - motion_boxes = [] + def detect(self, frame: np.ndarray) -> list[tuple[int, int, int, int]]: + motion_boxes: list[tuple[int, int, int, int]] = [] if not self.config.enabled: return motion_boxes # if ptz motor is moving from autotracking, quickly return # a single box that is 80% of the frame - if ( + if self.ptz_metrics is not None and ( self.ptz_metrics.autotracker_enabled.value and not self.ptz_metrics.motor_stopped.is_set() ): @@ -130,19 +132,19 @@ class ImprovedMotionDetector(MotionDetector): # dilate the thresholded image to fill in holes, then find contours # on thresholded image - thresh_dilated = cv2.dilate(thresh, None, iterations=1) + thresh_dilated = cv2.dilate(thresh, None, iterations=1) # type: ignore[call-overload] contours = cv2.findContours( thresh_dilated, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE ) contours = grab_cv2_contours(contours) # loop over the contours - total_contour_area = 0 + total_contour_area: float = 0 for c in contours: # if the contour is big enough, count it as motion contour_area = cv2.contourArea(c) total_contour_area += contour_area - if contour_area > self.config.contour_area: + if contour_area > (self.config.contour_area or 0): x, y, w, h = cv2.boundingRect(c) motion_boxes.append( ( @@ -159,7 +161,7 @@ class ImprovedMotionDetector(MotionDetector): # check if the motor has just stopped from autotracking # if so, reassign the average to the current frame so we begin with a new baseline - if ( + if self.ptz_metrics is not None and ( # ensure we only do this for cameras with autotracking enabled self.ptz_metrics.autotracker_enabled.value and self.ptz_metrics.motor_stopped.is_set() diff --git a/frigate/mypy.ini b/frigate/mypy.ini index e1da675be..9c44e9f38 100644 --- a/frigate/mypy.ini +++ b/frigate/mypy.ini @@ -47,13 +47,13 @@ ignore_errors = false [mypy-frigate.jobs.*] ignore_errors = false -[mypy-frigate.motion] +[mypy-frigate.motion.*] ignore_errors = false -[mypy-frigate.object_detection] +[mypy-frigate.object_detection.*] ignore_errors = false -[mypy-frigate.output] +[mypy-frigate.output.*] ignore_errors = false [mypy-frigate.ptz] diff --git a/frigate/object_detection/base.py b/frigate/object_detection/base.py index d2a54afbc..a62fe4843 100644 --- a/frigate/object_detection/base.py +++ b/frigate/object_detection/base.py @@ -7,6 +7,7 @@ from abc import ABC, abstractmethod from collections import deque from multiprocessing import Queue, Value from multiprocessing.synchronize import Event as MpEvent +from typing import Any, Optional import numpy as np import zmq @@ -34,26 +35,25 @@ logger = logging.getLogger(__name__) class ObjectDetector(ABC): @abstractmethod - def detect(self, tensor_input, threshold: float = 0.4): + def detect(self, tensor_input: np.ndarray, threshold: float = 0.4) -> list: pass class BaseLocalDetector(ObjectDetector): def __init__( self, - detector_config: BaseDetectorConfig = None, - labels: str = None, - stop_event: MpEvent = None, - ): + detector_config: Optional[BaseDetectorConfig] = None, + labels: Optional[str] = None, + stop_event: Optional[MpEvent] = None, + ) -> None: self.fps = EventsPerSecond() if labels is None: - self.labels = {} + self.labels: dict[int, str] = {} else: self.labels = load_labels(labels) - if detector_config: + if detector_config and detector_config.model: self.input_transform = tensor_transform(detector_config.model.input_tensor) - self.dtype = detector_config.model.input_dtype else: self.input_transform = None @@ -77,10 +77,10 @@ class BaseLocalDetector(ObjectDetector): return tensor_input - def detect(self, tensor_input: np.ndarray, threshold=0.4): + def detect(self, tensor_input: np.ndarray, threshold: float = 0.4) -> list: detections = [] - raw_detections = self.detect_raw(tensor_input) + raw_detections = self.detect_raw(tensor_input) # type: ignore[attr-defined] for d in raw_detections: if int(d[0]) < 0 or int(d[0]) >= len(self.labels): @@ -96,28 +96,28 @@ class BaseLocalDetector(ObjectDetector): class LocalObjectDetector(BaseLocalDetector): - def detect_raw(self, tensor_input: np.ndarray): + def detect_raw(self, tensor_input: np.ndarray) -> np.ndarray: tensor_input = self._transform_input(tensor_input) - return self.detect_api.detect_raw(tensor_input=tensor_input) + return self.detect_api.detect_raw(tensor_input=tensor_input) # type: ignore[no-any-return] class AsyncLocalObjectDetector(BaseLocalDetector): - def async_send_input(self, tensor_input: np.ndarray, connection_id: str): + def async_send_input(self, tensor_input: np.ndarray, connection_id: str) -> None: tensor_input = self._transform_input(tensor_input) - return self.detect_api.send_input(connection_id, tensor_input) + self.detect_api.send_input(connection_id, tensor_input) - def async_receive_output(self): + def async_receive_output(self) -> Any: return self.detect_api.receive_output() class DetectorRunner(FrigateProcess): def __init__( self, - name, + name: str, detection_queue: Queue, cameras: list[str], - avg_speed: Value, - start_time: Value, + avg_speed: Any, + start_time: Any, config: FrigateConfig, detector_config: BaseDetectorConfig, stop_event: MpEvent, @@ -129,11 +129,11 @@ class DetectorRunner(FrigateProcess): self.start_time = start_time self.config = config self.detector_config = detector_config - self.outputs: dict = {} + self.outputs: dict[str, Any] = {} - def create_output_shm(self, name: str): + def create_output_shm(self, name: str) -> None: out_shm = UntrackedSharedMemory(name=f"out-{name}", create=False) - out_np = np.ndarray((20, 6), dtype=np.float32, buffer=out_shm.buf) + out_np: np.ndarray = np.ndarray((20, 6), dtype=np.float32, buffer=out_shm.buf) self.outputs[name] = {"shm": out_shm, "np": out_np} def run(self) -> None: @@ -155,8 +155,8 @@ class DetectorRunner(FrigateProcess): connection_id, ( 1, - self.detector_config.model.height, - self.detector_config.model.width, + self.detector_config.model.height, # type: ignore[union-attr] + self.detector_config.model.width, # type: ignore[union-attr] 3, ), ) @@ -187,11 +187,11 @@ class DetectorRunner(FrigateProcess): class AsyncDetectorRunner(FrigateProcess): def __init__( self, - name, + name: str, detection_queue: Queue, cameras: list[str], - avg_speed: Value, - start_time: Value, + avg_speed: Any, + start_time: Any, config: FrigateConfig, detector_config: BaseDetectorConfig, stop_event: MpEvent, @@ -203,15 +203,15 @@ class AsyncDetectorRunner(FrigateProcess): self.start_time = start_time self.config = config self.detector_config = detector_config - self.outputs: dict = {} + self.outputs: dict[str, Any] = {} self._frame_manager: SharedMemoryFrameManager | None = None self._publisher: ObjectDetectorPublisher | None = None self._detector: AsyncLocalObjectDetector | None = None - self.send_times = deque() + self.send_times: deque[float] = deque() - def create_output_shm(self, name: str): + def create_output_shm(self, name: str) -> None: out_shm = UntrackedSharedMemory(name=f"out-{name}", create=False) - out_np = np.ndarray((20, 6), dtype=np.float32, buffer=out_shm.buf) + out_np: np.ndarray = np.ndarray((20, 6), dtype=np.float32, buffer=out_shm.buf) self.outputs[name] = {"shm": out_shm, "np": out_np} def _detect_worker(self) -> None: @@ -222,12 +222,13 @@ class AsyncDetectorRunner(FrigateProcess): except queue.Empty: continue + assert self._frame_manager is not None input_frame = self._frame_manager.get( connection_id, ( 1, - self.detector_config.model.height, - self.detector_config.model.width, + self.detector_config.model.height, # type: ignore[union-attr] + self.detector_config.model.width, # type: ignore[union-attr] 3, ), ) @@ -238,11 +239,13 @@ class AsyncDetectorRunner(FrigateProcess): # mark start time and send to accelerator self.send_times.append(time.perf_counter()) + assert self._detector is not None self._detector.async_send_input(input_frame, connection_id) def _result_worker(self) -> None: logger.info("Starting Result Worker Thread") while not self.stop_event.is_set(): + assert self._detector is not None connection_id, detections = self._detector.async_receive_output() # Handle timeout case (queue.Empty) - just continue @@ -256,6 +259,7 @@ class AsyncDetectorRunner(FrigateProcess): duration = time.perf_counter() - ts # release input buffer + assert self._frame_manager is not None self._frame_manager.close(connection_id) if connection_id not in self.outputs: @@ -264,6 +268,7 @@ class AsyncDetectorRunner(FrigateProcess): # write results and publish if detections is not None: self.outputs[connection_id]["np"][:] = detections[:] + assert self._publisher is not None self._publisher.publish(connection_id) # update timers @@ -330,11 +335,14 @@ class ObjectDetectProcess: self.stop_event = stop_event self.start_or_restart() - def stop(self): + def stop(self) -> None: # if the process has already exited on its own, just return if self.detect_process and self.detect_process.exitcode: return + if self.detect_process is None: + return + logging.info("Waiting for detection process to exit gracefully...") self.detect_process.join(timeout=30) if self.detect_process.exitcode is None: @@ -343,8 +351,8 @@ class ObjectDetectProcess: self.detect_process.join() logging.info("Detection process has exited...") - def start_or_restart(self): - self.detection_start.value = 0.0 + def start_or_restart(self) -> None: + self.detection_start.value = 0.0 # type: ignore[attr-defined] if (self.detect_process is not None) and self.detect_process.is_alive(): self.stop() @@ -389,17 +397,19 @@ class RemoteObjectDetector: self.detection_queue = detection_queue self.stop_event = stop_event self.shm = UntrackedSharedMemory(name=self.name, create=False) - self.np_shm = np.ndarray( + self.np_shm: np.ndarray = np.ndarray( (1, model_config.height, model_config.width, 3), dtype=np.uint8, buffer=self.shm.buf, ) self.out_shm = UntrackedSharedMemory(name=f"out-{self.name}", create=False) - self.out_np_shm = np.ndarray((20, 6), dtype=np.float32, buffer=self.out_shm.buf) + self.out_np_shm: np.ndarray = np.ndarray( + (20, 6), dtype=np.float32, buffer=self.out_shm.buf + ) self.detector_subscriber = ObjectDetectorSubscriber(name) - def detect(self, tensor_input, threshold=0.4): - detections = [] + def detect(self, tensor_input: np.ndarray, threshold: float = 0.4) -> list: + detections: list = [] if self.stop_event.is_set(): return detections @@ -431,7 +441,7 @@ class RemoteObjectDetector: self.fps.update() return detections - def cleanup(self): + def cleanup(self) -> None: self.detector_subscriber.stop() self.shm.unlink() self.out_shm.unlink() diff --git a/frigate/object_detection/util.py b/frigate/object_detection/util.py index ea8bd4226..4e351d66a 100644 --- a/frigate/object_detection/util.py +++ b/frigate/object_detection/util.py @@ -13,10 +13,10 @@ class RequestStore: A thread-safe hash-based response store that handles creating requests. """ - def __init__(self): + def __init__(self) -> None: self.request_counter = 0 self.request_counter_lock = threading.Lock() - self.input_queue = queue.Queue() + self.input_queue: queue.Queue[tuple[int, ndarray]] = queue.Queue() def __get_request_id(self) -> int: with self.request_counter_lock: @@ -45,17 +45,19 @@ class ResponseStore: their request's result appears. """ - def __init__(self): - self.responses = {} # Maps request_id -> (original_input, infer_results) + def __init__(self) -> None: + self.responses: dict[ + int, ndarray + ] = {} # Maps request_id -> (original_input, infer_results) self.lock = threading.Lock() self.cond = threading.Condition(self.lock) - def put(self, request_id: int, response: ndarray): + def put(self, request_id: int, response: ndarray) -> None: with self.cond: self.responses[request_id] = response self.cond.notify_all() - def get(self, request_id: int, timeout=None) -> ndarray: + def get(self, request_id: int, timeout: float | None = None) -> ndarray: with self.cond: if not self.cond.wait_for( lambda: request_id in self.responses, timeout=timeout @@ -65,7 +67,9 @@ class ResponseStore: return self.responses.pop(request_id) -def tensor_transform(desired_shape: InputTensorEnum): +def tensor_transform( + desired_shape: InputTensorEnum, +) -> tuple[int, int, int, int] | None: # Currently this function only supports BHWC permutations if desired_shape == InputTensorEnum.nhwc: return None diff --git a/frigate/output/birdseye.py b/frigate/output/birdseye.py index 5d80de33c..8b0fea6d7 100644 --- a/frigate/output/birdseye.py +++ b/frigate/output/birdseye.py @@ -4,13 +4,13 @@ import datetime import glob import logging import math -import multiprocessing as mp import os import queue import subprocess as sp import threading import time import traceback +from multiprocessing.synchronize import Event as MpEvent from typing import Any, Optional import cv2 @@ -74,25 +74,25 @@ class Canvas: self, canvas_width: int, canvas_height: int, - scaling_factor: int, + scaling_factor: float, ) -> None: self.scaling_factor = scaling_factor gcd = math.gcd(canvas_width, canvas_height) self.aspect = get_standard_aspect_ratio( - (canvas_width / gcd), (canvas_height / gcd) + int(canvas_width / gcd), int(canvas_height / gcd) ) self.width = canvas_width - self.height = (self.width * self.aspect[1]) / self.aspect[0] - self.coefficient_cache: dict[int, int] = {} + self.height: float = (self.width * self.aspect[1]) / self.aspect[0] + self.coefficient_cache: dict[int, float] = {} self.aspect_cache: dict[str, tuple[int, int]] = {} - def get_aspect(self, coefficient: int) -> tuple[int, int]: + def get_aspect(self, coefficient: float) -> tuple[float, float]: return (self.aspect[0] * coefficient, self.aspect[1] * coefficient) - def get_coefficient(self, camera_count: int) -> int: + def get_coefficient(self, camera_count: int) -> float: return self.coefficient_cache.get(camera_count, self.scaling_factor) - def set_coefficient(self, camera_count: int, coefficient: int) -> None: + def set_coefficient(self, camera_count: int, coefficient: float) -> None: self.coefficient_cache[camera_count] = coefficient def get_camera_aspect( @@ -105,7 +105,7 @@ class Canvas: gcd = math.gcd(camera_width, camera_height) camera_aspect = get_standard_aspect_ratio( - camera_width / gcd, camera_height / gcd + int(camera_width / gcd), int(camera_height / gcd) ) self.aspect_cache[cam_name] = camera_aspect return camera_aspect @@ -116,7 +116,7 @@ class FFMpegConverter(threading.Thread): self, ffmpeg: FfmpegConfig, input_queue: queue.Queue, - stop_event: mp.Event, + stop_event: MpEvent, in_width: int, in_height: int, out_width: int, @@ -128,7 +128,7 @@ class FFMpegConverter(threading.Thread): self.camera = "birdseye" self.input_queue = input_queue self.stop_event = stop_event - self.bd_pipe = None + self.bd_pipe: int | None = None if birdseye_rtsp: self.recreate_birdseye_pipe() @@ -181,7 +181,8 @@ class FFMpegConverter(threading.Thread): os.close(stdin) self.reading_birdseye = False - def __write(self, b) -> None: + def __write(self, b: bytes) -> None: + assert self.process.stdin is not None self.process.stdin.write(b) if self.bd_pipe: @@ -200,13 +201,13 @@ class FFMpegConverter(threading.Thread): return - def read(self, length): + def read(self, length: int) -> Any: try: - return self.process.stdout.read1(length) + return self.process.stdout.read1(length) # type: ignore[union-attr] except ValueError: return False - def exit(self): + def exit(self) -> None: if self.bd_pipe: os.close(self.bd_pipe) @@ -233,8 +234,8 @@ class BroadcastThread(threading.Thread): self, camera: str, converter: FFMpegConverter, - websocket_server, - stop_event: mp.Event, + websocket_server: Any, + stop_event: MpEvent, ): super().__init__() self.camera = camera @@ -242,7 +243,7 @@ class BroadcastThread(threading.Thread): self.websocket_server = websocket_server self.stop_event = stop_event - def run(self): + def run(self) -> None: while not self.stop_event.is_set(): buf = self.converter.read(65536) if buf: @@ -270,16 +271,16 @@ class BirdsEyeFrameManager: def __init__( self, config: FrigateConfig, - stop_event: mp.Event, + stop_event: MpEvent, ): self.config = config width, height = get_canvas_shape(config.birdseye.width, config.birdseye.height) self.frame_shape = (height, width) self.yuv_shape = (height * 3 // 2, width) - self.frame = np.ndarray(self.yuv_shape, dtype=np.uint8) + self.frame: np.ndarray = np.ndarray(self.yuv_shape, dtype=np.uint8) self.canvas = Canvas(width, height, config.birdseye.layout.scaling_factor) self.stop_event = stop_event - self.last_refresh_time = 0 + self.last_refresh_time: float = 0 # initialize the frame as black and with the Frigate logo self.blank_frame = np.zeros(self.yuv_shape, np.uint8) @@ -323,15 +324,15 @@ class BirdsEyeFrameManager: self.frame[:] = self.blank_frame - self.cameras = {} + self.cameras: dict[str, Any] = {} for camera in self.config.cameras.keys(): self.add_camera(camera) - self.camera_layout = [] - self.active_cameras = set() + self.camera_layout: list[Any] = [] + self.active_cameras: set[str] = set() self.last_output_time = 0.0 - def add_camera(self, cam: str): + def add_camera(self, cam: str) -> None: """Add a camera to self.cameras with the correct structure.""" settings = self.config.cameras[cam] # precalculate the coordinates for all the channels @@ -361,16 +362,21 @@ class BirdsEyeFrameManager: }, } - def remove_camera(self, cam: str): + def remove_camera(self, cam: str) -> None: """Remove a camera from self.cameras.""" if cam in self.cameras: del self.cameras[cam] - def clear_frame(self): + def clear_frame(self) -> None: logger.debug("Clearing the birdseye frame") self.frame[:] = self.blank_frame - def copy_to_position(self, position, camera=None, frame: np.ndarray = None): + def copy_to_position( + self, + position: Any, + camera: Optional[str] = None, + frame: Optional[np.ndarray] = None, + ) -> None: if camera is None: frame = None channel_dims = None @@ -389,7 +395,9 @@ class BirdsEyeFrameManager: channel_dims, ) - def camera_active(self, mode, object_box_count, motion_box_count): + def camera_active( + self, mode: Any, object_box_count: int, motion_box_count: int + ) -> bool: if mode == BirdseyeModeEnum.continuous: return True @@ -399,6 +407,8 @@ class BirdsEyeFrameManager: if mode == BirdseyeModeEnum.objects and object_box_count > 0: return True + return False + def get_camera_coordinates(self) -> dict[str, dict[str, int]]: """Return the coordinates of each camera in the current layout.""" coordinates = {} @@ -451,7 +461,7 @@ class BirdsEyeFrameManager: - self.cameras[active_camera]["last_active_frame"] ), ) - active_cameras = limited_active_cameras[:max_cameras] + active_cameras = set(limited_active_cameras[:max_cameras]) max_camera_refresh = True self.last_refresh_time = now @@ -510,7 +520,7 @@ class BirdsEyeFrameManager: # center camera view in canvas and ensure that it fits if scaled_width < self.canvas.width: - coefficient = 1 + coefficient: float = 1 x_offset = int((self.canvas.width - scaled_width) / 2) else: coefficient = self.canvas.width / scaled_width @@ -557,7 +567,7 @@ class BirdsEyeFrameManager: calculating = False self.canvas.set_coefficient(len(active_cameras), coefficient) - self.camera_layout = layout_candidate + self.camera_layout = layout_candidate or [] frame_changed = True # Draw the layout @@ -577,10 +587,12 @@ class BirdsEyeFrameManager: self, cameras_to_add: list[str], coefficient: float, - ) -> tuple[Any]: + ) -> Optional[list[list[Any]]]: """Calculate the optimal layout for 2+ cameras.""" - def map_layout(camera_layout: list[list[Any]], row_height: int): + def map_layout( + camera_layout: list[list[Any]], row_height: int + ) -> tuple[int, int, Optional[list[list[Any]]]]: """Map the calculated layout.""" candidate_layout = [] starting_x = 0 @@ -777,11 +789,11 @@ class Birdseye: def __init__( self, config: FrigateConfig, - stop_event: mp.Event, - websocket_server, + stop_event: MpEvent, + websocket_server: Any, ) -> None: self.config = config - self.input = queue.Queue(maxsize=10) + self.input: queue.Queue[bytes] = queue.Queue(maxsize=10) self.converter = FFMpegConverter( config.ffmpeg, self.input, @@ -806,7 +818,7 @@ class Birdseye: ) if config.birdseye.restream: - self.birdseye_buffer = self.frame_manager.create( + self.birdseye_buffer: Any = self.frame_manager.create( "birdseye", self.birdseye_manager.yuv_shape[0] * self.birdseye_manager.yuv_shape[1], ) diff --git a/frigate/output/camera.py b/frigate/output/camera.py index 2311ec659..917e38dd1 100644 --- a/frigate/output/camera.py +++ b/frigate/output/camera.py @@ -1,10 +1,11 @@ """Handle outputting individual cameras via jsmpeg.""" import logging -import multiprocessing as mp import queue import subprocess as sp import threading +from multiprocessing.synchronize import Event as MpEvent +from typing import Any from frigate.config import CameraConfig, FfmpegConfig @@ -17,7 +18,7 @@ class FFMpegConverter(threading.Thread): camera: str, ffmpeg: FfmpegConfig, input_queue: queue.Queue, - stop_event: mp.Event, + stop_event: MpEvent, in_width: int, in_height: int, out_width: int, @@ -64,16 +65,17 @@ class FFMpegConverter(threading.Thread): start_new_session=True, ) - def __write(self, b) -> None: + def __write(self, b: bytes) -> None: + assert self.process.stdin is not None self.process.stdin.write(b) - def read(self, length): + def read(self, length: int) -> Any: try: - return self.process.stdout.read1(length) + return self.process.stdout.read1(length) # type: ignore[union-attr] except ValueError: return False - def exit(self): + def exit(self) -> None: self.process.terminate() try: @@ -98,8 +100,8 @@ class BroadcastThread(threading.Thread): self, camera: str, converter: FFMpegConverter, - websocket_server, - stop_event: mp.Event, + websocket_server: Any, + stop_event: MpEvent, ): super().__init__() self.camera = camera @@ -107,7 +109,7 @@ class BroadcastThread(threading.Thread): self.websocket_server = websocket_server self.stop_event = stop_event - def run(self): + def run(self) -> None: while not self.stop_event.is_set(): buf = self.converter.read(65536) if buf: @@ -133,15 +135,15 @@ class BroadcastThread(threading.Thread): class JsmpegCamera: def __init__( - self, config: CameraConfig, stop_event: mp.Event, websocket_server + self, config: CameraConfig, stop_event: MpEvent, websocket_server: Any ) -> None: self.config = config - self.input = queue.Queue(maxsize=config.detect.fps) + self.input: queue.Queue[bytes] = queue.Queue(maxsize=config.detect.fps) width = int( config.live.height * (config.frame_shape[1] / config.frame_shape[0]) ) self.converter = FFMpegConverter( - config.name, + config.name or "", config.ffmpeg, self.input, stop_event, @@ -152,13 +154,13 @@ class JsmpegCamera: config.live.quality, ) self.broadcaster = BroadcastThread( - config.name, self.converter, websocket_server, stop_event + config.name or "", self.converter, websocket_server, stop_event ) self.converter.start() self.broadcaster.start() - def write_frame(self, frame_bytes) -> None: + def write_frame(self, frame_bytes: bytes) -> None: try: self.input.put_nowait(frame_bytes) except queue.Full: diff --git a/frigate/output/output.py b/frigate/output/output.py index 83962e1c9..22bcbb31f 100644 --- a/frigate/output/output.py +++ b/frigate/output/output.py @@ -61,6 +61,12 @@ def check_disabled_camera_update( # last camera update was more than 1 second ago # need to send empty data to birdseye because current # frame is now out of date + cam_width = config.cameras[camera].detect.width + cam_height = config.cameras[camera].detect.height + + if cam_width is None or cam_height is None: + raise ValueError(f"Camera {camera} detect dimensions not configured") + if birdseye and offline_time < 10: # we only need to send blank frames to birdseye at the beginning of a camera being offline birdseye.write_data( @@ -68,10 +74,7 @@ def check_disabled_camera_update( [], [], now, - get_blank_yuv_frame( - config.cameras[camera].detect.width, - config.cameras[camera].detect.height, - ), + get_blank_yuv_frame(cam_width, cam_height), ) if not has_enabled_camera and birdseye: @@ -173,7 +176,7 @@ class OutputProcess(FrigateProcess): birdseye_config_subscriber.check_for_update() ) - if update_topic is not None: + if update_topic is not None and birdseye_config is not None: previous_global_mode = self.config.birdseye.mode self.config.birdseye = birdseye_config @@ -198,7 +201,10 @@ class OutputProcess(FrigateProcess): birdseye, ) - (topic, data) = detection_subscriber.check_for_update(timeout=1) + _result = detection_subscriber.check_for_update(timeout=1) + if _result is None: + continue + (topic, data) = _result now = datetime.datetime.now().timestamp() if now - last_disabled_cam_check > 5: @@ -208,7 +214,7 @@ class OutputProcess(FrigateProcess): self.config, birdseye, preview_recorders, preview_write_times ) - if not topic: + if not topic or data is None: continue ( @@ -262,11 +268,15 @@ class OutputProcess(FrigateProcess): jsmpeg_cameras[camera].write_frame(frame.tobytes()) # send output data to birdseye if websocket is connected or restreaming - if self.config.birdseye.enabled and ( - self.config.birdseye.restream - or any( - ws.environ["PATH_INFO"].endswith("birdseye") - for ws in websocket_server.manager + if ( + self.config.birdseye.enabled + and birdseye is not None + and ( + self.config.birdseye.restream + or any( + ws.environ["PATH_INFO"].endswith("birdseye") + for ws in websocket_server.manager + ) ) ): birdseye.write_data( @@ -282,9 +292,12 @@ class OutputProcess(FrigateProcess): move_preview_frames("clips") while True: - (topic, data) = detection_subscriber.check_for_update(timeout=0) + _cleanup_result = detection_subscriber.check_for_update(timeout=0) + if _cleanup_result is None: + break + (topic, data) = _cleanup_result - if not topic: + if not topic or data is None: break ( @@ -322,7 +335,7 @@ class OutputProcess(FrigateProcess): logger.info("exiting output process...") -def move_preview_frames(loc: str): +def move_preview_frames(loc: str) -> None: preview_holdover = os.path.join(CLIPS_DIR, "preview_restart_cache") preview_cache = os.path.join(CACHE_DIR, "preview_frames") diff --git a/frigate/output/preview.py b/frigate/output/preview.py index 2c439038a..041d952d9 100644 --- a/frigate/output/preview.py +++ b/frigate/output/preview.py @@ -22,7 +22,6 @@ from frigate.ffmpeg_presets import ( parse_preset_hardware_acceleration_encode, ) from frigate.models import Previews -from frigate.track.object_processing import TrackedObject from frigate.util.image import copy_yuv_to_position, get_blank_yuv_frame, get_yuv_crop logger = logging.getLogger(__name__) @@ -66,7 +65,9 @@ def get_cache_image_name(camera: str, frame_time: float) -> str: ) -def get_most_recent_preview_frame(camera: str, before: float = None) -> str | None: +def get_most_recent_preview_frame( + camera: str, before: float | None = None +) -> str | None: """Get the most recent preview frame for a camera.""" if not os.path.exists(PREVIEW_CACHE_DIR): return None @@ -147,12 +148,12 @@ class FFMpegConverter(threading.Thread): if t_idx == item_count - 1: # last frame does not get a duration playlist.append( - f"file '{get_cache_image_name(self.config.name, self.frame_times[t_idx])}'" + f"file '{get_cache_image_name(self.config.name, self.frame_times[t_idx])}'" # type: ignore[arg-type] ) continue playlist.append( - f"file '{get_cache_image_name(self.config.name, self.frame_times[t_idx])}'" + f"file '{get_cache_image_name(self.config.name, self.frame_times[t_idx])}'" # type: ignore[arg-type] ) playlist.append( f"duration {self.frame_times[t_idx + 1] - self.frame_times[t_idx]}" @@ -199,30 +200,33 @@ class FFMpegConverter(threading.Thread): # unlink files from cache # don't delete last frame as it will be used as first frame in next segment for t in self.frame_times[0:-1]: - Path(get_cache_image_name(self.config.name, t)).unlink(missing_ok=True) + Path(get_cache_image_name(self.config.name, t)).unlink(missing_ok=True) # type: ignore[arg-type] class PreviewRecorder: def __init__(self, config: CameraConfig) -> None: self.config = config - self.start_time = 0 - self.last_output_time = 0 + self.camera_name: str = config.name or "" + self.start_time: float = 0 + self.last_output_time: float = 0 self.offline = False - self.output_frames = [] + self.output_frames: list[float] = [] - if config.detect.width > config.detect.height: + if config.detect.width is None or config.detect.height is None: + raise ValueError("Detect width and height must be set for previews.") + + self.detect_width: int = config.detect.width + self.detect_height: int = config.detect.height + + if self.detect_width > self.detect_height: self.out_height = PREVIEW_HEIGHT self.out_width = ( - int((config.detect.width / config.detect.height) * self.out_height) - // 4 - * 4 + int((self.detect_width / self.detect_height) * self.out_height) // 4 * 4 ) else: self.out_width = PREVIEW_HEIGHT self.out_height = ( - int((config.detect.height / config.detect.width) * self.out_width) - // 4 - * 4 + int((self.detect_height / self.detect_width) * self.out_width) // 4 * 4 ) # create communication for finished previews @@ -302,7 +306,7 @@ class PreviewRecorder: ) self.start_time = frame_time self.last_output_time = frame_time - self.output_frames: list[float] = [] + self.output_frames = [] def should_write_frame( self, @@ -342,7 +346,9 @@ class PreviewRecorder: def write_frame_to_cache(self, frame_time: float, frame: np.ndarray) -> None: # resize yuv frame - small_frame = np.zeros((self.out_height * 3 // 2, self.out_width), np.uint8) + small_frame: np.ndarray = np.zeros( + (self.out_height * 3 // 2, self.out_width), np.uint8 + ) copy_yuv_to_position( small_frame, (0, 0), @@ -356,7 +362,7 @@ class PreviewRecorder: cv2.COLOR_YUV2BGR_I420, ) cv2.imwrite( - get_cache_image_name(self.config.name, frame_time), + get_cache_image_name(self.camera_name, frame_time), small_frame, [ int(cv2.IMWRITE_WEBP_QUALITY), @@ -396,7 +402,7 @@ class PreviewRecorder: ).start() else: logger.debug( - f"Not saving preview for {self.config.name} because there are no saved frames." + f"Not saving preview for {self.camera_name} because there are no saved frames." ) self.reset_frame_cache(frame_time) @@ -416,9 +422,7 @@ class PreviewRecorder: if not self.offline: self.write_frame_to_cache( frame_time, - get_blank_yuv_frame( - self.config.detect.width, self.config.detect.height - ), + get_blank_yuv_frame(self.detect_width, self.detect_height), ) self.offline = True @@ -431,9 +435,9 @@ class PreviewRecorder: return old_frame_path = get_cache_image_name( - self.config.name, self.output_frames[-1] + self.camera_name, self.output_frames[-1] ) - new_frame_path = get_cache_image_name(self.config.name, frame_time) + new_frame_path = get_cache_image_name(self.camera_name, frame_time) shutil.copy(old_frame_path, new_frame_path) # save last frame to ensure consistent duration @@ -447,13 +451,12 @@ class PreviewRecorder: self.reset_frame_cache(frame_time) def stop(self) -> None: - self.config_subscriber.stop() self.requestor.stop() def get_active_objects( - frame_time: float, camera_config: CameraConfig, all_objects: list[TrackedObject] -) -> list[TrackedObject]: + frame_time: float, camera_config: CameraConfig, all_objects: list[dict[str, Any]] +) -> list[dict[str, Any]]: """get active objects for detection.""" return [ o From c0124938b3cad696fc198b1b96a39a37df372fda Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Wed, 25 Mar 2026 14:14:32 -0500 Subject: [PATCH 06/72] Tweaks (#22630) * fix stage overlay size * add audio filter config and load audio labels * remove add button from object and audio labels in settings * tests * update classification docs * tweak wording * don't require restart for timestamp_style changes * add optional i18n prefix for select widgets * use i18n enum prefix for timestamp position * add i18n for all presets --- .../object_classification.md | 13 +++- .../state_classification.md | 13 +++- frigate/config/camera/updater.py | 3 + frigate/config/config.py | 19 +++++- frigate/test/test_config.py | 61 ++++++++++++++++++- frigate/track/object_processing.py | 1 + web/public/locales/en/config/global.json | 2 +- web/public/locales/en/views/settings.json | 23 ++++++- .../config-form/section-configs/audio.ts | 10 +++ .../config-form/section-configs/objects.ts | 5 ++ .../section-configs/timestamp_style.ts | 5 +- .../theme/widgets/SelectWidget.tsx | 12 +++- web/src/views/live/LiveCameraView.tsx | 13 +++- 13 files changed, 168 insertions(+), 12 deletions(-) diff --git a/docs/docs/configuration/custom_classification/object_classification.md b/docs/docs/configuration/custom_classification/object_classification.md index caf05d8f3..7b5d73d75 100644 --- a/docs/docs/configuration/custom_classification/object_classification.md +++ b/docs/docs/configuration/custom_classification/object_classification.md @@ -102,8 +102,19 @@ If examples for some of your classes do not appear in the grid, you can continue ### Improving the Model +:::tip Diversity matters far more than volume + +Selecting dozens of nearly identical images is one of the fastest ways to degrade model performance. MobileNetV2 can overfit quickly when trained on homogeneous data — the model learns what *that exact moment* looked like rather than what actually defines the class. **This is why Frigate does not implement bulk training in the UI.** + +For more detail, see [Frigate Tip: Best Practices for Training Face and Custom Classification Models](https://github.com/blakeblackshear/frigate/discussions/21374). + +::: + +- **Start small and iterate**: Begin with a small, representative set of images per class. Models often begin working well with surprisingly few examples and improve naturally over time. +- **Favor hard examples**: When images appear in the Recent Classifications tab, prioritize images scoring below 90–100% or those captured under new lighting, weather, or distance conditions. +- **Avoid bulk training similar images**: Training large batches of images that already score 100% (or close) adds little new information and increases the risk of overfitting. +- **The wizard is just the starting point**: You don’t need to find and label every class upfront. Missing classes will naturally appear in Recent Classifications, and those images tend to be more valuable because they represent new conditions and edge cases. - **Problem framing**: Keep classes visually distinct and relevant to the chosen object types. -- **Data collection**: Use the model’s Recent Classification tab to gather balanced examples across times of day, weather, and distances. - **Preprocessing**: Ensure examples reflect object crops similar to Frigate’s boxes; keep the subject centered. - **Labels**: Keep label names short and consistent; include a `none` class if you plan to ignore uncertain predictions for sub labels. - **Threshold**: Tune `threshold` per model to reduce false assignments. Start at `0.8` and adjust based on validation. diff --git a/docs/docs/configuration/custom_classification/state_classification.md b/docs/docs/configuration/custom_classification/state_classification.md index c41d05439..3cadd9054 100644 --- a/docs/docs/configuration/custom_classification/state_classification.md +++ b/docs/docs/configuration/custom_classification/state_classification.md @@ -70,10 +70,21 @@ Once some images are assigned, training will begin automatically. ### Improving the Model +:::tip Diversity matters far more than volume + +Selecting dozens of nearly identical images is one of the fastest ways to degrade model performance. MobileNetV2 can overfit quickly when trained on homogeneous data — the model learns what *that exact moment* looked like rather than what actually defines the state. This often leads to models that work perfectly under the original conditions but become unstable when day turns to night, weather changes, or seasonal lighting shifts. **This is why Frigate does not implement bulk training in the UI.** + +For more detail, see [Frigate Tip: Best Practices for Training Face and Custom Classification Models](https://github.com/blakeblackshear/frigate/discussions/21374). + +::: + +- **Start small and iterate**: Begin with a small, representative set of images per class. Models often begin working well with surprisingly few examples and improve naturally over time. - **Problem framing**: Keep classes visually distinct and state-focused (e.g., `open`, `closed`, `unknown`). Avoid combining object identity with state in a single model unless necessary. - **Data collection**: Use the model's Recent Classifications tab to gather balanced examples across times of day and weather. - **When to train**: Focus on cases where the model is entirely incorrect or flips between states when it should not. There's no need to train additional images when the model is already working consistently. -- **Selecting training images**: Images scoring below 100% due to new conditions (e.g., first snow of the year, seasonal changes) or variations (e.g., objects temporarily in view, insects at night) are good candidates for training, as they represent scenarios different from the default state. Training these lower-scoring images that differ from existing training data helps prevent overfitting. Avoid training large quantities of images that look very similar, especially if they already score 100% as this can lead to overfitting. +- **Favor hard examples**: When images appear in the Recent Classifications tab, prioritize images scoring below 90–100% or those captured under new conditions (e.g., first snow of the year, seasonal changes, objects temporarily in view, insects at night). These represent scenarios different from the default state and help prevent overfitting. +- **Avoid bulk training similar images**: Training large batches of images that already score 100% (or close) adds little new information and increases the risk of overfitting. +- **The wizard is just the starting point**: You don't need to find and label every state upfront. Missing states will naturally appear in Recent Classifications, and those images tend to be more valuable because they represent new conditions and edge cases. ## Debugging Classification Models diff --git a/frigate/config/camera/updater.py b/frigate/config/camera/updater.py index 6474edf43..1965f3813 100644 --- a/frigate/config/camera/updater.py +++ b/frigate/config/camera/updater.py @@ -32,6 +32,7 @@ class CameraConfigUpdateEnum(str, Enum): face_recognition = "face_recognition" lpr = "lpr" snapshots = "snapshots" + timestamp_style = "timestamp_style" zones = "zones" @@ -133,6 +134,8 @@ class CameraConfigUpdateSubscriber: config.snapshots = updated_config elif update_type == CameraConfigUpdateEnum.onvif: config.onvif = updated_config + elif update_type == CameraConfigUpdateEnum.timestamp_style: + config.timestamp_style = updated_config elif update_type == CameraConfigUpdateEnum.zones: config.zones = updated_config diff --git a/frigate/config/config.py b/frigate/config/config.py index 19d0b73a3..d1b037509 100644 --- a/frigate/config/config.py +++ b/frigate/config/config.py @@ -25,6 +25,7 @@ from frigate.plus import PlusApi from frigate.util.builtin import ( deep_merge, get_ffmpeg_arg_list, + load_labels, ) from frigate.util.config import ( CURRENT_CONFIG_VERSION, @@ -40,7 +41,7 @@ from frigate.util.services import auto_detect_hwaccel from .auth import AuthConfig from .base import FrigateBaseModel from .camera import CameraConfig, CameraLiveConfig -from .camera.audio import AudioConfig +from .camera.audio import AudioConfig, AudioFilterConfig from .camera.birdseye import BirdseyeConfig from .camera.detect import DetectConfig from .camera.ffmpeg import FfmpegConfig @@ -473,7 +474,7 @@ class FrigateConfig(FrigateBaseModel): live: CameraLiveConfig = Field( default_factory=CameraLiveConfig, title="Live playback", - description="Settings used by the Web UI to control live stream resolution and quality.", + description="Settings to control the jsmpeg live stream resolution and quality. This does not affect restreamed cameras that use go2rtc for live view.", ) motion: Optional[MotionConfig] = Field( default=None, @@ -671,6 +672,12 @@ class FrigateConfig(FrigateBaseModel): detector_config.model = model self.detectors[key] = detector_config + all_audio_labels = { + label + for label in load_labels("/audio-labelmap.txt", prefill=521).values() + if label + } + for name, camera in self.cameras.items(): modified_global_config = global_config.copy() @@ -791,6 +798,14 @@ class FrigateConfig(FrigateBaseModel): camera_config.review.genai.enabled ) + if camera_config.audio.filters is None: + camera_config.audio.filters = {} + + audio_keys = all_audio_labels + audio_keys = audio_keys - camera_config.audio.filters.keys() + for key in audio_keys: + camera_config.audio.filters[key] = AudioFilterConfig() + # Add default filters object_keys = camera_config.objects.track if camera_config.objects.filters is None: diff --git a/frigate/test/test_config.py b/frigate/test/test_config.py index 61184c769..132be131f 100644 --- a/frigate/test/test_config.py +++ b/frigate/test/test_config.py @@ -10,7 +10,7 @@ from ruamel.yaml.constructor import DuplicateKeyError from frigate.config import BirdseyeModeEnum, FrigateConfig from frigate.const import MODEL_CACHE_DIR from frigate.detectors import DetectorTypeEnum -from frigate.util.builtin import deep_merge +from frigate.util.builtin import deep_merge, load_labels class TestConfig(unittest.TestCase): @@ -288,6 +288,65 @@ class TestConfig(unittest.TestCase): frigate_config = FrigateConfig(**config) assert "dog" in frigate_config.cameras["back"].objects.filters + def test_default_audio_filters(self): + config = { + "mqtt": {"host": "mqtt"}, + "audio": {"listen": ["speech", "yell"]}, + "cameras": { + "back": { + "ffmpeg": { + "inputs": [ + {"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]} + ] + }, + "detect": { + "height": 1080, + "width": 1920, + "fps": 5, + }, + } + }, + } + + frigate_config = FrigateConfig(**config) + all_audio_labels = { + label + for label in load_labels("/audio-labelmap.txt", prefill=521).values() + if label + } + + assert all_audio_labels.issubset( + set(frigate_config.cameras["back"].audio.filters.keys()) + ) + + def test_override_audio_filters(self): + config = { + "mqtt": {"host": "mqtt"}, + "cameras": { + "back": { + "ffmpeg": { + "inputs": [ + {"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]} + ] + }, + "detect": { + "height": 1080, + "width": 1920, + "fps": 5, + }, + "audio": { + "listen": ["speech", "yell"], + "filters": {"speech": {"threshold": 0.9}}, + }, + } + }, + } + + frigate_config = FrigateConfig(**config) + assert "speech" in frigate_config.cameras["back"].audio.filters + assert frigate_config.cameras["back"].audio.filters["speech"].threshold == 0.9 + assert "babbling" in frigate_config.cameras["back"].audio.filters + def test_inherit_object_filters(self): config = { "mqtt": {"host": "mqtt"}, diff --git a/frigate/track/object_processing.py b/frigate/track/object_processing.py index 1a15e27ee..3fae8da6f 100644 --- a/frigate/track/object_processing.py +++ b/frigate/track/object_processing.py @@ -81,6 +81,7 @@ class TrackedObjectProcessor(threading.Thread): CameraConfigUpdateEnum.motion, CameraConfigUpdateEnum.objects, CameraConfigUpdateEnum.remove, + CameraConfigUpdateEnum.timestamp_style, CameraConfigUpdateEnum.zones, ], ) diff --git a/web/public/locales/en/config/global.json b/web/public/locales/en/config/global.json index e653818fa..7ca564ee6 100644 --- a/web/public/locales/en/config/global.json +++ b/web/public/locales/en/config/global.json @@ -752,7 +752,7 @@ }, "live": { "label": "Live playback", - "description": "Settings used by the Web UI to control live stream resolution and quality.", + "description": "Settings to control the jsmpeg live stream resolution and quality. This does not affect restreamed cameras that use go2rtc for live view.", "streams": { "label": "Live stream names", "description": "Mapping of configured stream names to restream/go2rtc names used for live playback." diff --git a/web/public/locales/en/views/settings.json b/web/public/locales/en/views/settings.json index 4109b4821..e5c2c7851 100644 --- a/web/public/locales/en/views/settings.json +++ b/web/public/locales/en/views/settings.json @@ -825,6 +825,12 @@ "area": "Area" } }, + "timestampPosition": { + "tl": "Top left", + "tr": "Top right", + "bl": "Bottom left", + "br": "Bottom right" + }, "users": { "title": "Users", "management": { @@ -1342,7 +1348,22 @@ "preset-nvidia": "NVIDIA GPU", "preset-jetson-h264": "NVIDIA Jetson (H.264)", "preset-jetson-h265": "NVIDIA Jetson (H.265)", - "preset-rkmpp": "Rockchip RKMPP" + "preset-rkmpp": "Rockchip RKMPP", + "preset-http-jpeg-generic": "HTTP JPEG (Generic)", + "preset-http-mjpeg-generic": "HTTP MJPEG (Generic)", + "preset-http-reolink": "HTTP - Reolink Cameras", + "preset-rtmp-generic": "RTMP (Generic)", + "preset-rtsp-generic": "RTSP (Generic)", + "preset-rtsp-restream": "RTSP - Restream from go2rtc", + "preset-rtsp-restream-low-latency": "RTSP - Restream from go2rtc (Low Latency)", + "preset-rtsp-udp": "RTSP - UDP", + "preset-rtsp-blue-iris": "RTSP - Blue Iris", + "preset-record-generic": "Record (Generic, no audio)", + "preset-record-generic-audio-copy": "Record (Generic + Copy Audio)", + "preset-record-generic-audio-aac": "Record (Generic + Audio to AAC)", + "preset-record-mjpeg": "Record - MJPEG Cameras", + "preset-record-jpeg": "Record - JPEG Cameras", + "preset-record-ubiquiti": "Record - Ubiquiti Cameras" } }, "cameraInputs": { diff --git a/web/src/components/config-form/section-configs/audio.ts b/web/src/components/config-form/section-configs/audio.ts index 740d76f78..a112fa0db 100644 --- a/web/src/components/config-form/section-configs/audio.ts +++ b/web/src/components/config-form/section-configs/audio.ts @@ -19,6 +19,16 @@ const audio: SectionConfigOverrides = { hiddenFields: ["enabled_in_config"], advancedFields: ["min_volume", "max_not_heard", "num_threads"], uiSchema: { + filters: { + "ui:options": { + expandable: false, + }, + }, + "filters.*": { + "ui:options": { + additionalPropertyKeyReadonly: true, + }, + }, listen: { "ui:widget": "audioLabels", }, diff --git a/web/src/components/config-form/section-configs/objects.ts b/web/src/components/config-form/section-configs/objects.ts index bf5f6c350..e30ddf9d9 100644 --- a/web/src/components/config-form/section-configs/objects.ts +++ b/web/src/components/config-form/section-configs/objects.ts @@ -29,6 +29,11 @@ const objects: SectionConfigOverrides = { ], advancedFields: ["genai"], uiSchema: { + filters: { + "ui:options": { + expandable: false, + }, + }, "filters.*.min_area": { "ui:options": { suppressMultiSchema: true, diff --git a/web/src/components/config-form/section-configs/timestamp_style.ts b/web/src/components/config-form/section-configs/timestamp_style.ts index 2f51b2416..e43373c26 100644 --- a/web/src/components/config-form/section-configs/timestamp_style.ts +++ b/web/src/components/config-form/section-configs/timestamp_style.ts @@ -4,12 +4,13 @@ const timestampStyle: SectionConfigOverrides = { base: { sectionDocs: "/configuration/reference", restartRequired: [], - fieldOrder: ["position", "format", "color", "thickness"], + fieldOrder: ["position", "format", "thickness", "color"], hiddenFields: ["effect", "enabled_in_config"], advancedFields: [], uiSchema: { position: { "ui:size": "xs", + "ui:options": { enumI18nPrefix: "timestampPosition" }, }, format: { "ui:size": "xs", @@ -17,7 +18,7 @@ const timestampStyle: SectionConfigOverrides = { }, }, global: { - restartRequired: ["position", "format", "color", "thickness", "effect"], + restartRequired: [], }, camera: { restartRequired: [], diff --git a/web/src/components/config-form/theme/widgets/SelectWidget.tsx b/web/src/components/config-form/theme/widgets/SelectWidget.tsx index d5047e959..46c2d0701 100644 --- a/web/src/components/config-form/theme/widgets/SelectWidget.tsx +++ b/web/src/components/config-form/theme/widgets/SelectWidget.tsx @@ -1,5 +1,6 @@ // Select Widget - maps to shadcn/ui Select import type { WidgetProps } from "@rjsf/utils"; +import { useTranslation } from "react-i18next"; import { Select, SelectContent, @@ -21,9 +22,18 @@ export function SelectWidget(props: WidgetProps) { schema, } = props; + const { t } = useTranslation(["views/settings"]); const { enumOptions = [] } = options; + const enumI18nPrefix = options["enumI18nPrefix"] as string | undefined; const fieldClassName = getSizedFieldClassName(options, "sm"); + const getLabel = (option: { value: unknown; label: string }) => { + if (enumI18nPrefix) { + return t(`${enumI18nPrefix}.${option.value}`); + } + return option.label; + }; + return (