mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-08-07 03:11:15 +03:00
Semantic Search for Detections (#11899)
* Initial re-implementation of semantic search * put docker-compose back and make reindex match docs * remove debug code and fix import * fix docs * manually build pysqlite3 as binaries are only available for x86-64 * update comment in build_pysqlite3.sh * only embed objects * better error handling when genai fails * ask ollama to pull requested model at startup * update ollama docs * address some PR review comments * fix lint * use IPC to write description, update docs for reindex * remove gemini-pro-vision from docs as it will be unavailable soon * fix OpenAI doc available models * fix api error in gemini and metadata for embeddings
This commit is contained in:
Jason Hunter
committed by
Nicolas Mowen
Nicolas Mowen
parent
f4f3cfa911
commit
36cbffcc5e
@@ -1,14 +1,9 @@
|
||||
"""Facilitates communication between processes."""
|
||||
|
||||
import threading
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
|
||||
import zmq
|
||||
|
||||
SOCKET_CONTROL = "inproc://control.detections_updater"
|
||||
SOCKET_PUB = "ipc:///tmp/cache/detect_pub"
|
||||
SOCKET_SUB = "ipc:///tmp/cache/detect_sub"
|
||||
from .zmq_proxy import Publisher, Subscriber
|
||||
|
||||
|
||||
class DetectionTypeEnum(str, Enum):
|
||||
@@ -18,85 +13,31 @@ class DetectionTypeEnum(str, Enum):
|
||||
audio = "audio"
|
||||
|
||||
|
||||
class DetectionProxyRunner(threading.Thread):
|
||||
def __init__(self, context: zmq.Context[zmq.Socket]) -> None:
|
||||
threading.Thread.__init__(self)
|
||||
self.name = "detection_proxy"
|
||||
self.context = context
|
||||
|
||||
def run(self) -> None:
|
||||
"""Run the proxy."""
|
||||
control = self.context.socket(zmq.REP)
|
||||
control.connect(SOCKET_CONTROL)
|
||||
incoming = self.context.socket(zmq.XSUB)
|
||||
incoming.bind(SOCKET_PUB)
|
||||
outgoing = self.context.socket(zmq.XPUB)
|
||||
outgoing.bind(SOCKET_SUB)
|
||||
|
||||
zmq.proxy_steerable(
|
||||
incoming, outgoing, None, control
|
||||
) # blocking, will unblock terminate message is received
|
||||
incoming.close()
|
||||
outgoing.close()
|
||||
|
||||
|
||||
class DetectionProxy:
|
||||
"""Proxies video and audio detections."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.context = zmq.Context()
|
||||
self.control = self.context.socket(zmq.REQ)
|
||||
self.control.bind(SOCKET_CONTROL)
|
||||
self.runner = DetectionProxyRunner(self.context)
|
||||
self.runner.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self.control.send("TERMINATE".encode()) # tell the proxy to stop
|
||||
self.runner.join()
|
||||
self.context.destroy()
|
||||
|
||||
|
||||
class DetectionPublisher:
|
||||
class DetectionPublisher(Publisher):
|
||||
"""Simplifies receiving video and audio detections."""
|
||||
|
||||
topic_base = "detection/"
|
||||
|
||||
def __init__(self, topic: DetectionTypeEnum) -> None:
|
||||
self.topic = topic
|
||||
self.context = zmq.Context()
|
||||
self.socket = self.context.socket(zmq.PUB)
|
||||
self.socket.connect(SOCKET_PUB)
|
||||
|
||||
def send_data(self, payload: any) -> None:
|
||||
"""Publish detection."""
|
||||
self.socket.send_string(self.topic.value, flags=zmq.SNDMORE)
|
||||
self.socket.send_json(payload)
|
||||
|
||||
def stop(self) -> None:
|
||||
self.socket.close()
|
||||
self.context.destroy()
|
||||
topic = topic.value
|
||||
super().__init__(topic)
|
||||
|
||||
|
||||
class DetectionSubscriber:
|
||||
class DetectionSubscriber(Subscriber):
|
||||
"""Simplifies receiving video and audio detections."""
|
||||
|
||||
topic_base = "detection/"
|
||||
|
||||
def __init__(self, topic: DetectionTypeEnum) -> None:
|
||||
self.context = zmq.Context()
|
||||
self.socket = self.context.socket(zmq.SUB)
|
||||
self.socket.setsockopt_string(zmq.SUBSCRIBE, topic.value)
|
||||
self.socket.connect(SOCKET_SUB)
|
||||
topic = topic.value
|
||||
super().__init__(topic)
|
||||
|
||||
def get_data(self, timeout: float = None) -> Optional[tuple[str, any]]:
|
||||
"""Returns detections or None if no update."""
|
||||
try:
|
||||
has_update, _, _ = zmq.select([self.socket], [], [], timeout)
|
||||
def check_for_update(
|
||||
self, timeout: float = None
|
||||
) -> Optional[tuple[DetectionTypeEnum, any]]:
|
||||
return super().check_for_update(timeout)
|
||||
|
||||
if has_update:
|
||||
topic = DetectionTypeEnum[self.socket.recv_string(flags=zmq.NOBLOCK)]
|
||||
return (topic, self.socket.recv_json())
|
||||
except zmq.ZMQError:
|
||||
pass
|
||||
|
||||
return (None, None)
|
||||
|
||||
def stop(self) -> None:
|
||||
self.socket.close()
|
||||
self.context.destroy()
|
||||
def _return_object(self, topic: str, payload: any) -> any:
|
||||
if payload is None:
|
||||
return (None, None)
|
||||
return (DetectionTypeEnum[topic[len(self.topic_base) :]], payload)
|
||||
|
||||
@@ -14,9 +14,10 @@ from frigate.const import (
|
||||
INSERT_PREVIEW,
|
||||
REQUEST_REGION_GRID,
|
||||
UPDATE_CAMERA_ACTIVITY,
|
||||
UPDATE_EVENT_DESCRIPTION,
|
||||
UPSERT_REVIEW_SEGMENT,
|
||||
)
|
||||
from frigate.models import Previews, Recordings, ReviewSegment
|
||||
from frigate.models import Event, Previews, Recordings, ReviewSegment
|
||||
from frigate.ptz.onvif import OnvifCommandEnum, OnvifController
|
||||
from frigate.types import PTZMetricsTypes
|
||||
from frigate.util.object import get_camera_regions_grid
|
||||
@@ -128,6 +129,10 @@ class Dispatcher:
|
||||
).execute()
|
||||
elif topic == UPDATE_CAMERA_ACTIVITY:
|
||||
self.camera_activity = payload
|
||||
elif topic == UPDATE_EVENT_DESCRIPTION:
|
||||
event: Event = Event.get(Event.id == payload["id"])
|
||||
event.data["description"] = payload["description"]
|
||||
event.save()
|
||||
elif topic == "onConnect":
|
||||
camera_status = self.camera_activity.copy()
|
||||
|
||||
|
||||
@@ -1,100 +1,51 @@
|
||||
"""Facilitates communication between processes."""
|
||||
|
||||
import zmq
|
||||
|
||||
from frigate.events.types import EventStateEnum, EventTypeEnum
|
||||
|
||||
SOCKET_PUSH_PULL = "ipc:///tmp/cache/events"
|
||||
SOCKET_PUSH_PULL_END = "ipc:///tmp/cache/events_ended"
|
||||
from .zmq_proxy import Publisher, Subscriber
|
||||
|
||||
|
||||
class EventUpdatePublisher:
|
||||
class EventUpdatePublisher(Publisher):
|
||||
"""Publishes events (objects, audio, manual)."""
|
||||
|
||||
topic_base = "event/"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.context = zmq.Context()
|
||||
self.socket = self.context.socket(zmq.PUSH)
|
||||
self.socket.connect(SOCKET_PUSH_PULL)
|
||||
super().__init__("update")
|
||||
|
||||
def publish(
|
||||
self, payload: tuple[EventTypeEnum, EventStateEnum, str, dict[str, any]]
|
||||
) -> None:
|
||||
"""There is no communication back to the processes."""
|
||||
self.socket.send_json(payload)
|
||||
|
||||
def stop(self) -> None:
|
||||
self.socket.close()
|
||||
self.context.destroy()
|
||||
super().publish(payload)
|
||||
|
||||
|
||||
class EventUpdateSubscriber:
|
||||
class EventUpdateSubscriber(Subscriber):
|
||||
"""Receives event updates."""
|
||||
|
||||
topic_base = "event/"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.context = zmq.Context()
|
||||
self.socket = self.context.socket(zmq.PULL)
|
||||
self.socket.bind(SOCKET_PUSH_PULL)
|
||||
|
||||
def check_for_update(
|
||||
self, timeout=1
|
||||
) -> tuple[EventTypeEnum, EventStateEnum, str, dict[str, any]]:
|
||||
"""Returns events or None if no update."""
|
||||
try:
|
||||
has_update, _, _ = zmq.select([self.socket], [], [], timeout)
|
||||
|
||||
if has_update:
|
||||
return self.socket.recv_json()
|
||||
except zmq.ZMQError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
def stop(self) -> None:
|
||||
self.socket.close()
|
||||
self.context.destroy()
|
||||
super().__init__("update")
|
||||
|
||||
|
||||
class EventEndPublisher:
|
||||
class EventEndPublisher(Publisher):
|
||||
"""Publishes events that have ended."""
|
||||
|
||||
topic_base = "event/"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.context = zmq.Context()
|
||||
self.socket = self.context.socket(zmq.PUSH)
|
||||
self.socket.connect(SOCKET_PUSH_PULL_END)
|
||||
super().__init__("finalized")
|
||||
|
||||
def publish(
|
||||
self, payload: tuple[EventTypeEnum, EventStateEnum, str, dict[str, any]]
|
||||
) -> None:
|
||||
"""There is no communication back to the processes."""
|
||||
self.socket.send_json(payload)
|
||||
|
||||
def stop(self) -> None:
|
||||
self.socket.close()
|
||||
self.context.destroy()
|
||||
super().publish(payload)
|
||||
|
||||
|
||||
class EventEndSubscriber:
|
||||
class EventEndSubscriber(Subscriber):
|
||||
"""Receives events that have ended."""
|
||||
|
||||
topic_base = "event/"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.context = zmq.Context()
|
||||
self.socket = self.context.socket(zmq.PULL)
|
||||
self.socket.bind(SOCKET_PUSH_PULL_END)
|
||||
|
||||
def check_for_update(
|
||||
self, timeout=1
|
||||
) -> tuple[EventTypeEnum, EventStateEnum, str, dict[str, any]]:
|
||||
"""Returns events ended or None if no update."""
|
||||
try:
|
||||
has_update, _, _ = zmq.select([self.socket], [], [], timeout)
|
||||
|
||||
if has_update:
|
||||
return self.socket.recv_json()
|
||||
except zmq.ZMQError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
def stop(self) -> None:
|
||||
self.socket.close()
|
||||
self.context.destroy()
|
||||
super().__init__("finalized")
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Facilitates communication over zmq proxy."""
|
||||
|
||||
import threading
|
||||
from typing import Optional
|
||||
|
||||
import zmq
|
||||
|
||||
SOCKET_PUB = "ipc:///tmp/cache/proxy_pub"
|
||||
SOCKET_SUB = "ipc:///tmp/cache/proxy_sub"
|
||||
|
||||
|
||||
class ZmqProxyRunner(threading.Thread):
|
||||
def __init__(self, context: zmq.Context[zmq.Socket]) -> None:
|
||||
threading.Thread.__init__(self)
|
||||
self.name = "detection_proxy"
|
||||
self.context = context
|
||||
|
||||
def run(self) -> None:
|
||||
"""Run the proxy."""
|
||||
incoming = self.context.socket(zmq.XSUB)
|
||||
incoming.bind(SOCKET_PUB)
|
||||
outgoing = self.context.socket(zmq.XPUB)
|
||||
outgoing.bind(SOCKET_SUB)
|
||||
|
||||
# Blocking: This will unblock (via exception) when we destroy the context
|
||||
# The incoming and outgoing sockets will be closed automatically
|
||||
# when the context is destroyed as well.
|
||||
try:
|
||||
zmq.proxy(incoming, outgoing)
|
||||
except zmq.ZMQError:
|
||||
pass
|
||||
|
||||
|
||||
class ZmqProxy:
|
||||
"""Proxies video and audio detections."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.context = zmq.Context()
|
||||
self.runner = ZmqProxyRunner(self.context)
|
||||
self.runner.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
# destroying the context will tell the proxy to stop
|
||||
self.context.destroy()
|
||||
self.runner.join()
|
||||
|
||||
|
||||
class Publisher:
|
||||
"""Publishes messages."""
|
||||
|
||||
topic_base: str = ""
|
||||
|
||||
def __init__(self, topic: str = "") -> None:
|
||||
self.topic = f"{self.topic_base}{topic}"
|
||||
self.context = zmq.Context()
|
||||
self.socket = self.context.socket(zmq.PUB)
|
||||
self.socket.connect(SOCKET_PUB)
|
||||
|
||||
def publish(self, payload: any, sub_topic: str = "") -> None:
|
||||
"""Publish message."""
|
||||
self.socket.send_string(f"{self.topic}{sub_topic}", flags=zmq.SNDMORE)
|
||||
self.socket.send_json(payload)
|
||||
|
||||
def stop(self) -> None:
|
||||
self.socket.close()
|
||||
self.context.destroy()
|
||||
|
||||
|
||||
class Subscriber:
|
||||
"""Receives messages."""
|
||||
|
||||
topic_base: str = ""
|
||||
|
||||
def __init__(self, topic: str = "") -> None:
|
||||
self.topic = f"{self.topic_base}{topic}"
|
||||
self.context = zmq.Context()
|
||||
self.socket = self.context.socket(zmq.SUB)
|
||||
self.socket.setsockopt_string(zmq.SUBSCRIBE, self.topic)
|
||||
self.socket.connect(SOCKET_SUB)
|
||||
|
||||
def check_for_update(self, timeout: float = 1) -> Optional[tuple[str, any]]:
|
||||
"""Returns message or None if no update."""
|
||||
try:
|
||||
has_update, _, _ = zmq.select([self.socket], [], [], timeout)
|
||||
|
||||
if has_update:
|
||||
topic = self.socket.recv_string(flags=zmq.NOBLOCK)
|
||||
payload = self.socket.recv_json()
|
||||
return self._return_object(topic, payload)
|
||||
except zmq.ZMQError:
|
||||
pass
|
||||
|
||||
return self._return_object("", None)
|
||||
|
||||
def stop(self) -> None:
|
||||
self.socket.close()
|
||||
self.context.destroy()
|
||||
|
||||
def _return_object(self, topic: str, payload: any) -> any:
|
||||
return payload
|
||||
Reference in New Issue
Block a user