+ {reindexProgress.time_remaining >= 0 && (
+
+
+ Estimated time remaining:
+
+ {formatSecondsToDuration(
+ reindexProgress.time_remaining,
+ ) || "Finishing shortly"}
+
+ )}
Thumbnails embedded:
diff --git a/web/src/types/ws.ts b/web/src/types/ws.ts
index cab759074..238ef4a8a 100644
--- a/web/src/types/ws.ts
+++ b/web/src/types/ws.ts
@@ -67,6 +67,7 @@ export type EmbeddingsReindexProgressType = {
descriptions: number;
processed_objects: number;
total_objects: number;
+ time_remaining: number;
};
export type ToggleableSetting = "ON" | "OFF";
diff --git a/web/src/utils/dateUtil.ts b/web/src/utils/dateUtil.ts
index dc5589884..8509155e1 100644
--- a/web/src/utils/dateUtil.ts
+++ b/web/src/utils/dateUtil.ts
@@ -229,6 +229,23 @@ export const getDurationFromTimestamps = (
return duration;
};
+/**
+ *
+ * @param seconds - number of seconds to convert into hours, minutes and seconds
+ * @returns string - formatted duration in hours, minutes and seconds
+ */
+export const formatSecondsToDuration = (seconds: number): string => {
+ if (isNaN(seconds) || seconds < 0) {
+ return "Invalid duration";
+ }
+
+ const duration = intervalToDuration({ start: 0, end: seconds * 1000 });
+ return formatDuration(duration, {
+ format: ["hours", "minutes", "seconds"],
+ delimiter: ", ",
+ });
+};
+
/**
* Adapted from https://stackoverflow.com/a/29268535 this takes a timezone string and
* returns the offset of that timezone from UTC in minutes.
From 748087483cba9bcd28029e0083147674a9ca85b2 Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Fri, 11 Oct 2024 08:05:28 -0500
Subject: [PATCH 09/27] Use number keys on keyboard to move ptz camera to
presets (#14278)
* Use number keys on keyboard to move ptz camera to presets
* clean up
---
web/src/views/live/LiveCameraView.tsx | 32 +++++++++++++++++++++++++--
1 file changed, 30 insertions(+), 2 deletions(-)
diff --git a/web/src/views/live/LiveCameraView.tsx b/web/src/views/live/LiveCameraView.tsx
index 8a8146e14..2af08e497 100644
--- a/web/src/views/live/LiveCameraView.tsx
+++ b/web/src/views/live/LiveCameraView.tsx
@@ -531,9 +531,37 @@ function PtzControlPanel({
);
useKeyboardListener(
- ["ArrowLeft", "ArrowRight", "ArrowUp", "ArrowDown", "+", "-"],
+ [
+ "ArrowLeft",
+ "ArrowRight",
+ "ArrowUp",
+ "ArrowDown",
+ "+",
+ "-",
+ "1",
+ "2",
+ "3",
+ "4",
+ "5",
+ "6",
+ "7",
+ "8",
+ "9",
+ ],
(key, modifiers) => {
- if (modifiers.repeat) {
+ if (modifiers.repeat || !key) {
+ return;
+ }
+
+ if (["1", "2", "3", "4", "5", "6", "7", "8", "9"].includes(key)) {
+ const presetNumber = parseInt(key);
+ if (
+ ptz &&
+ (ptz.presets?.length ?? 0) > 0 &&
+ presetNumber <= ptz.presets.length
+ ) {
+ sendPtz(`preset_${ptz.presets[presetNumber - 1]}`);
+ }
return;
}
From 6df541e1fd989d74ef70b7ffb776ba7822b833e1 Mon Sep 17 00:00:00 2001
From: Nicolas Mowen
Date: Fri, 11 Oct 2024 10:47:23 -0600
Subject: [PATCH 10/27] Openvino models (#14283)
* Enable model conversion cache for openvino
* Use openvino directly for onnx embeddings if available
* Don't fail if zmq is busy
---
frigate/comms/dispatcher.py | 9 ++--
frigate/comms/inter_process.py | 7 +++-
frigate/detectors/plugins/openvino.py | 3 ++
frigate/embeddings/functions/onnx.py | 35 ++++++----------
frigate/util/model.py | 60 +++++++++++++++++++++++++++
5 files changed, 85 insertions(+), 29 deletions(-)
diff --git a/frigate/comms/dispatcher.py b/frigate/comms/dispatcher.py
index 8b310a4bb..4a3862eaf 100644
--- a/frigate/comms/dispatcher.py
+++ b/frigate/comms/dispatcher.py
@@ -142,10 +142,11 @@ class Dispatcher:
)
def handle_update_model_state():
- model = payload["model"]
- state = payload["state"]
- self.model_state[model] = ModelStatusTypesEnum[state]
- self.publish("model_state", json.dumps(self.model_state))
+ if payload:
+ model = payload["model"]
+ state = payload["state"]
+ self.model_state[model] = ModelStatusTypesEnum[state]
+ self.publish("model_state", json.dumps(self.model_state))
def handle_model_state():
self.publish("model_state", json.dumps(self.model_state.copy()))
diff --git a/frigate/comms/inter_process.py b/frigate/comms/inter_process.py
index 32cec49e4..850e2435c 100644
--- a/frigate/comms/inter_process.py
+++ b/frigate/comms/inter_process.py
@@ -65,8 +65,11 @@ class InterProcessRequestor:
def send_data(self, topic: str, data: any) -> any:
"""Sends data and then waits for reply."""
- self.socket.send_json((topic, data))
- return self.socket.recv_json()
+ try:
+ self.socket.send_json((topic, data))
+ return self.socket.recv_json()
+ except zmq.ZMQError:
+ return ""
def stop(self) -> None:
self.socket.close()
diff --git a/frigate/detectors/plugins/openvino.py b/frigate/detectors/plugins/openvino.py
index 5dc998487..51e48530b 100644
--- a/frigate/detectors/plugins/openvino.py
+++ b/frigate/detectors/plugins/openvino.py
@@ -3,6 +3,7 @@ import os
import numpy as np
import openvino as ov
+import openvino.properties as props
from pydantic import Field
from typing_extensions import Literal
@@ -34,6 +35,8 @@ class OvDetector(DetectionApi):
logger.error(f"OpenVino model file {detector_config.model.path} not found.")
raise FileNotFoundError
+ os.makedirs("/config/model_cache/openvino", exist_ok=True)
+ self.ov_core.set_property({props.cache_dir: "/config/model_cache/openvino"})
self.interpreter = self.ov_core.compile_model(
model=detector_config.model.path, device_name=detector_config.device
)
diff --git a/frigate/embeddings/functions/onnx.py b/frigate/embeddings/functions/onnx.py
index ae9fe33bc..1e50e07b1 100644
--- a/frigate/embeddings/functions/onnx.py
+++ b/frigate/embeddings/functions/onnx.py
@@ -5,7 +5,6 @@ from io import BytesIO
from typing import Callable, Dict, List, Optional, Union
import numpy as np
-import onnxruntime as ort
import requests
from PIL import Image
@@ -19,7 +18,7 @@ from frigate.comms.inter_process import InterProcessRequestor
from frigate.const import MODEL_CACHE_DIR, UPDATE_MODEL_STATE
from frigate.types import ModelStatusTypesEnum
from frigate.util.downloader import ModelDownloader
-from frigate.util.model import get_ort_providers
+from frigate.util.model import ONNXModelRunner
warnings.filterwarnings(
"ignore",
@@ -54,16 +53,12 @@ class GenericONNXEmbedding:
self.download_urls = download_urls
self.embedding_function = embedding_function
self.model_type = model_type # 'text' or 'vision'
- self.providers, self.provider_options = get_ort_providers(
- force_cpu=device == "CPU",
- requires_fp16=model_size == "large" or self.model_type == "text",
- openvino_device=device,
- )
-
+ self.model_size = model_size
+ self.device = device
self.download_path = os.path.join(MODEL_CACHE_DIR, self.model_name)
self.tokenizer = None
self.feature_extractor = None
- self.session = None
+ self.runner = None
files_names = list(self.download_urls.keys()) + (
[self.tokenizer_file] if self.tokenizer_file else []
)
@@ -124,15 +119,17 @@ class GenericONNXEmbedding:
)
def _load_model_and_tokenizer(self):
- if self.session is None:
+ if self.runner is None:
if self.downloader:
self.downloader.wait_for_download()
if self.model_type == "text":
self.tokenizer = self._load_tokenizer()
else:
self.feature_extractor = self._load_feature_extractor()
- self.session = self._load_model(
- os.path.join(self.download_path, self.model_file)
+ self.runner = ONNXModelRunner(
+ os.path.join(self.download_path, self.model_file),
+ self.device,
+ self.model_size,
)
def _load_tokenizer(self):
@@ -149,14 +146,6 @@ class GenericONNXEmbedding:
f"{MODEL_CACHE_DIR}/{self.model_name}",
)
- def _load_model(self, path: str) -> Optional[ort.InferenceSession]:
- if os.path.exists(path):
- return ort.InferenceSession(
- path, providers=self.providers, provider_options=self.provider_options
- )
- else:
- return None
-
def _process_image(self, image):
if isinstance(image, str):
if image.startswith("http"):
@@ -170,7 +159,7 @@ class GenericONNXEmbedding:
) -> List[np.ndarray]:
self._load_model_and_tokenizer()
- if self.session is None or (
+ if self.runner is None or (
self.tokenizer is None and self.feature_extractor is None
):
logger.error(
@@ -188,14 +177,14 @@ class GenericONNXEmbedding:
images=processed_images, return_tensors="np"
)
- input_names = [input.name for input in self.session.get_inputs()]
+ input_names = self.runner.get_input_names()
onnx_inputs = {
name: processed_inputs[name]
for name in input_names
if name in processed_inputs
}
- outputs = self.session.run(None, onnx_inputs)
+ outputs = self.runner.run(onnx_inputs)
embeddings = self.embedding_function(outputs)
return [embedding for embedding in embeddings]
diff --git a/frigate/util/model.py b/frigate/util/model.py
index fabade387..951e61370 100644
--- a/frigate/util/model.py
+++ b/frigate/util/model.py
@@ -1,9 +1,16 @@
"""Model Utils"""
import os
+from typing import Any
import onnxruntime as ort
+try:
+ import openvino as ov
+except ImportError:
+ # openvino is not included
+ pass
+
def get_ort_providers(
force_cpu: bool = False, openvino_device: str = "AUTO", requires_fp16: bool = False
@@ -42,3 +49,56 @@ def get_ort_providers(
options.append({})
return (providers, options)
+
+
+class ONNXModelRunner:
+ """Run onnx models optimally based on available hardware."""
+
+ def __init__(self, model_path: str, device: str, requires_fp16: bool = False):
+ self.model_path = model_path
+ self.ort: ort.InferenceSession = None
+ self.ov: ov.Core = None
+ providers, options = get_ort_providers(device == "CPU", device, requires_fp16)
+
+ if "OpenVINOExecutionProvider" in providers:
+ # use OpenVINO directly
+ self.type = "ov"
+ self.ov = ov.Core()
+ self.ov.set_property(
+ {ov.properties.cache_dir: "/config/model_cache/openvino"}
+ )
+ self.interpreter = self.ov.compile_model(
+ model=model_path, device_name=device
+ )
+ else:
+ # Use ONNXRuntime
+ self.type = "ort"
+ self.ort = ort.InferenceSession(
+ model_path, providers=providers, provider_options=options
+ )
+
+ def get_input_names(self) -> list[str]:
+ if self.type == "ov":
+ input_names = []
+
+ for input in self.interpreter.inputs:
+ input_names.extend(input.names)
+
+ return input_names
+ elif self.type == "ort":
+ return [input.name for input in self.ort.get_inputs()]
+
+ def run(self, input: dict[str, Any]) -> Any:
+ if self.type == "ov":
+ infer_request = self.interpreter.create_infer_request()
+ input_tensor = list(input.values())
+
+ if len(input_tensor) == 1:
+ input_tensor = ov.Tensor(array=input_tensor[0])
+ else:
+ input_tensor = ov.Tensor(array=input_tensor)
+
+ infer_request.infer(input_tensor)
+ return [infer_request.get_output_tensor().data]
+ elif self.type == "ort":
+ return self.ort.run(None, input)
From d4b9b5a7dd439ed04fe2a6a60d0007fb0c0c3029 Mon Sep 17 00:00:00 2001
From: Nicolas Mowen
Date: Fri, 11 Oct 2024 12:03:47 -0600
Subject: [PATCH 11/27] Reduce onnx memory usage (#14285)
---
frigate/util/model.py | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
diff --git a/frigate/util/model.py b/frigate/util/model.py
index 951e61370..008f5169a 100644
--- a/frigate/util/model.py
+++ b/frigate/util/model.py
@@ -16,7 +16,14 @@ def get_ort_providers(
force_cpu: bool = False, openvino_device: str = "AUTO", requires_fp16: bool = False
) -> tuple[list[str], list[dict[str, any]]]:
if force_cpu:
- return (["CPUExecutionProvider"], [{}])
+ return (
+ ["CPUExecutionProvider"],
+ [
+ {
+ "arena_extend_strategy": "kSameAsRequested",
+ }
+ ],
+ )
providers = ort.get_available_providers()
options = []
@@ -28,6 +35,7 @@ def get_ort_providers(
if not requires_fp16 or os.environ.get("USE_FP_16", "True") != "False":
options.append(
{
+ "arena_extend_strategy": "kSameAsRequested",
"trt_fp16_enable": requires_fp16,
"trt_timing_cache_enable": True,
"trt_engine_cache_enable": True,
@@ -41,10 +49,17 @@ def get_ort_providers(
os.makedirs("/config/model_cache/openvino/ort", exist_ok=True)
options.append(
{
+ "arena_extend_strategy": "kSameAsRequested",
"cache_dir": "/config/model_cache/openvino/ort",
"device_type": openvino_device,
}
)
+ elif provider == "CPUExecutionProvider":
+ options.append(
+ {
+ "arena_extend_strategy": "kSameAsRequested",
+ }
+ )
else:
options.append({})
From 8a8a0c7decfb0aa70cdc87d8a7de3263d4bec265 Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Fri, 11 Oct 2024 13:11:11 -0500
Subject: [PATCH 12/27] Embeddings normalization fixes (#14284)
* Use cosine distance metric for vec tables
* Only apply normalization to multi modal searches
* Catch possible edge case in stddev calc
* Use sigmoid function for normalization for multi modal searches only
* Ensure we get model state on initial page load
* Only save stats for multi modal searches and only use cosine similarity for image -> image search
---
frigate/api/event.py | 33 ++++++++++++++---------------
frigate/db/sqlitevecq.py | 4 ++--
frigate/embeddings/util.py | 7 +++---
web/src/pages/Explore.tsx | 9 ++++++++
web/src/views/search/SearchView.tsx | 14 ++++++++----
5 files changed, 41 insertions(+), 26 deletions(-)
diff --git a/frigate/api/event.py b/frigate/api/event.py
index 3a8d003ad..c716bba13 100644
--- a/frigate/api/event.py
+++ b/frigate/api/event.py
@@ -473,12 +473,7 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends())
)
thumb_result = context.search_thumbnail(search_event)
- thumb_ids = dict(
- zip(
- [result[0] for result in thumb_result],
- context.thumb_stats.normalize([result[1] for result in thumb_result]),
- )
- )
+ thumb_ids = {result[0]: result[1] for result in thumb_result}
search_results = {
event_id: {"distance": distance, "source": "thumbnail"}
for event_id, distance in thumb_ids.items()
@@ -486,15 +481,18 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends())
else:
search_types = search_type.split(",")
+ # only save stats for multi-modal searches
+ save_stats = "thumbnail" in search_types and "description" in search_types
+
if "thumbnail" in search_types:
thumb_result = context.search_thumbnail(query)
+
+ thumb_distances = context.thumb_stats.normalize(
+ [result[1] for result in thumb_result], save_stats
+ )
+
thumb_ids = dict(
- zip(
- [result[0] for result in thumb_result],
- context.thumb_stats.normalize(
- [result[1] for result in thumb_result]
- ),
- )
+ zip([result[0] for result in thumb_result], thumb_distances)
)
search_results.update(
{
@@ -505,12 +503,13 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends())
if "description" in search_types:
desc_result = context.search_description(query)
- desc_ids = dict(
- zip(
- [result[0] for result in desc_result],
- context.desc_stats.normalize([result[1] for result in desc_result]),
- )
+
+ desc_distances = context.desc_stats.normalize(
+ [result[1] for result in desc_result], save_stats
)
+
+ desc_ids = dict(zip([result[0] for result in desc_result], desc_distances))
+
for event_id, distance in desc_ids.items():
if (
event_id not in search_results
diff --git a/frigate/db/sqlitevecq.py b/frigate/db/sqlitevecq.py
index 398adbd2d..ccb75ae54 100644
--- a/frigate/db/sqlitevecq.py
+++ b/frigate/db/sqlitevecq.py
@@ -42,12 +42,12 @@ class SqliteVecQueueDatabase(SqliteQueueDatabase):
self.execute_sql("""
CREATE VIRTUAL TABLE IF NOT EXISTS vec_thumbnails USING vec0(
id TEXT PRIMARY KEY,
- thumbnail_embedding FLOAT[768]
+ thumbnail_embedding FLOAT[768] distance_metric=cosine
);
""")
self.execute_sql("""
CREATE VIRTUAL TABLE IF NOT EXISTS vec_descriptions USING vec0(
id TEXT PRIMARY KEY,
- description_embedding FLOAT[768]
+ description_embedding FLOAT[768] distance_metric=cosine
);
""")
diff --git a/frigate/embeddings/util.py b/frigate/embeddings/util.py
index 0b2acd4d6..bc1a952ec 100644
--- a/frigate/embeddings/util.py
+++ b/frigate/embeddings/util.py
@@ -20,10 +20,11 @@ class ZScoreNormalization:
@property
def stddev(self):
- return math.sqrt(self.variance)
+ return math.sqrt(self.variance) if self.variance > 0 else 0.0
- def normalize(self, distances: list[float]):
- self._update(distances)
+ def normalize(self, distances: list[float], save_stats: bool):
+ if save_stats:
+ self._update(distances)
if self.stddev == 0:
return distances
return [
diff --git a/web/src/pages/Explore.tsx b/web/src/pages/Explore.tsx
index a3d7d3085..d3c5f7d9b 100644
--- a/web/src/pages/Explore.tsx
+++ b/web/src/pages/Explore.tsx
@@ -2,6 +2,7 @@ import {
useEmbeddingsReindexProgress,
useEventUpdate,
useModelState,
+ useWs,
} from "@/api/ws";
import ActivityIndicator from "@/components/indicators/activity-indicator";
import AnimatedCircularProgressBar from "@/components/ui/circular-progress-bar";
@@ -202,6 +203,14 @@ export default function Explore() {
// model states
+ const { send: sendCommand } = useWs("model_state", "modelState");
+
+ useEffect(() => {
+ sendCommand("modelState");
+ // only run on mount
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, []);
+
const { payload: textModelState } = useModelState(
"jinaai/jina-clip-v1-text_model_fp16.onnx",
);
diff --git a/web/src/views/search/SearchView.tsx b/web/src/views/search/SearchView.tsx
index 203942083..4c33f7dc8 100644
--- a/web/src/views/search/SearchView.tsx
+++ b/web/src/views/search/SearchView.tsx
@@ -187,13 +187,19 @@ export default function SearchView({
}
}, [searchResults, searchDetail]);
- // confidence score - probably needs tweaking
+ // confidence score
const zScoreToConfidence = (score: number) => {
- // Sigmoid function: 1 / (1 + e^x)
- const confidence = 1 / (1 + Math.exp(score));
+ // Normalizing is not needed for similarity searches
+ // Sigmoid function for normalized: 1 / (1 + e^x)
+ // Cosine for similarity
+ if (searchFilter) {
+ const notNormalized = searchFilter?.search_type?.includes("similarity");
- return Math.round(confidence * 100);
+ const confidence = notNormalized ? 1 - score : 1 / (1 + Math.exp(score));
+
+ return Math.round(confidence * 100);
+ }
};
const hasExistingSearch = useMemo(
From 6e332bbdf8600dc22a297d5098281d6e3d141f69 Mon Sep 17 00:00:00 2001
From: Nicolas Mowen
Date: Fri, 11 Oct 2024 16:08:14 -0600
Subject: [PATCH 13/27] Remove device config and use model size to configure
device used (#14290)
* Remove device config and use model size to configure device used
* Don't show Frigate+ submission when in progress
* Add docs link for bounding box colors
---
docs/docs/configuration/reference.md | 3 +--
docs/docs/configuration/semantic_search.md | 15 ++++++++-------
frigate/config/semantic_search.py | 1 -
frigate/embeddings/embeddings.py | 2 +-
.../overlay/detail/SearchDetailDialog.tsx | 2 +-
web/src/views/settings/ObjectSettingsView.tsx | 13 +++++++++++++
6 files changed, 24 insertions(+), 12 deletions(-)
diff --git a/docs/docs/configuration/reference.md b/docs/docs/configuration/reference.md
index 234478714..604791621 100644
--- a/docs/docs/configuration/reference.md
+++ b/docs/docs/configuration/reference.md
@@ -518,9 +518,8 @@ semantic_search:
enabled: False
# Optional: Re-index embeddings database from historical tracked objects (default: shown below)
reindex: False
- # Optional: Set device used to run embeddings, options are AUTO, CPU, GPU. (default: shown below)
- device: "AUTO"
# Optional: Set the model size used for embeddings. (default: shown below)
+ # NOTE: small model runs on CPU and large model runs on GPU
model_size: "small"
# Optional: Configuration for AI generated tracked object descriptions
diff --git a/docs/docs/configuration/semantic_search.md b/docs/docs/configuration/semantic_search.md
index 87ccbf802..a569e8f1a 100644
--- a/docs/docs/configuration/semantic_search.md
+++ b/docs/docs/configuration/semantic_search.md
@@ -29,25 +29,26 @@ If you are enabling the Search feature for the first time, be advised that Friga
### Jina AI CLIP
-:::tip
-
-The CLIP models are downloaded in ONNX format, which means they will be accelerated using GPU hardware when available. This depends on the Docker build that is used. See [the object detector docs](../configuration/object_detectors.md) for more information.
-
-:::
-
The vision model is able to embed both images and text into the same vector space, which allows `image -> image` and `text -> image` similarity searches. Frigate uses this model on tracked objects to encode the thumbnail image and store it in the database. When searching for tracked objects via text in the search box, Frigate will perform a `text -> image` similarity search against this embedding. When clicking "Find Similar" in the tracked object detail pane, Frigate will perform an `image -> image` similarity search to retrieve the closest matching thumbnails.
The text model is used to embed tracked object descriptions and perform searches against them. Descriptions can be created, viewed, and modified on the Search page when clicking on the gray tracked object chip at the top left of each review item. See [the Generative AI docs](/configuration/genai.md) for more information on how to automatically generate tracked object descriptions.
Differently weighted CLIP models are available and can be selected by setting the `model_size` config option:
+:::tip
+
+The CLIP models are downloaded in ONNX format, which means they will be accelerated using GPU hardware when available. This depends on the Docker build that is used. See [the object detector docs](../configuration/object_detectors.md) for more information.
+
+:::
+
```yaml
semantic_search:
enabled: True
model_size: small
```
-Using `large` as the model size setting employs the full Jina model appropriate for high performance systems running a GPU. The `small` size uses a quantized version of the model that uses much less RAM and runs faster on CPU with a very negligible difference in embedding quality. Most users will not need to change this setting from the default of `small`.
+- Configuring the `large` model employs the full Jina model and will automatically run on the GPU if applicable.
+- Configuring the `small` model employs a quantized version of the model that uses much less RAM and runs faster on CPU with a very negligible difference in embedding quality.
## Usage
diff --git a/frigate/config/semantic_search.py b/frigate/config/semantic_search.py
index fdaf0fff4..2891050a1 100644
--- a/frigate/config/semantic_search.py
+++ b/frigate/config/semantic_search.py
@@ -12,7 +12,6 @@ class SemanticSearchConfig(FrigateBaseModel):
reindex: Optional[bool] = Field(
default=False, title="Reindex all detections on startup."
)
- device: str = Field(default="AUTO", title="Device Type")
model_size: str = Field(
default="small", title="The size of the embeddings model used."
)
diff --git a/frigate/embeddings/embeddings.py b/frigate/embeddings/embeddings.py
index 4883e0914..9c8d4abbd 100644
--- a/frigate/embeddings/embeddings.py
+++ b/frigate/embeddings/embeddings.py
@@ -127,7 +127,7 @@ class Embeddings:
model_size=config.model_size,
model_type="vision",
requestor=self.requestor,
- device=self.config.device,
+ device="GPU" if config.model_size == "large" else "CPU",
)
def upsert_thumbnail(self, event_id: str, thumbnail: bytes):
diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx
index 843f2de59..4063f3a59 100644
--- a/web/src/components/overlay/detail/SearchDetailDialog.tsx
+++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx
@@ -554,7 +554,7 @@ function ObjectSnapshotTab({
- {state == "reviewing" && (
+ {state == "reviewing" && search.end_time && (
<>
+
+
From de86c3768778d824116366457db8fb1ae53ceceb Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Sat, 12 Oct 2024 07:11:22 -0500
Subject: [PATCH 14/27] Prevent single letter words from matching filter
suggestions (#14297)
---
web/src/components/input/InputWithTags.tsx | 21 +++++++++++++++++++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/web/src/components/input/InputWithTags.tsx b/web/src/components/input/InputWithTags.tsx
index 6c06e67e7..5d0786346 100644
--- a/web/src/components/input/InputWithTags.tsx
+++ b/web/src/components/input/InputWithTags.tsx
@@ -7,6 +7,7 @@ import {
LuChevronUp,
LuTrash2,
LuStar,
+ LuSearch,
} from "react-icons/lu";
import {
FilterType,
@@ -161,8 +162,12 @@ export default function InputWithTags({
.map((word) => word.trim())
.lastIndexOf(words.filter((word) => word.trim() !== "").pop() || "");
const currentWord = words[lastNonEmptyWordIndex];
+ if (words.at(-1) === "") {
+ return current_suggestions;
+ }
+
return current_suggestions.filter((suggestion) =>
- suggestion.toLowerCase().includes(currentWord.toLowerCase()),
+ suggestion.toLowerCase().startsWith(currentWord),
);
},
[inputValue, suggestions, currentFilterType],
@@ -636,7 +641,19 @@ export default function InputWithTags({
inputFocused ? "visible" : "hidden",
)}
>
- {(Object.keys(filters).length > 0 || isSimilaritySearch) && (
+ {!currentFilterType && inputValue && (
+
+ handleSearch(inputValue)}
+ >
+
+
+ )}
+ {(Object.keys(filters).filter((key) => key !== "query").length > 0 ||
+ isSimilaritySearch) && (
{isSimilaritySearch && (
From 1e1610671e44448ccf12e991192cdabf842dfe86 Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Sat, 12 Oct 2024 07:12:02 -0500
Subject: [PATCH 15/27] Add info icons for popovers in debug view (#14296)
---
web/src/views/settings/ObjectSettingsView.tsx | 96 +++++++++++++++----
1 file changed, 75 insertions(+), 21 deletions(-)
diff --git a/web/src/views/settings/ObjectSettingsView.tsx b/web/src/views/settings/ObjectSettingsView.tsx
index 8b6c99f4f..927d08ee0 100644
--- a/web/src/views/settings/ObjectSettingsView.tsx
+++ b/web/src/views/settings/ObjectSettingsView.tsx
@@ -11,13 +11,17 @@ import { usePersistence } from "@/hooks/use-persistence";
import { Skeleton } from "@/components/ui/skeleton";
import { useCameraActivity } from "@/hooks/use-camera-activity";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
+import {
+ Popover,
+ PopoverContent,
+ PopoverTrigger,
+} from "@/components/ui/popover";
import { ObjectType } from "@/types/ws";
import useDeepMemo from "@/hooks/use-deep-memo";
import { Card } from "@/components/ui/card";
import { getIconForLabel } from "@/utils/iconUtil";
import { capitalizeFirstLetter } from "@/utils/stringUtil";
-import { Link } from "react-router-dom";
-import { LuExternalLink } from "react-icons/lu";
+import { LuInfo } from "react-icons/lu";
type ObjectSettingsViewProps = {
selectedCamera?: string;
@@ -37,6 +41,30 @@ export default function ObjectSettingsView({
param: "bbox",
title: "Bounding boxes",
description: "Show bounding boxes around tracked objects",
+ info: (
+ <>
+
+ Object Bounding Box Colors
+
+
+
+ At startup, different colors will be assigned to each object label
+
+
+ A dark blue thin line indicates that object is not detected at
+ this current point in time
+
+
+ A gray thin line indicates that object is detected as being
+ stationary
+
+
+ A thick line indicates that object is the subject of autotracking
+ (when enabled)
+
+
+
+ ),
},
{
param: "timestamp",
@@ -57,12 +85,34 @@ export default function ObjectSettingsView({
param: "motion",
title: "Motion boxes",
description: "Show boxes around areas where motion is detected",
+ info: (
+ <>
+
+ Motion Boxes
+
+
+ Red boxes will be overlaid on areas of the frame where motion is
+ currently being detected
+
+
+ ),
},
{
param: "regions",
title: "Regions",
description:
"Show a box of the region of interest sent to the object detector",
+ info: (
+ <>
+
+ Region Boxes
+
+
+ Bright green boxes will be overlaid on areas of interest in the
+ frame that are being sent to the object detector.
+
+
+ ),
},
];
@@ -136,17 +186,6 @@ export default function ObjectSettingsView({
statistics. The object list shows a time-delayed summary of detected
objects.
-
-
@@ -158,19 +197,34 @@ export default function ObjectSettingsView({
- {DEBUG_OPTIONS.map(({ param, title, description }) => (
+ {DEBUG_OPTIONS.map(({ param, title, description, info }) => (
-
- {title}
-
-
+
+
+ {title}
+
+ {info && (
+
+
+
+ Info
+
+
+
+ {info}
+
+
+ )}
+
+
{description}
From 48c60621b6fcca746a11366d481ee6b1c9c2522d Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Sat, 12 Oct 2024 07:19:24 -0500
Subject: [PATCH 16/27] Fix substitution on genai prompts (#14298)
---
frigate/embeddings/maintainer.py | 2 +-
frigate/genai/__init__.py | 7 ++++---
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py
index 238efcfdf..c7060b9a6 100644
--- a/frigate/embeddings/maintainer.py
+++ b/frigate/embeddings/maintainer.py
@@ -259,7 +259,7 @@ class EmbeddingMaintainer(threading.Thread):
camera_config = self.config.cameras[event.camera]
description = self.genai_client.generate_description(
- camera_config, thumbnails, event.label
+ camera_config, thumbnails, event
)
if not description:
diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py
index caf13082d..dccb74c1d 100644
--- a/frigate/genai/__init__.py
+++ b/frigate/genai/__init__.py
@@ -5,6 +5,7 @@ import os
from typing import Optional
from frigate.config import CameraConfig, GenAIConfig, GenAIProviderEnum
+from frigate.models import Event
PROVIDERS = {}
@@ -31,12 +32,12 @@ class GenAIClient:
self,
camera_config: CameraConfig,
thumbnails: list[bytes],
- label: str,
+ event: Event,
) -> Optional[str]:
"""Generate a description for the frame."""
prompt = camera_config.genai.object_prompts.get(
- label, camera_config.genai.prompt
- ).format(label=label)
+ event.label, camera_config.genai.prompt
+ ).format(**event)
return self._send(prompt, thumbnails)
def _init_provider(self):
From 40bb4765d4682d07fe2816aabf2b50464f333d34 Mon Sep 17 00:00:00 2001
From: Nicolas Mowen
Date: Sat, 12 Oct 2024 07:37:22 -0600
Subject: [PATCH 17/27] Add support for more icons (#14299)
---
web/src/utils/iconUtil.tsx | 49 +++++++++++++++++++++++++++++++++++---
1 file changed, 46 insertions(+), 3 deletions(-)
diff --git a/web/src/utils/iconUtil.tsx b/web/src/utils/iconUtil.tsx
index f5ab70c9e..e3c1f5508 100644
--- a/web/src/utils/iconUtil.tsx
+++ b/web/src/utils/iconUtil.tsx
@@ -7,16 +7,31 @@ import {
FaCarSide,
FaCat,
FaCheckCircle,
+ FaDhl,
FaDog,
FaFedex,
FaFire,
FaFootballBall,
+ FaHockeyPuck,
+ FaHorse,
FaMotorcycle,
FaMouse,
+ FaRegTrashAlt,
+ FaUmbrella,
FaUps,
FaUsps,
} from "react-icons/fa";
-import { GiDeer, GiHummingbird, GiPolarBear, GiSailboat } from "react-icons/gi";
+import {
+ GiDeer,
+ GiFox,
+ GiGoat,
+ GiHummingbird,
+ GiPolarBear,
+ GiPostStamp,
+ GiRabbit,
+ GiRaccoonHead,
+ GiSailboat,
+} from "react-icons/gi";
import { LuBox, LuLassoSelect } from "react-icons/lu";
import * as LuIcons from "react-icons/lu";
import { MdRecordVoiceOver } from "react-icons/md";
@@ -53,8 +68,12 @@ export function getIconForLabel(label: string, className?: string) {
case "bark":
case "dog":
return
Date: Sat, 12 Oct 2024 12:36:10 -0600
Subject: [PATCH 19/27] Fixes for model downloading (#14305)
* Use different requestor for downloaders
* Handle case where lock is left over from failed partial download
* close requestor
* Formatting
---
frigate/embeddings/functions/onnx.py | 1 -
frigate/util/downloader.py | 11 +++++++++--
2 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/frigate/embeddings/functions/onnx.py b/frigate/embeddings/functions/onnx.py
index 1e50e07b1..e836ba960 100644
--- a/frigate/embeddings/functions/onnx.py
+++ b/frigate/embeddings/functions/onnx.py
@@ -71,7 +71,6 @@ class GenericONNXEmbedding:
model_name=self.model_name,
download_path=self.download_path,
file_names=files_names,
- requestor=self.requestor,
download_func=self._download_model,
)
self.downloader.ensure_model_files()
diff --git a/frigate/util/downloader.py b/frigate/util/downloader.py
index ce5030566..6685b0bb8 100644
--- a/frigate/util/downloader.py
+++ b/frigate/util/downloader.py
@@ -19,6 +19,13 @@ class FileLock:
self.path = path
self.lock_file = f"{path}.lock"
+ # we have not acquired the lock yet so it should not exist
+ if os.path.exists(self.lock_file):
+ try:
+ os.remove(self.lock_file)
+ except Exception:
+ pass
+
def acquire(self):
parent_dir = os.path.dirname(self.lock_file)
os.makedirs(parent_dir, exist_ok=True)
@@ -44,7 +51,6 @@ class ModelDownloader:
download_path: str,
file_names: List[str],
download_func: Callable[[str], None],
- requestor: InterProcessRequestor,
silent: bool = False,
):
self.model_name = model_name
@@ -52,7 +58,7 @@ class ModelDownloader:
self.file_names = file_names
self.download_func = download_func
self.silent = silent
- self.requestor = requestor
+ self.requestor = InterProcessRequestor()
self.download_thread = None
self.download_complete = threading.Event()
@@ -91,6 +97,7 @@ class ModelDownloader:
},
)
+ self.requestor.stop()
self.download_complete.set()
@staticmethod
From 0fc799978004950711b9fbff0b2be64a29482ff3 Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Sat, 12 Oct 2024 14:44:01 -0500
Subject: [PATCH 20/27] Improve reindex completion flag (#14308)
---
frigate/embeddings/embeddings.py | 5 ++++-
web/src/pages/Explore.tsx | 22 ++++++++++++++--------
web/src/types/ws.ts | 1 +
3 files changed, 19 insertions(+), 9 deletions(-)
diff --git a/frigate/embeddings/embeddings.py b/frigate/embeddings/embeddings.py
index 9c8d4abbd..b5b166b00 100644
--- a/frigate/embeddings/embeddings.py
+++ b/frigate/embeddings/embeddings.py
@@ -176,6 +176,7 @@ class Embeddings:
"processed_objects": 0,
"total_objects": 0,
"time_remaining": 0,
+ "status": "indexing",
}
self.requestor.send_data(UPDATE_EMBEDDINGS_REINDEX_PROGRESS, totals)
@@ -255,6 +256,8 @@ class Embeddings:
"Embedded %d thumbnails and %d descriptions in %s seconds",
totals["thumbnails"],
totals["descriptions"],
- time.time() - st,
+ round(time.time() - st, 1),
)
+ totals["status"] = "completed"
+
self.requestor.send_data(UPDATE_EMBEDDINGS_REINDEX_PROGRESS, totals)
diff --git a/web/src/pages/Explore.tsx b/web/src/pages/Explore.tsx
index e81889fb0..4aebaefd1 100644
--- a/web/src/pages/Explore.tsx
+++ b/web/src/pages/Explore.tsx
@@ -195,13 +195,18 @@ export default function Explore() {
const { payload: reindexProgress } = useEmbeddingsReindexProgress();
- const embeddingsReindexing = useMemo(
- () =>
- reindexProgress
- ? reindexProgress.total_objects - reindexProgress.processed_objects > 0
- : undefined,
- [reindexProgress],
- );
+ const embeddingsReindexing = useMemo(() => {
+ if (reindexProgress) {
+ switch (reindexProgress.status) {
+ case "indexing":
+ return true;
+ case "completed":
+ return false;
+ default:
+ return undefined;
+ }
+ }
+ }, [reindexProgress]);
// model states
@@ -320,7 +325,8 @@ export default function Explore() {
Tracked objects processed:
- {reindexProgress.processed_objects}
+ {reindexProgress.processed_objects} /{" "}
+ {reindexProgress.total_objects}
diff --git a/web/src/types/ws.ts b/web/src/types/ws.ts
index 238ef4a8a..397b213f6 100644
--- a/web/src/types/ws.ts
+++ b/web/src/types/ws.ts
@@ -68,6 +68,7 @@ export type EmbeddingsReindexProgressType = {
processed_objects: number;
total_objects: number;
time_remaining: number;
+ status: string;
};
export type ToggleableSetting = "ON" | "OFF";
From e8b2fde753f50421dde2ef725b51ac6f5999ee5f Mon Sep 17 00:00:00 2001
From: Nicolas Mowen
Date: Sun, 13 Oct 2024 11:33:27 -0600
Subject: [PATCH 21/27] Support batch embeddings when reindexing (#14320)
* Refactor onnx embeddings to handle multiple inputs by default
* Process items in batches when reindexing
---
frigate/embeddings/embeddings.py | 97 ++++++++++++++++++----------
frigate/embeddings/functions/onnx.py | 35 +++++-----
2 files changed, 82 insertions(+), 50 deletions(-)
diff --git a/frigate/embeddings/embeddings.py b/frigate/embeddings/embeddings.py
index b5b166b00..8d12feb32 100644
--- a/frigate/embeddings/embeddings.py
+++ b/frigate/embeddings/embeddings.py
@@ -6,6 +6,7 @@ import logging
import os
import time
+from numpy import ndarray
from PIL import Image
from playhouse.shortcuts import model_to_dict
@@ -88,12 +89,6 @@ class Embeddings:
},
)
- def jina_text_embedding_function(outputs):
- return outputs[0]
-
- def jina_vision_embedding_function(outputs):
- return outputs[0]
-
self.text_embedding = GenericONNXEmbedding(
model_name="jinaai/jina-clip-v1",
model_file="text_model_fp16.onnx",
@@ -101,7 +96,6 @@ class Embeddings:
download_urls={
"text_model_fp16.onnx": "https://huggingface.co/jinaai/jina-clip-v1/resolve/main/onnx/text_model_fp16.onnx",
},
- embedding_function=jina_text_embedding_function,
model_size=config.model_size,
model_type="text",
requestor=self.requestor,
@@ -123,14 +117,13 @@ class Embeddings:
model_name="jinaai/jina-clip-v1",
model_file=model_file,
download_urls=download_urls,
- embedding_function=jina_vision_embedding_function,
model_size=config.model_size,
model_type="vision",
requestor=self.requestor,
device="GPU" if config.model_size == "large" else "CPU",
)
- def upsert_thumbnail(self, event_id: str, thumbnail: bytes):
+ def upsert_thumbnail(self, event_id: str, thumbnail: bytes) -> ndarray:
# Convert thumbnail bytes to PIL Image
image = Image.open(io.BytesIO(thumbnail)).convert("RGB")
embedding = self.vision_embedding([image])[0]
@@ -145,7 +138,25 @@ class Embeddings:
return embedding
- def upsert_description(self, event_id: str, description: str):
+ def batch_upsert_thumbnail(self, event_thumbs: dict[str, bytes]) -> list[ndarray]:
+ images = [
+ Image.open(io.BytesIO(thumb)).convert("RGB")
+ for thumb in event_thumbs.values()
+ ]
+ ids = list(event_thumbs.keys())
+ embeddings = self.vision_embedding(images)
+ items = [(ids[i], serialize(embeddings[i])) for i in range(len(ids))]
+
+ self.db.execute_sql(
+ """
+ INSERT OR REPLACE INTO vec_thumbnails(id, thumbnail_embedding)
+ VALUES {}
+ """.format(", ".join(["(?, ?)"] * len(items))),
+ items,
+ )
+ return embeddings
+
+ def upsert_description(self, event_id: str, description: str) -> ndarray:
embedding = self.text_embedding([description])[0]
self.db.execute_sql(
"""
@@ -157,6 +168,21 @@ class Embeddings:
return embedding
+ def batch_upsert_description(self, event_descriptions: dict[str, str]) -> ndarray:
+ embeddings = self.text_embedding(list(event_descriptions.values()))
+ ids = list(event_descriptions.keys())
+ items = [(ids[i], serialize(embeddings[i])) for i in range(len(ids))]
+
+ self.db.execute_sql(
+ """
+ INSERT OR REPLACE INTO vec_descriptions(id, description_embedding)
+ VALUES {}
+ """.format(", ".join(["(?, ?)"] * len(items))),
+ items,
+ )
+
+ return embeddings
+
def reindex(self) -> None:
logger.info("Indexing tracked object embeddings...")
@@ -192,9 +218,8 @@ class Embeddings:
)
totals["total_objects"] = total_events
- batch_size = 100
+ batch_size = 32
current_page = 1
- processed_events = 0
events = (
Event.select()
@@ -208,37 +233,43 @@ class Embeddings:
while len(events) > 0:
event: Event
+ batch_thumbs = {}
+ batch_descs = {}
for event in events:
- thumbnail = base64.b64decode(event.thumbnail)
- self.upsert_thumbnail(event.id, thumbnail)
+ batch_thumbs[event.id] = base64.b64decode(event.thumbnail)
totals["thumbnails"] += 1
if description := event.data.get("description", "").strip():
+ batch_descs[event.id] = description
totals["descriptions"] += 1
- self.upsert_description(event.id, description)
totals["processed_objects"] += 1
- # report progress every 10 events so we don't spam the logs
- if (totals["processed_objects"] % 10) == 0:
- progress = (processed_events / total_events) * 100
- logger.debug(
- "Processed %d/%d events (%.2f%% complete) | Thumbnails: %d, Descriptions: %d",
- processed_events,
- total_events,
- progress,
- totals["thumbnails"],
- totals["descriptions"],
- )
+ # run batch embedding
+ self.batch_upsert_thumbnail(batch_thumbs)
- # Calculate time remaining
- elapsed_time = time.time() - st
- avg_time_per_event = elapsed_time / totals["processed_objects"]
- remaining_events = total_events - totals["processed_objects"]
- time_remaining = avg_time_per_event * remaining_events
- totals["time_remaining"] = int(time_remaining)
+ if batch_descs:
+ self.batch_upsert_description(batch_descs)
- self.requestor.send_data(UPDATE_EMBEDDINGS_REINDEX_PROGRESS, totals)
+ # report progress every batch so we don't spam the logs
+ progress = (totals["processed_objects"] / total_events) * 100
+ logger.debug(
+ "Processed %d/%d events (%.2f%% complete) | Thumbnails: %d, Descriptions: %d",
+ totals["processed_objects"],
+ total_events,
+ progress,
+ totals["thumbnails"],
+ totals["descriptions"],
+ )
+
+ # Calculate time remaining
+ elapsed_time = time.time() - st
+ avg_time_per_event = elapsed_time / totals["processed_objects"]
+ remaining_events = total_events - totals["processed_objects"]
+ time_remaining = avg_time_per_event * remaining_events
+ totals["time_remaining"] = int(time_remaining)
+
+ self.requestor.send_data(UPDATE_EMBEDDINGS_REINDEX_PROGRESS, totals)
# Move to the next page
current_page += 1
diff --git a/frigate/embeddings/functions/onnx.py b/frigate/embeddings/functions/onnx.py
index e836ba960..765a7e88c 100644
--- a/frigate/embeddings/functions/onnx.py
+++ b/frigate/embeddings/functions/onnx.py
@@ -2,7 +2,7 @@ import logging
import os
import warnings
from io import BytesIO
-from typing import Callable, Dict, List, Optional, Union
+from typing import Dict, List, Optional, Union
import numpy as np
import requests
@@ -39,7 +39,6 @@ class GenericONNXEmbedding:
model_name: str,
model_file: str,
download_urls: Dict[str, str],
- embedding_function: Callable[[List[np.ndarray]], np.ndarray],
model_size: str,
model_type: str,
requestor: InterProcessRequestor,
@@ -51,7 +50,6 @@ class GenericONNXEmbedding:
self.tokenizer_file = tokenizer_file
self.requestor = requestor
self.download_urls = download_urls
- self.embedding_function = embedding_function
self.model_type = model_type # 'text' or 'vision'
self.model_size = model_size
self.device = device
@@ -157,7 +155,6 @@ class GenericONNXEmbedding:
self, inputs: Union[List[str], List[Image.Image], List[str]]
) -> List[np.ndarray]:
self._load_model_and_tokenizer()
-
if self.runner is None or (
self.tokenizer is None and self.feature_extractor is None
):
@@ -167,23 +164,27 @@ class GenericONNXEmbedding:
return []
if self.model_type == "text":
- processed_inputs = self.tokenizer(
- inputs, padding=True, truncation=True, return_tensors="np"
- )
+ processed_inputs = [
+ self.tokenizer(text, padding=True, truncation=True, return_tensors="np")
+ for text in inputs
+ ]
else:
processed_images = [self._process_image(img) for img in inputs]
- processed_inputs = self.feature_extractor(
- images=processed_images, return_tensors="np"
- )
+ processed_inputs = [
+ self.feature_extractor(images=image, return_tensors="np")
+ for image in processed_images
+ ]
input_names = self.runner.get_input_names()
- onnx_inputs = {
- name: processed_inputs[name]
- for name in input_names
- if name in processed_inputs
- }
+ onnx_inputs = {name: [] for name in input_names}
+ input: dict[str, any]
+ for input in processed_inputs:
+ for key, value in input.items():
+ if key in input_names:
+ onnx_inputs[key].append(value[0])
- outputs = self.runner.run(onnx_inputs)
- embeddings = self.embedding_function(outputs)
+ for key in onnx_inputs.keys():
+ onnx_inputs[key] = np.array(onnx_inputs[key])
+ embeddings = self.runner.run(onnx_inputs)[0]
return [embedding for embedding in embeddings]
From 92ac025e43f230d3cd16d2dba3d67c8de2b2f8b7 Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Sun, 13 Oct 2024 12:34:39 -0500
Subject: [PATCH 22/27] Don't show submit to frigate plus card if plus is
disabled (#14319)
---
.../overlay/detail/SearchDetailDialog.tsx | 102 +++++++++---------
1 file changed, 52 insertions(+), 50 deletions(-)
diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx
index 4063f3a59..a04d5b8c1 100644
--- a/web/src/components/overlay/detail/SearchDetailDialog.tsx
+++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx
@@ -536,57 +536,59 @@ function ObjectSnapshotTab({
/>
)}
-
-
-
-
- Submit To Frigate+
-
-
- Objects in locations you want to avoid are not false
- positives. Submitting them as false positives will confuse
- the model.
-
-
- {state == "reviewing" && search.end_time && (
- <>
-
{
- setState("uploading");
- onSubmitToPlus(false);
- }}
- >
- This is a {search?.label}
-
-
{
- setState("uploading");
- onSubmitToPlus(true);
- }}
- >
- This is not a {search?.label}
-
-
- )}
- {state == "uploading" &&
}
- {state == "submitted" && (
-
-
- Submitted
+ {search.plus_id !== "not_enabled" && (
+
+
+
+
+ Submit To Frigate+
- )}
-
-
+
+ Objects in locations you want to avoid are not false
+ positives. Submitting them as false positives will confuse
+ the model.
+
+
+
+
+ {state == "reviewing" && search.end_time && (
+ <>
+
{
+ setState("uploading");
+ onSubmitToPlus(false);
+ }}
+ >
+ This is a {search?.label}
+
+
{
+ setState("uploading");
+ onSubmitToPlus(true);
+ }}
+ >
+ This is not a {search?.label}
+
+
+ )}
+ {state == "uploading" &&
}
+ {state == "submitted" && (
+
+
+ )}
+
+
+
+ )}
From 66d0ad58031bcd1fc8811503313038c3e4683552 Mon Sep 17 00:00:00 2001
From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>
Date: Sun, 13 Oct 2024 12:46:40 -0500
Subject: [PATCH 23/27] See a preview when using the timeline to export footage
(#14321)
* custom hook and generic video player component
* add export preview dialog
* export preview dialog when using timeline export
* refactor search detail dialog to use new generic video player component
* clean up
---
.../components/filter/ReviewFilterGroup.tsx | 2 +
web/src/components/overlay/ExportDialog.tsx | 59 ++++++++-
.../overlay/MobileReviewSettingsDrawer.tsx | 13 +-
.../components/overlay/SaveExportOverlay.tsx | 28 +++--
.../overlay/detail/SearchDetailDialog.tsx | 118 +++++-------------
.../components/player/GenericVideoPlayer.tsx | 52 ++++++++
web/src/hooks/use-video-dimensions.ts | 45 +++++++
web/src/views/recording/RecordingView.tsx | 5 +
8 files changed, 224 insertions(+), 98 deletions(-)
create mode 100644 web/src/components/player/GenericVideoPlayer.tsx
create mode 100644 web/src/hooks/use-video-dimensions.ts
diff --git a/web/src/components/filter/ReviewFilterGroup.tsx b/web/src/components/filter/ReviewFilterGroup.tsx
index 6d3ee010a..a52755e6c 100644
--- a/web/src/components/filter/ReviewFilterGroup.tsx
+++ b/web/src/components/filter/ReviewFilterGroup.tsx
@@ -241,6 +241,8 @@ export default function ReviewFilterGroup({
mode="none"
setMode={() => {}}
setRange={() => {}}
+ showExportPreview={false}
+ setShowExportPreview={() => {}}
/>
)}
diff --git a/web/src/components/overlay/ExportDialog.tsx b/web/src/components/overlay/ExportDialog.tsx
index c9018c579..577415420 100644
--- a/web/src/components/overlay/ExportDialog.tsx
+++ b/web/src/components/overlay/ExportDialog.tsx
@@ -2,6 +2,7 @@ import { useCallback, useMemo, useState } from "react";
import {
Dialog,
DialogContent,
+ DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
@@ -22,10 +23,13 @@ import { FrigateConfig } from "@/types/frigateConfig";
import { Popover, PopoverContent, PopoverTrigger } from "../ui/popover";
import { TimezoneAwareCalendar } from "./ReviewActivityCalendar";
import { SelectSeparator } from "../ui/select";
-import { isDesktop, isIOS } from "react-device-detect";
+import { isDesktop, isIOS, isMobile } from "react-device-detect";
import { Drawer, DrawerContent, DrawerTrigger } from "../ui/drawer";
import SaveExportOverlay from "./SaveExportOverlay";
import { getUTCOffset } from "@/utils/dateUtil";
+import { baseUrl } from "@/api/baseUrl";
+import { cn } from "@/lib/utils";
+import { GenericVideoPlayer } from "../player/GenericVideoPlayer";
const EXPORT_OPTIONS = [
"1",
@@ -44,8 +48,10 @@ type ExportDialogProps = {
currentTime: number;
range?: TimeRange;
mode: ExportMode;
+ showPreview: boolean;
setRange: (range: TimeRange | undefined) => void;
setMode: (mode: ExportMode) => void;
+ setShowPreview: (showPreview: boolean) => void;
};
export default function ExportDialog({
camera,
@@ -53,10 +59,13 @@ export default function ExportDialog({
currentTime,
range,
mode,
+ showPreview,
setRange,
setMode,
+ setShowPreview,
}: ExportDialogProps) {
const [name, setName] = useState("");
+
const onStartExport = useCallback(() => {
if (!range) {
toast.error("No valid time range selected", { position: "top-center" });
@@ -109,9 +118,16 @@ export default function ExportDialog({
return (
<>
+
setShowPreview(true)}
onSave={() => onStartExport()}
onCancel={() => setMode("none")}
/>
@@ -525,3 +541,44 @@ function CustomTimeSelector({
);
}
+
+type ExportPreviewDialogProps = {
+ camera: string;
+ range?: TimeRange;
+ showPreview: boolean;
+ setShowPreview: (showPreview: boolean) => void;
+};
+
+export function ExportPreviewDialog({
+ camera,
+ range,
+ showPreview,
+ setShowPreview,
+}: ExportPreviewDialogProps) {
+ if (!range) {
+ return null;
+ }
+
+ const source = `${baseUrl}vod/${camera}/start/${range.after}/end/${range.before}/index.m3u8`;
+
+ return (
+
+
+
+ Preview Export
+
+ Preview Export
+
+
+
+
+ );
+}
diff --git a/web/src/components/overlay/MobileReviewSettingsDrawer.tsx b/web/src/components/overlay/MobileReviewSettingsDrawer.tsx
index c9879b8cb..fe0e13c11 100644
--- a/web/src/components/overlay/MobileReviewSettingsDrawer.tsx
+++ b/web/src/components/overlay/MobileReviewSettingsDrawer.tsx
@@ -3,7 +3,7 @@ import { Drawer, DrawerContent, DrawerTrigger } from "../ui/drawer";
import { Button } from "../ui/button";
import { FaArrowDown, FaCalendarAlt, FaCog, FaFilter } from "react-icons/fa";
import { TimeRange } from "@/types/timeline";
-import { ExportContent } from "./ExportDialog";
+import { ExportContent, ExportPreviewDialog } from "./ExportDialog";
import { ExportMode } from "@/types/filter";
import ReviewActivityCalendar from "./ReviewActivityCalendar";
import { SelectSeparator } from "../ui/select";
@@ -34,12 +34,14 @@ type MobileReviewSettingsDrawerProps = {
currentTime: number;
range?: TimeRange;
mode: ExportMode;
+ showExportPreview: boolean;
reviewSummary?: ReviewSummary;
allLabels: string[];
allZones: string[];
onUpdateFilter: (filter: ReviewFilter) => void;
setRange: (range: TimeRange | undefined) => void;
setMode: (mode: ExportMode) => void;
+ setShowExportPreview: (showPreview: boolean) => void;
};
export default function MobileReviewSettingsDrawer({
features = DEFAULT_DRAWER_FEATURES,
@@ -50,12 +52,14 @@ export default function MobileReviewSettingsDrawer({
currentTime,
range,
mode,
+ showExportPreview,
reviewSummary,
allLabels,
allZones,
onUpdateFilter,
setRange,
setMode,
+ setShowExportPreview,
}: MobileReviewSettingsDrawerProps) {
const [drawerMode, setDrawerMode] = useState("none");
@@ -282,6 +286,13 @@ export default function MobileReviewSettingsDrawer({
show={mode == "timeline"}
onSave={() => onStartExport()}
onCancel={() => setMode("none")}
+ onPreview={() => setShowExportPreview(true)}
+ />
+ void;
onSave: () => void;
onCancel: () => void;
};
export default function SaveExportOverlay({
className,
show,
+ onPreview,
onSave,
onCancel,
}: SaveExportOverlayProps) {
@@ -24,6 +26,22 @@ export default function SaveExportOverlay({
"mx-auto mt-5 text-center",
)}
>
+
+
+
+
-