Refactor ONNX embedding class to use a base class and type-specific classes (#16703)

* Move onnx runner

* Build out base embedding

* Convert text embedding to separate class

* Move image embedding to separate

* Move LPR to separate class

* Remove mono embedding

* Simplify model downloading

* Reorganize jina v1 embeddings

* Cleanup

* Cleanup for review
This commit is contained in:
Nicolas Mowen
2025-02-20 10:17:07 -06:00
committed by GitHub
parent 649e5cfda5
commit c736b1dae5
8 changed files with 701 additions and 449 deletions
+3 -25
View File
@@ -22,7 +22,7 @@ from frigate.types import ModelStatusTypesEnum
from frigate.util.builtin import serialize
from frigate.util.path import get_event_thumbnail_bytes
from .functions.onnx import GenericONNXEmbedding, ModelTypeEnum
from .onnx.jina_v1_embedding import JinaV1ImageEmbedding, JinaV1TextEmbedding
logger = logging.getLogger(__name__)
@@ -97,36 +97,14 @@ class Embeddings:
},
)
self.text_embedding = GenericONNXEmbedding(
model_name="jinaai/jina-clip-v1",
model_file="text_model_fp16.onnx",
tokenizer_file="tokenizer",
download_urls={
"text_model_fp16.onnx": "https://huggingface.co/jinaai/jina-clip-v1/resolve/main/onnx/text_model_fp16.onnx",
},
self.text_embedding = JinaV1TextEmbedding(
model_size=config.semantic_search.model_size,
model_type=ModelTypeEnum.text,
requestor=self.requestor,
device="CPU",
)
model_file = (
"vision_model_fp16.onnx"
if self.config.semantic_search.model_size == "large"
else "vision_model_quantized.onnx"
)
download_urls = {
model_file: f"https://huggingface.co/jinaai/jina-clip-v1/resolve/main/onnx/{model_file}",
"preprocessor_config.json": "https://huggingface.co/jinaai/jina-clip-v1/resolve/main/preprocessor_config.json",
}
self.vision_embedding = GenericONNXEmbedding(
model_name="jinaai/jina-clip-v1",
model_file=model_file,
download_urls=download_urls,
self.vision_embedding = JinaV1ImageEmbedding(
model_size=config.semantic_search.model_size,
model_type=ModelTypeEnum.vision,
requestor=self.requestor,
device="GPU" if config.semantic_search.model_size == "large" else "CPU",
)