Audio transcription support (#18398)

* install new packages for transcription support

* add config options

* audio maintainer modifications to support transcription

* pass main config to audio process

* embeddings support

* api and transcription post processor

* embeddings maintainer support for post processor

* live audio transcription with sherpa and faster-whisper

* update dispatcher with live transcription topic

* frontend websocket

* frontend live transcription

* frontend changes for speech events

* i18n changes

* docs

* mqtt docs

* fix linter

* use float16 and small model on gpu for real-time

* fix return value and use requestor to embed description instead of passing embeddings

* run real-time transcription in its own thread

* tweaks

* publish live transcriptions on their own topic instead of tracked_object_update

* config validator and docs

* clarify docs
This commit is contained in:
Josh Hawkins
2025-08-16 10:20:33 -05:00
committed by Blake Blackshear
parent 2385c403ee
commit 6dc36fcbb4
29 changed files with 2322 additions and 51 deletions
@@ -78,6 +78,7 @@ import { TbFaceId } from "react-icons/tb";
import { useIsAdmin } from "@/hooks/use-is-admin";
import FaceSelectionDialog from "../FaceSelectionDialog";
import { getTranslatedLabel } from "@/utils/i18n";
import { CgTranscript } from "react-icons/cg";
const SEARCH_TABS = [
"details",
@@ -710,6 +711,34 @@ function ObjectDetailsTab({
[search, t],
);
// speech transcription
const onTranscribe = useCallback(() => {
axios
.put(`/audio/transcribe`, { event_id: search.id })
.then((resp) => {
if (resp.status == 202) {
toast.success(t("details.item.toast.success.audioTranscription"), {
position: "top-center",
});
}
})
.catch((error) => {
const errorMessage =
error.response?.data?.message ||
error.response?.data?.detail ||
"Unknown error";
toast.error(
t("details.item.toast.error.audioTranscription", {
errorMessage,
}),
{
position: "top-center",
},
);
});
}, [search, t]);
return (
<div className="flex flex-col gap-5">
<div className="flex w-full flex-row">
@@ -893,6 +922,16 @@ function ObjectDetailsTab({
</Button>
</FaceSelectionDialog>
)}
{config?.cameras[search?.camera].audio_transcription.enabled &&
search?.label == "speech" &&
search?.end_time && (
<Button className="w-full" onClick={onTranscribe}>
<div className="flex gap-1">
<CgTranscript />
{t("itemMenu.audioTranscription.label")}
</div>
</Button>
)}
</div>
</div>
</div>