From 6607ad297091d4515469e38ced3802ef38216f1b Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Fri, 4 Oct 2024 18:03:40 -0500 Subject: [PATCH] fully remove chroma and add transformers env var --- docker/main/Dockerfile | 8 ++--- .../s6-rc.d/chroma-log/consumer-for | 1 - .../chroma-log/dependencies.d/log-prepare | 0 .../s6-rc.d/chroma-log/pipeline-name | 1 - .../etc/s6-overlay/s6-rc.d/chroma-log/run | 4 --- .../etc/s6-overlay/s6-rc.d/chroma-log/type | 1 - .../s6-rc.d/chroma/dependencies.d/base | 0 .../etc/s6-overlay/s6-rc.d/chroma/finish | 28 ----------------- .../s6-overlay/s6-rc.d/chroma/producer-for | 1 - .../rootfs/etc/s6-overlay/s6-rc.d/chroma/run | 27 ----------------- .../s6-overlay/s6-rc.d/chroma/timeout-kill | 1 - .../rootfs/etc/s6-overlay/s6-rc.d/chroma/type | 1 - .../s6-rc.d/frigate/dependencies.d/chroma | 0 .../etc/s6-overlay/s6-rc.d/log-prepare/run | 2 +- docker/main/rootfs/usr/local/chroma | 14 --------- .../semantic_search/get_search_settings.py | 30 ------------------- 16 files changed, 3 insertions(+), 116 deletions(-) delete mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/consumer-for delete mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/dependencies.d/log-prepare delete mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/pipeline-name delete mode 100755 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/run delete mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/type delete mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/dependencies.d/base delete mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/finish delete mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/producer-for delete mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/run delete mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/timeout-kill delete mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/type delete mode 100644 docker/main/rootfs/etc/s6-overlay/s6-rc.d/frigate/dependencies.d/chroma delete mode 100755 docker/main/rootfs/usr/local/chroma delete mode 100644 docker/main/rootfs/usr/local/semantic_search/get_search_settings.py diff --git a/docker/main/Dockerfile b/docker/main/Dockerfile index 342727de7..3e0dc83db 100644 --- a/docker/main/Dockerfile +++ b/docker/main/Dockerfile @@ -197,12 +197,8 @@ ARG APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn ENV NVIDIA_VISIBLE_DEVICES=all ENV NVIDIA_DRIVER_CAPABILITIES="compute,video,utility" -# Turn off Chroma Telemetry: https://docs.trychroma.com/telemetry#opting-out -ENV ANONYMIZED_TELEMETRY=False -# Allow resetting the chroma database -ENV ALLOW_RESET=True -# Disable tokenizer parallelism warning -ENV TOKENIZERS_PARALLELISM=true +# https://github.com/huggingface/transformers/issues/27214 +ENV TRANSFORMERS_NO_ADVISORY_WARNINGS=1 ENV PATH="/usr/local/go2rtc/bin:/usr/local/tempio/bin:/usr/local/nginx/sbin:${PATH}" ENV LIBAVFORMAT_VERSION_MAJOR=60 diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/consumer-for b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/consumer-for deleted file mode 100644 index 4b935d3cb..000000000 --- a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/consumer-for +++ /dev/null @@ -1 +0,0 @@ -chroma \ No newline at end of file diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/dependencies.d/log-prepare b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/dependencies.d/log-prepare deleted file mode 100644 index e69de29bb..000000000 diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/pipeline-name b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/pipeline-name deleted file mode 100644 index 71256e9ed..000000000 --- a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/pipeline-name +++ /dev/null @@ -1 +0,0 @@ -chroma-pipeline \ No newline at end of file diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/run b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/run deleted file mode 100755 index 2e47fd3eb..000000000 --- a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/run +++ /dev/null @@ -1,4 +0,0 @@ -#!/command/with-contenv bash -# shellcheck shell=bash - -exec logutil-service /dev/shm/logs/chroma diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/type b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/type deleted file mode 100644 index 5883cff0c..000000000 --- a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma-log/type +++ /dev/null @@ -1 +0,0 @@ -longrun diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/dependencies.d/base b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/dependencies.d/base deleted file mode 100644 index e69de29bb..000000000 diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/finish b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/finish deleted file mode 100644 index b6206b4cc..000000000 --- a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/finish +++ /dev/null @@ -1,28 +0,0 @@ -#!/command/with-contenv bash -# shellcheck shell=bash -# Take down the S6 supervision tree when the service exits - -set -o errexit -o nounset -o pipefail - -# Logs should be sent to stdout so that s6 can collect them - -declare exit_code_container -exit_code_container=$(cat /run/s6-linux-init-container-results/exitcode) -readonly exit_code_container -readonly exit_code_service="${1}" -readonly exit_code_signal="${2}" -readonly service="ChromaDB" - -echo "[INFO] Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})" - -if [[ "${exit_code_service}" -eq 256 ]]; then - if [[ "${exit_code_container}" -eq 0 ]]; then - echo $((128 + exit_code_signal)) >/run/s6-linux-init-container-results/exitcode - fi -elif [[ "${exit_code_service}" -ne 0 ]]; then - if [[ "${exit_code_container}" -eq 0 ]]; then - echo "${exit_code_service}" >/run/s6-linux-init-container-results/exitcode - fi -fi - -exec /run/s6/basedir/bin/halt diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/producer-for b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/producer-for deleted file mode 100644 index c17b71e87..000000000 --- a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/producer-for +++ /dev/null @@ -1 +0,0 @@ -chroma-log diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/run b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/run deleted file mode 100644 index ef477d8a2..000000000 --- a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/run +++ /dev/null @@ -1,27 +0,0 @@ -#!/command/with-contenv bash -# shellcheck shell=bash -# Start the Frigate service - -set -o errexit -o nounset -o pipefail - -# Logs should be sent to stdout so that s6 can collect them - -# Tell S6-Overlay not to restart this service -s6-svc -O . - -search_enabled=`python3 /usr/local/semantic_search/get_search_settings.py | jq -r .enabled` - -# Replace the bash process with the Frigate process, redirecting stderr to stdout -exec 2>&1 - -if [[ "$search_enabled" == 'true' ]]; then - echo "[INFO] Starting ChromaDB..." - exec /usr/local/chroma run --path /config/chroma --host 127.0.0.1 -else - while true - do - sleep 9999 - continue - done - exit 0 -fi diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/timeout-kill b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/timeout-kill deleted file mode 100644 index 6f4f41844..000000000 --- a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/timeout-kill +++ /dev/null @@ -1 +0,0 @@ -120000 diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/type b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/type deleted file mode 100644 index 5883cff0c..000000000 --- a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/chroma/type +++ /dev/null @@ -1 +0,0 @@ -longrun diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/frigate/dependencies.d/chroma b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/frigate/dependencies.d/chroma deleted file mode 100644 index e69de29bb..000000000 diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/log-prepare/run b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/log-prepare/run index 0661f01c2..c493e320e 100755 --- a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/log-prepare/run +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/log-prepare/run @@ -4,7 +4,7 @@ set -o errexit -o nounset -o pipefail -dirs=(/dev/shm/logs/frigate /dev/shm/logs/go2rtc /dev/shm/logs/nginx /dev/shm/logs/certsync /dev/shm/logs/chroma) +dirs=(/dev/shm/logs/frigate /dev/shm/logs/go2rtc /dev/shm/logs/nginx /dev/shm/logs/certsync) mkdir -p "${dirs[@]}" chown nobody:nogroup "${dirs[@]}" diff --git a/docker/main/rootfs/usr/local/chroma b/docker/main/rootfs/usr/local/chroma deleted file mode 100755 index 5147db387..000000000 --- a/docker/main/rootfs/usr/local/chroma +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/python3 -# -*- coding: utf-8 -*-s -__import__("pysqlite3") - -import re -import sys - -sys.modules["sqlite3"] = sys.modules.pop("pysqlite3") - -from chromadb.cli.cli import app - -if __name__ == "__main__": - sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0]) - sys.exit(app()) diff --git a/docker/main/rootfs/usr/local/semantic_search/get_search_settings.py b/docker/main/rootfs/usr/local/semantic_search/get_search_settings.py deleted file mode 100644 index ec3c9c1fa..000000000 --- a/docker/main/rootfs/usr/local/semantic_search/get_search_settings.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Prints the semantic_search config as json to stdout.""" - -import json -import os - -from ruamel.yaml import YAML - -yaml = YAML() - -config_file = os.environ.get("CONFIG_FILE", "/config/config.yml") - -# Check if we can use .yaml instead of .yml -config_file_yaml = config_file.replace(".yml", ".yaml") -if os.path.isfile(config_file_yaml): - config_file = config_file_yaml - -try: - with open(config_file) as f: - raw_config = f.read() - - if config_file.endswith((".yaml", ".yml")): - config: dict[str, any] = yaml.load(raw_config) - elif config_file.endswith(".json"): - config: dict[str, any] = json.loads(raw_config) -except FileNotFoundError: - config: dict[str, any] = {} - -search_config: dict[str, any] = config.get("semantic_search", {"enabled": False}) - -print(json.dumps(search_config))