mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-02 03:40:31 +03:00
build(tensorrt): JetPack 7 / L4T R39 image with GPU-accelerated ONNX detector
Adds a `-tensorrt-jp7` Frigate image for JetPack 7.2 / L4T R39.2 Jetson hosts, built on nvcr.io/nvidia/tensorrt:26.02-py3-igpu (TensorRT 10.11, CUDA 13, py3.12), keeping the existing JP6 path unchanged. ONNX Runtime GPU is built from source (no public aarch64 onnxruntime-gpu wheel), TensorRT-Python branch is selected by the base image, and TensorRT runtime library checks are major-version aware. VALIDATED on a real AGX Orin (L4T R39.2 / nv_tegra_release R39 rev 2.0): - image frigate:test-tensorrt-jp7 builds (rc=0, 16.3GB, sha256:c9e4d382f1603ee130ee4a7315b4f71f9461405e3785707251505e2d6d088df3) - ONNX Runtime 1.25.1 exposes CUDAExecutionProvider, and a real Add-model inference RAN on the iGPU CUDA EP (functional, not just listed) - frigate.util.model.get_ort_providers(False,"AUTO") = [CUDA, CPU] (CUDA first, CPU last, TensorRT EP excluded) -> the ONNX detector GPU-accelerates on JP7 - /etc/TENSORRT_VER = 10.11.0 Native `type: tensorrt` (.trt gen) + the ORT TensorRT EP stay DRAFT-GATED: the L4T R39 host ships NO libnvdla_compiler.so (absent from host AND base image), so `import tensorrt` and libonnxruntime_providers_tensorrt.so fail to load. ONNX detector GPU acceleration is the supported JP7 path; native trt is deferred. Build fixes the new noble/CUDA-13 base surfaced (beyond the plan): - build_nginx.sh: enable deb-src for the deb822 ubuntu.sources (Ubuntu 24.04) - tensorrt_libyolo.sh: strip -lnvToolsExt (removed in CUDA 13) + -lnvparsers (dropped in TensorRT 10) when those libs are absent - docker/main: noble/py3.12 build adjustments (Dockerfile, build_sqlite_vec.sh) Reproducible: `make -C docker/tensorrt ... local-trt-jp7` (JETPACK7_ARGS) on any arm64 builder; built on-device only because the GPU smoke test needs the iGPU.
This commit is contained in:
Jorge Ramirez
+72
@@ -0,0 +1,72 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euxo pipefail
|
||||
|
||||
ONNXRUNTIME_VERSION=${ONNXRUNTIME_VERSION:-1.25.1}
|
||||
ONNXRUNTIME_BRANCH=${ONNXRUNTIME_BRANCH:-rel-1.25.1}
|
||||
ORT_PARALLEL=${ORT_PARALLEL:-4}
|
||||
ORT_NVCC_THREADS=${ORT_NVCC_THREADS:-1}
|
||||
|
||||
mkdir -p /trt-wheels /workspace
|
||||
|
||||
pip3 install --upgrade pip setuptools wheel packaging numpy
|
||||
|
||||
command -v nvcc
|
||||
test -f /usr/local/cuda/include/cuda.h
|
||||
|
||||
require_header() {
|
||||
local pattern="$1"
|
||||
local header
|
||||
|
||||
for header in /usr/include/${pattern} /usr/include/aarch64-linux-gnu/${pattern}; do
|
||||
if [[ -e "${header}" ]]; then
|
||||
echo "${header}"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Missing required header matching ${pattern}" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
require_ldconfig_entry() {
|
||||
local lib_name="$1"
|
||||
|
||||
if ! ldconfig -p | grep -q "${lib_name}\\.so"; then
|
||||
echo "Missing ldconfig entry for ${lib_name}" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
require_header "cudnn.h"
|
||||
require_header "cudnn_version*.h"
|
||||
require_header "NvInfer.h"
|
||||
require_header "NvOnnxParser.h"
|
||||
|
||||
require_ldconfig_entry "libnvinfer"
|
||||
require_ldconfig_entry "libnvinfer_plugin"
|
||||
require_ldconfig_entry "libnvonnxparser"
|
||||
|
||||
cd /workspace
|
||||
rm -rf onnxruntime
|
||||
git clone --recursive --branch "${ONNXRUNTIME_BRANCH}" https://github.com/microsoft/onnxruntime.git onnxruntime
|
||||
|
||||
cd /workspace/onnxruntime
|
||||
./build.sh \
|
||||
--config Release \
|
||||
--update \
|
||||
--build \
|
||||
--build_wheel \
|
||||
--parallel "${ORT_PARALLEL}" \
|
||||
--nvcc_threads "${ORT_NVCC_THREADS}" \
|
||||
--allow_running_as_root \
|
||||
--compile_no_warning_as_error \
|
||||
--skip_tests \
|
||||
--use_cuda \
|
||||
--cuda_home /usr/local/cuda \
|
||||
--use_tensorrt \
|
||||
--cudnn_home /usr/lib/aarch64-linux-gnu \
|
||||
--tensorrt_home /usr/lib/aarch64-linux-gnu \
|
||||
--cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=OFF
|
||||
|
||||
cp build/Linux/Release/dist/onnxruntime_gpu-"${ONNXRUNTIME_VERSION}"-*.whl /trt-wheels/
|
||||
@@ -2,6 +2,19 @@
|
||||
|
||||
set -euxo pipefail
|
||||
|
||||
TENSORRT_PYTHON_BRANCH=${TENSORRT_PYTHON_BRANCH:-release/8.6}
|
||||
|
||||
if [[ -n "${TENSORRT_VER:-}" ]]; then
|
||||
TENSORRT_MAJOR_MINOR="${TENSORRT_VER%.*}"
|
||||
EXPECTED_TENSORRT_PYTHON_BRANCH="release/${TENSORRT_MAJOR_MINOR}"
|
||||
|
||||
if [[ "${TENSORRT_PYTHON_BRANCH}" == "auto" ]]; then
|
||||
TENSORRT_PYTHON_BRANCH="${EXPECTED_TENSORRT_PYTHON_BRANCH}"
|
||||
elif [[ "${TENSORRT_PYTHON_BRANCH}" != "${EXPECTED_TENSORRT_PYTHON_BRANCH}" ]]; then
|
||||
echo "WARNING: TENSORRT_PYTHON_BRANCH=${TENSORRT_PYTHON_BRANCH} does not match TENSORRT_VER=${TENSORRT_VER}; auto would select ${EXPECTED_TENSORRT_PYTHON_BRANCH}" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
mkdir -p /trt-wheels
|
||||
|
||||
if [[ "${TARGETARCH}" == "arm64" ]]; then
|
||||
@@ -12,13 +25,44 @@ if [[ "${TARGETARCH}" == "arm64" ]]; then
|
||||
# Get python-tensorrt source
|
||||
mkdir -p /workspace
|
||||
cd /workspace
|
||||
git clone -b release/8.6 https://github.com/NVIDIA/TensorRT.git --depth=1
|
||||
git clone -b "${TENSORRT_PYTHON_BRANCH}" https://github.com/NVIDIA/TensorRT.git --depth=1
|
||||
|
||||
# The TensorRT python build runs the legacy `setup.py bdist_wheel`, which on the JP7 base
|
||||
# otherwise resolves the deadsnakes/Debian system setuptools in /usr/lib/python3/dist-packages.
|
||||
# That copy's wheel.bdist_wheel routes through distutils `install` -> `install_lib`, whose
|
||||
# finalize_options reads a Debian-only `install_layout` option that the install command lacks
|
||||
# under this setuptools, crashing with "AttributeError: install_layout". Install a clean,
|
||||
# non-Debian setuptools+wheel into /usr/local (which python3.11 imports ahead of /usr/lib)
|
||||
# so bdist_wheel uses a consistent toolchain. JP6 already had a compatible setuptools.
|
||||
pip3 install --upgrade --ignore-installed 'setuptools>=70.1,<81' 'wheel>=0.43'
|
||||
|
||||
# Collect dependencies
|
||||
EXT_PATH=/workspace/external && mkdir -p $EXT_PATH
|
||||
pip3 install pybind11 && ln -s /usr/local/lib/python3.11/dist-packages/pybind11 $EXT_PATH/pybind11
|
||||
ln -s /usr/include/python3.11 $EXT_PATH/python3.11
|
||||
ln -s /usr/include/aarch64-linux-gnu/NvOnnxParser.h /workspace/TensorRT/parsers/onnx/
|
||||
|
||||
# TensorRT 10's python onnx bindings (pyOnnx.cpp) #include onnx-tensorrt headers such as
|
||||
# errorHelpers.hpp that live ONLY in the parsers/onnx submodule (onnx-tensorrt), not in the
|
||||
# base image's system include or the shallow TensorRT clone. TRT 8.6 bindings did not need
|
||||
# these, so a lone NvOnnxParser.h symlink sufficed there. Populate the submodule at its pinned
|
||||
# ref so CMake's ONNX_INC_DIR=${TENSORRT_ROOT}/parsers/onnx resolves every header it needs.
|
||||
cd /workspace/TensorRT
|
||||
if git submodule update --init --depth=1 parsers/onnx 2>/dev/null && [[ -e parsers/onnx/errorHelpers.hpp ]]; then
|
||||
:
|
||||
else
|
||||
# Fallback: clone onnx-tensorrt at the branch recorded in .gitmodules for this TRT release.
|
||||
ONNX_TRT_BRANCH="$(git config -f .gitmodules submodule.parsers/onnx.branch || echo '')"
|
||||
rm -rf parsers/onnx
|
||||
git clone --depth=1 ${ONNX_TRT_BRANCH:+-b "${ONNX_TRT_BRANCH}"} https://github.com/onnx/onnx-tensorrt.git parsers/onnx
|
||||
fi
|
||||
# Prefer the system NvOnnxParser.h (matches the installed libnvonnxparser) when present.
|
||||
for header in /usr/include/aarch64-linux-gnu/NvOnnxParser.h /usr/include/NvOnnxParser.h; do
|
||||
if [[ -e "$header" ]]; then
|
||||
ln -sf "$header" /workspace/TensorRT/parsers/onnx/NvOnnxParser.h
|
||||
break
|
||||
fi
|
||||
done
|
||||
cd /workspace
|
||||
|
||||
# Build wheel
|
||||
cd /workspace/TensorRT/python
|
||||
|
||||
@@ -10,11 +10,12 @@ set -o errexit -o nounset -o pipefail
|
||||
|
||||
MODEL_CACHE_DIR=${MODEL_CACHE_DIR:-"/config/model_cache/tensorrt"}
|
||||
TRT_VER=${TRT_VER:-$(cat /etc/TENSORRT_VER)}
|
||||
TRT_MAJOR=${TRT_VER%%.*}
|
||||
OUTPUT_FOLDER="${MODEL_CACHE_DIR}/${TRT_VER}"
|
||||
YOLO_MODELS=${YOLO_MODELS:-""}
|
||||
|
||||
# Create output folder
|
||||
mkdir -p ${OUTPUT_FOLDER}
|
||||
mkdir -p "${OUTPUT_FOLDER}"
|
||||
|
||||
FIRST_MODEL=true
|
||||
MODEL_DOWNLOAD=""
|
||||
@@ -28,9 +29,9 @@ fi
|
||||
for model in ${YOLO_MODELS//,/ }
|
||||
do
|
||||
# Remove old link in case path/version changed
|
||||
rm -f ${MODEL_CACHE_DIR}/${model}.trt
|
||||
rm -f "${MODEL_CACHE_DIR}/${model}.trt"
|
||||
|
||||
if [[ ! -f ${OUTPUT_FOLDER}/${model}.trt ]]; then
|
||||
if [[ ! -f "${OUTPUT_FOLDER}/${model}.trt" ]]; then
|
||||
if [[ ${FIRST_MODEL} = true ]]; then
|
||||
MODEL_DOWNLOAD="${model%-dla}";
|
||||
MODEL_CONVERT="${model}"
|
||||
@@ -40,7 +41,7 @@ do
|
||||
MODEL_CONVERT+=",${model}";
|
||||
fi
|
||||
else
|
||||
ln -s ${OUTPUT_FOLDER}/${model}.trt ${MODEL_CACHE_DIR}/${model}.trt
|
||||
ln -s "${OUTPUT_FOLDER}/${model}.trt" "${MODEL_CACHE_DIR}/${model}.trt"
|
||||
fi
|
||||
done
|
||||
|
||||
@@ -50,8 +51,8 @@ if [[ -z ${MODEL_CONVERT} ]]; then
|
||||
fi
|
||||
|
||||
# Setup ENV to select GPU for conversion
|
||||
if [ ! -z ${TRT_MODEL_PREP_DEVICE+x} ]; then
|
||||
if [ ! -z ${CUDA_VISIBLE_DEVICES+x} ]; then
|
||||
if [ -n "${TRT_MODEL_PREP_DEVICE+x}" ]; then
|
||||
if [ -n "${CUDA_VISIBLE_DEVICES+x}" ]; then
|
||||
PREVIOUS_CVD="$CUDA_VISIBLE_DEVICES"
|
||||
unset CUDA_VISIBLE_DEVICES
|
||||
fi
|
||||
@@ -67,12 +68,30 @@ if [[ "$(arch)" == "aarch64" ]]; then
|
||||
if [[ ! -e /usr/lib/aarch64-linux-gnu/tegra && ! -e /usr/lib/aarch64-linux-gnu/tegra-egl ]]; then
|
||||
echo "ERROR: Container must be launched with nvidia runtime"
|
||||
exit 1
|
||||
elif [[ ! -e /usr/lib/aarch64-linux-gnu/libnvinfer.so.8 ||
|
||||
! -e /usr/lib/aarch64-linux-gnu/libnvinfer_plugin.so.8 ||
|
||||
! -e /usr/lib/aarch64-linux-gnu/libnvparsers.so.8 ||
|
||||
! -e /usr/lib/aarch64-linux-gnu/libnvonnxparser.so.8 ]]; then
|
||||
echo "ERROR: Please run the following on the HOST:"
|
||||
echo " sudo apt install libnvinfer8 libnvinfer-plugin8 libnvparsers8 libnvonnxparsers8 nvidia-container"
|
||||
fi
|
||||
|
||||
TRT_LIB_DIR=/usr/lib/aarch64-linux-gnu
|
||||
REQUIRED_TRT_LIBS=(
|
||||
"${TRT_LIB_DIR}/libnvinfer.so.${TRT_MAJOR}"
|
||||
"${TRT_LIB_DIR}/libnvinfer_plugin.so.${TRT_MAJOR}"
|
||||
"${TRT_LIB_DIR}/libnvonnxparser.so.${TRT_MAJOR}"
|
||||
)
|
||||
|
||||
if (( TRT_MAJOR < 10 )) || [[ -e "${TRT_LIB_DIR}/libnvparsers.so.${TRT_MAJOR}" ]]; then
|
||||
REQUIRED_TRT_LIBS+=("${TRT_LIB_DIR}/libnvparsers.so.${TRT_MAJOR}")
|
||||
fi
|
||||
|
||||
MISSING_TRT_LIBS=()
|
||||
for TRT_LIB in "${REQUIRED_TRT_LIBS[@]}"; do
|
||||
if [[ ! -e "${TRT_LIB}" ]]; then
|
||||
MISSING_TRT_LIBS+=("${TRT_LIB}")
|
||||
fi
|
||||
done
|
||||
|
||||
if (( ${#MISSING_TRT_LIBS[@]} > 0 )); then
|
||||
echo "ERROR: Missing TensorRT runtime libraries:"
|
||||
printf ' %s\n' "${MISSING_TRT_LIBS[@]}"
|
||||
echo "ERROR: Install the matching TensorRT ${TRT_MAJOR} runtime packages and nvidia-container on the HOST."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
@@ -83,33 +102,33 @@ echo "Generating the following TRT Models: ${MODEL_CONVERT}"
|
||||
cd /usr/local/src/tensorrt_demos/yolo
|
||||
|
||||
echo "Downloading yolo weights"
|
||||
./download_yolo.sh $MODEL_DOWNLOAD 2> /dev/null
|
||||
./download_yolo.sh "${MODEL_DOWNLOAD}" 2> /dev/null
|
||||
|
||||
for model in ${MODEL_CONVERT//,/ }
|
||||
do
|
||||
python3 yolo_to_onnx.py -m ${model%-dla} > /dev/null
|
||||
python3 yolo_to_onnx.py -m "${model%-dla}" > /dev/null
|
||||
|
||||
echo -e "\nGenerating ${model}.trt. This may take a few minutes.\n"; start=$(date +%s)
|
||||
if [[ $model == *-dla ]]; then
|
||||
cmd="python3 onnx_to_tensorrt.py -m ${model%-dla} --dla_core 0"
|
||||
cmd=(python3 onnx_to_tensorrt.py -m "${model%-dla}" --dla_core 0)
|
||||
else
|
||||
cmd="python3 onnx_to_tensorrt.py -m ${model}"
|
||||
cmd=(python3 onnx_to_tensorrt.py -m "${model}")
|
||||
fi
|
||||
$cmd > /tmp/onnx_to_tensorrt.log || { cat /tmp/onnx_to_tensorrt.log && continue; }
|
||||
"${cmd[@]}" > /tmp/onnx_to_tensorrt.log || { cat /tmp/onnx_to_tensorrt.log && continue; }
|
||||
|
||||
mv ${model%-dla}.trt ${OUTPUT_FOLDER}/${model}.trt;
|
||||
ln -s ${OUTPUT_FOLDER}/${model}.trt ${MODEL_CACHE_DIR}/${model}.trt
|
||||
mv "${model%-dla}.trt" "${OUTPUT_FOLDER}/${model}.trt";
|
||||
ln -s "${OUTPUT_FOLDER}/${model}.trt" "${MODEL_CACHE_DIR}/${model}.trt"
|
||||
echo "Generated ${model}.trt in $(($(date +%s)-start)) seconds"
|
||||
done
|
||||
|
||||
# Restore ENV after conversion
|
||||
if [ ! -z ${TRT_MODEL_PREP_DEVICE+x} ]; then
|
||||
if [ -n "${TRT_MODEL_PREP_DEVICE+x}" ]; then
|
||||
unset CUDA_VISIBLE_DEVICES
|
||||
if [ ! -z ${PREVIOUS_CVD+x} ]; then
|
||||
if [ -n "${PREVIOUS_CVD+x}" ]; then
|
||||
export CUDA_VISIBLE_DEVICES="$PREVIOUS_CVD"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Print which models exist in output folder
|
||||
echo "Available tensorrt models:"
|
||||
cd ${OUTPUT_FOLDER} && ls *.trt;
|
||||
cd "${OUTPUT_FOLDER}" && ls -- *.trt;
|
||||
|
||||
@@ -7,6 +7,21 @@ SCRIPT_DIR="/usr/local/src/tensorrt_demos"
|
||||
# Clone tensorrt_demos repo
|
||||
git clone --depth 1 https://github.com/NateMeyer/tensorrt_demos.git -b conditional_download
|
||||
|
||||
# CUDA 13 removed libnvToolsExt.so (NVTX is now header-only nvtx3). The plugin Makefile links
|
||||
# -lnvToolsExt only for optional NVTX profiling annotations, so strip it when the lib is absent —
|
||||
# the link then succeeds on JP7 / CUDA 13. No-op on CUDA 12 (JP6), where the lib still exists.
|
||||
if ! ldconfig -p | grep -q 'libnvToolsExt\.so'; then
|
||||
sed -i 's/-lnvToolsExt//g' ./tensorrt_demos/plugins/Makefile
|
||||
fi
|
||||
|
||||
# TensorRT 10 (JP7 base) dropped libnvparsers.so (the legacy UFF/Caffe parsers). The plugin
|
||||
# Makefile links -lnvparsers from an over-broad LIBS list, but the YOLO custom layer never uses
|
||||
# it, so strip it when the lib is absent — the link then succeeds on TRT 10. No-op on TRT 8
|
||||
# (JP5/JP6), where libnvparsers still ships.
|
||||
if ! ldconfig -p | grep -q 'libnvparsers\.so'; then
|
||||
sed -i 's/-lnvparsers//g' ./tensorrt_demos/plugins/Makefile
|
||||
fi
|
||||
|
||||
# Build libyolo
|
||||
if [ ! -e /usr/local/cuda ]; then
|
||||
ln -s /usr/local/cuda-* /usr/local/cuda
|
||||
|
||||
Reference in New Issue
Block a user