Merge 7933a83a42 into e79ff9a079

Co-authored-by: Nicolas Mowen <nickmowen213@gmail.com>

.github/workflows/ci.yml

@@ -225,3 +225,29 @@ jobs:
           sources: |
             ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:${{ env.SHORT_SHA }}-amd64
             ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:${{ env.SHORT_SHA }}-rpi
+  axera_build:
+    runs-on: ubuntu-22.04
+    name: AXERA Build
+    needs:
+      - amd64_build
+      - arm64_build
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v5
+        with:
+          persist-credentials: false
+      - name: Set up QEMU and Buildx
+        id: setup
+        uses: ./.github/actions/setup
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push Axera build
+        uses: docker/bake-action@v6
+        with:
+          source: .
+          push: true
+          targets: axcl
+          files: docker/axcl/axcl.hcl
+          set: |
+            axcl.tags=${{ steps.setup.outputs.image-name }}-axcl
+            *.cache-from=type=gha

docker/axcl/Dockerfile

@@ -0,0 +1,55 @@
+# syntax=docker/dockerfile:1.6
+
+# https://askubuntu.com/questions/972516/debian-frontend-environment-variable
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Globally set pip break-system-packages option to avoid having to specify it every time
+ARG PIP_BREAK_SYSTEM_PACKAGES=1
+
+
+FROM frigate AS frigate-axcl
+ARG TARGETARCH
+ARG PIP_BREAK_SYSTEM_PACKAGES
+
+# Install axpyengine
+RUN wget https://github.com/AXERA-TECH/pyaxengine/releases/download/0.1.3.rc1/axengine-0.1.3-py3-none-any.whl -O /axengine-0.1.3-py3-none-any.whl
+RUN pip3 install -i https://mirrors.aliyun.com/pypi/simple/ /axengine-0.1.3-py3-none-any.whl \
+    && rm /axengine-0.1.3-py3-none-any.whl
+
+# Install axcl
+RUN if [ "$TARGETARCH" = "amd64" ]; then \
+        echo "Installing x86_64 version of axcl"; \
+        wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/axcl_host_x86_64_V3.6.5_20250908154509_NO4973.deb -O /axcl.deb; \
+    else \
+        echo "Installing aarch64 version of axcl"; \
+        wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/axcl_host_aarch64_V3.6.5_20250908154509_NO4973.deb -O /axcl.deb; \
+    fi
+
+RUN mkdir /unpack_axcl && \
+    dpkg-deb -x /axcl.deb /unpack_axcl && \
+    cp -R /unpack_axcl/usr/bin/axcl /usr/bin/ && \
+    cp -R /unpack_axcl/usr/lib/axcl /usr/lib/ && \
+    rm -rf /unpack_axcl /axcl.deb
+
+
+# Install axcl ffmpeg
+RUN mkdir -p /usr/lib/ffmpeg/axcl
+
+RUN if [ "$TARGETARCH" = "amd64" ]; then \
+        wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/ffmpeg-x64 -O /usr/lib/ffmpeg/axcl/ffmpeg && \
+        wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/ffprobe-x64 -O /usr/lib/ffmpeg/axcl/ffprobe; \
+    else \
+        wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/ffmpeg-aarch64 -O /usr/lib/ffmpeg/axcl/ffmpeg && \
+        wget https://github.com/ivanshi1108/assets/releases/download/v0.16.2/ffprobe-aarch64 -O /usr/lib/ffmpeg/axcl/ffprobe; \
+    fi
+
+RUN chmod +x /usr/lib/ffmpeg/axcl/ffmpeg /usr/lib/ffmpeg/axcl/ffprobe
+
+# Set ldconfig path
+RUN echo "/usr/lib/axcl" > /etc/ld.so.conf.d/ax.conf
+
+# Set env
+ENV PATH="$PATH:/usr/bin/axcl"
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/axcl"
+
+ENTRYPOINT ["sh", "-c", "ldconfig && exec /init"]

docker/axcl/axcl.hcl

@@ -0,0 +1,13 @@
+target frigate {
+  dockerfile = "docker/main/Dockerfile"
+  platforms = ["linux/amd64", "linux/arm64"]
+  target = "frigate"
+}
+
+target axcl {
+  dockerfile = "docker/axcl/Dockerfile"
+  contexts = {
+    frigate = "target:frigate",
+  }
+  platforms = ["linux/amd64", "linux/arm64"]
+}

docker/axcl/axcl.mk

@@ -0,0 +1,15 @@
+BOARDS += axcl
+
+local-axcl: version
+	docker buildx bake --file=docker/axcl/axcl.hcl axcl \
+		--set axcl.tags=frigate:latest-axcl \
+		--load
+
+build-axcl: version
+	docker buildx bake --file=docker/axcl/axcl.hcl axcl \
+		--set axcl.tags=$(IMAGE_REPO):${GITHUB_REF_NAME}-$(COMMIT_HASH)-axcl
+
+push-axcl: build-axcl
+	docker buildx bake --file=docker/axcl/axcl.hcl axcl \
+		--set axcl.tags=$(IMAGE_REPO):${GITHUB_REF_NAME}-$(COMMIT_HASH)-axcl \
+		--push

docker/axcl/user_installation.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+# Update package list and install dependencies
+sudo apt-get update
+sudo apt-get install -y build-essential cmake git wget pciutils kmod udev
+
+# Check if gcc-12 is needed
+current_gcc_version=$(gcc --version | head -n1 | awk '{print $NF}')
+gcc_major_version=$(echo $current_gcc_version | cut -d'.' -f1)
+
+if [[ $gcc_major_version -lt 12 ]]; then
+    echo "Current GCC version ($current_gcc_version) is lower than 12, installing gcc-12..."
+    sudo apt-get install -y gcc-12
+    sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 12
+    echo "GCC-12 installed and set as default"
+else
+    echo "Current GCC version ($current_gcc_version) is sufficient, skipping GCC installation"
+fi
+
+# Determine architecture
+arch=$(uname -m)
+download_url=""
+
+if [[ $arch == "x86_64" ]]; then
+    download_url="https://github.com/ivanshi1108/assets/releases/download/v0.16.2/axcl_host_x86_64_V3.6.5_20250908154509_NO4973.deb"
+    deb_file="axcl_host_x86_64_V3.6.5_20250908154509_NO4973.deb"
+elif [[ $arch == "aarch64" ]]; then
+    download_url="https://github.com/ivanshi1108/assets/releases/download/v0.16.2/axcl_host_aarch64_V3.6.5_20250908154509_NO4973.deb"
+    deb_file="axcl_host_aarch64_V3.6.5_20250908154509_NO4973.deb"
+else
+    echo "Unsupported architecture: $arch"
+    exit 1
+fi
+
+# Download AXCL driver
+echo "Downloading AXCL driver for $arch..."
+wget "$download_url" -O "$deb_file"
+
+if [ $? -ne 0 ]; then
+    echo "Failed to download AXCL driver"
+    exit 1
+fi
+
+# Install AXCL driver
+echo "Installing AXCL driver..."
+sudo dpkg -i "$deb_file"
+
+if [ $? -ne 0 ]; then
+    echo "Failed to install AXCL driver, attempting to fix dependencies..."
+    sudo apt-get install -f -y
+    sudo dpkg -i "$deb_file"
+
+    if [ $? -ne 0 ]; then
+        echo "AXCL driver installation failed"
+        exit 1
+    fi
+fi
+
+# Update environment
+echo "Updating environment..."
+source /etc/profile
+
+# Verify installation
+echo "Verifying AXCL installation..."
+if command -v axcl-smi &> /dev/null; then
+    echo "AXCL driver detected, checking AI accelerator status..."
+
+    axcl_output=$(axcl-smi 2>&1)
+    axcl_exit_code=$?
+
+    echo "$axcl_output"
+
+    if [ $axcl_exit_code -eq 0 ]; then
+        echo "AXCL driver installation completed successfully!"
+    else
+        echo "AXCL driver installed but no AI accelerator detected or communication failed."
+        echo "Please check if the AI accelerator is properly connected and powered on."
+        exit 1
+    fi
+else
+    echo "axcl-smi command not found. AXCL driver installation may have failed."
+    exit 1
+fi

docker/main/requirements-wheels.txt

@@ -81,3 +81,5 @@ librosa==0.11.*
 soundfile==0.13.*
 # DeGirum detector
 degirum == 0.16.*
+# Memory profiling
+memray == 1.15.*

docs/docs/configuration/object_detectors.md

@@ -49,6 +49,11 @@ Frigate supports multiple different detectors that work on different types of ha
 
 - [Synaptics](#synaptics): synap models can run on Synaptics devices(e.g astra machina) with included NPUs.
 
+**AXERA** <CommunityBadge />
+
+- [AXEngine](#axera): axmodels can run on AXERA AI acceleration.
+
+
 **For Testing**
 
 - [CPU Detector (not recommended for actual use](#cpu-detector-not-recommended): Use a CPU to run tflite model, this is not recommended and in most cases OpenVINO can be used in CPU mode with better results.
@@ -1438,6 +1443,41 @@ model:
   input_pixel_format: rgb/bgr # look at the model.json to figure out which to put here
 ```
 
+## AXERA
+
+Hardware accelerated object detection is supported on the following SoCs:
+
+- AX650N
+- AX8850N
+
+This implementation uses the [AXera Pulsar2 Toolchain](https://huggingface.co/AXERA-TECH/Pulsar2).
+
+See the [installation docs](../frigate/installation.md#axera) for information on configuring the AXEngine hardware.
+
+### Configuration
+
+When configuring the AXEngine detector, you have to specify the model name.
+
+#### yolov9
+
+A yolov9 model is provided in the container at /axmodels and is used by this detector type by default.
+
+Use the model configuration shown below when using the axengine detector with the default axmodel:
+
+```yaml
+detectors:
+  axengine:
+    type: axengine
+
+model:
+  path: frigate-yolov9-tiny
+  model_type: yolo-generic
+  width: 320
+  height: 320
+  tensor_format: bgr
+  labelmap_path: /labelmap/coco-80.txt
+```
+
 # Models
 
 Some model types are not included in Frigate by default.

docs/docs/frigate/hardware.md

@@ -104,6 +104,10 @@ Frigate supports multiple different detectors that work on different types of ha
 
 - [Synaptics](#synaptics): synap models can run on Synaptics devices(e.g astra machina) with included NPUs to provide efficient object detection.
 
+**AXERA** <CommunityBadge />
+
+- [AXEngine](#axera): axera models can run on AXERA NPUs via AXEngine, delivering highly efficient object detection.
+
 :::
 
 ### Hailo-8
@@ -287,6 +291,14 @@ The inference time of a rk3588 with all 3 cores enabled is typically 25-30 ms fo
 | ssd mobilenet | ~ 25 ms                         |
 | yolov5m       | ~ 118 ms                        |
 
+### AXERA
+
+- **AXEngine** Default model is **yolov9**
+
+| Name             | AXERA AX650N/AX8850N Inference Time |
+| ---------------- | ----------------------------------- |
+| yolov9-tiny      | ~ 4 ms                              |
+
 ## What does Frigate use the CPU for and what does it use a detector for? (ELI5 Version)
 
 This is taken from a [user question on reddit](https://www.reddit.com/r/homeassistant/comments/q8mgau/comment/hgqbxh5/?utm_source=share&utm_medium=web2x&context=3). Modified slightly for clarity.
@@ -307,4 +319,4 @@ Basically - When you increase the resolution and/or the frame rate of the stream
 
 YES! The Coral does not help with decoding video streams.
 
-Decompressing video streams takes a significant amount of CPU power. Video compression uses key frames (also known as I-frames) to send a full frame in the video stream. The following frames only include the difference from the key frame, and the CPU has to compile each frame by merging the differences with the key frame. [More detailed explanation](https://support.video.ibm.com/hc/en-us/articles/18106203580316-Keyframes-InterFrame-Video-Compression). Higher resolutions and frame rates mean more processing power is needed to decode the video stream, so try and set them on the camera to avoid unnecessary decoding work.
+Decompressing video streams takes a significant amount of CPU power. Video compression uses key frames (also known as I-frames) to send a full frame in the video stream. The following frames only include the difference from the key frame, and the CPU has to compile each frame by merging the differences with the key frame. [More detailed explanation](https://support.video.ibm.com/hc/en-us/articles/18106203580316-Keyframes-InterFrame-Video-Compression). Higher resolutions and frame rates mean more processing power is needed to decode the video stream, so try and set them on the camera to avoid unnecessary decoding work.

docs/docs/frigate/installation.md

@@ -287,6 +287,42 @@ or add these options to your `docker run` command:
 
 Next, you should configure [hardware object detection](/configuration/object_detectors#synaptics) and [hardware video processing](/configuration/hardware_acceleration_video#synaptics).
 
+### AXERA
+
+<details>
+<summary>AXERA accelerators</summary>
+AXERA accelerators are available in an M.2 form factor, compatible with both Raspberry Pi and Orange Pi. This form factor has also been successfully tested on x86 platforms, making it a versatile choice for various computing environments.
+
+#### Installation
+
+Using AXERA accelerators requires the installation of the AXCL driver. We provide a convenient Linux script to complete this installation.
+
+Follow these steps for installation:
+
+1. Copy or download [this script](https://github.com/ivanshi1108/assets/releases/download/v0.16.2/user_installation.sh).
+2. Ensure it has execution permissions with `sudo chmod +x user_installation.sh`
+3. Run the script with `./user_installation.sh`
+
+#### Setup
+
+To set up Frigate, follow the default installation instructions, for example: `ghcr.io/blakeblackshear/frigate:stable`
+
+Next, grant Docker permissions to access your hardware by adding the following lines to your `docker-compose.yml` file:
+
+```yaml
+devices:
+  - /dev/axcl_host
+  - /dev/ax_mmb_dev
+  - /dev/msg_userdev
+```
+
+If you are using `docker run`, add this option to your command `--device /dev/axcl_host --device /dev/ax_mmb_dev --device /dev/msg_userdev`
+
+#### Configuration
+
+Finally, configure [hardware object detection](/configuration/object_detectors#axera) to complete the setup.
+</details>
+
 ## Docker
 
 Running through Docker with Docker Compose is the recommended install method.

docs/docs/troubleshooting/memory.md

@@ -0,0 +1,129 @@
+---
+id: memory
+title: Memory Troubleshooting
+---
+
+Frigate includes built-in memory profiling using [memray](https://bloomberg.github.io/memray/) to help diagnose memory issues. This feature allows you to profile specific Frigate modules to identify memory leaks, excessive allocations, or other memory-related problems.
+
+## Enabling Memory Profiling
+
+Memory profiling is controlled via the `FRIGATE_MEMRAY_MODULES` environment variable. Set it to a comma-separated list of module names you want to profile:
+
+```bash
+export FRIGATE_MEMRAY_MODULES="frigate.review_segment_manager,frigate.capture"
+```
+
+### Module Names
+
+Frigate processes are named using a module-based naming scheme. Common module names include:
+
+- `frigate.review_segment_manager` - Review segment processing
+- `frigate.recording_manager` - Recording management
+- `frigate.capture` - Camera capture processes (all cameras with this module name)
+- `frigate.process` - Camera processing/tracking (all cameras with this module name)
+- `frigate.output` - Output processing
+- `frigate.audio_manager` - Audio processing
+- `frigate.embeddings` - Embeddings processing
+
+You can also specify the full process name (including camera-specific identifiers) if you want to profile a specific camera:
+
+```bash
+export FRIGATE_MEMRAY_MODULES="frigate.capture:front_door"
+```
+
+When you specify a module name (e.g., `frigate.capture`), all processes with that module prefix will be profiled. For example, `frigate.capture` will profile all camera capture processes.
+
+## How It Works
+
+1. **Binary File Creation**: When profiling is enabled, memray creates a binary file (`.bin`) in `/config/memray_reports/` that is updated continuously in real-time as the process runs.
+
+2. **Automatic HTML Generation**: On normal process exit, Frigate automatically:
+
+   - Stops memray tracking
+   - Generates an HTML flamegraph report
+   - Saves it to `/config/memray_reports/<module_name>.html`
+
+3. **Crash Recovery**: If a process crashes (SIGKILL, segfault, etc.), the binary file is preserved with all data up to the crash point. You can manually generate the HTML report from the binary file.
+
+## Viewing Reports
+
+### Automatic Reports
+
+After a process exits normally, you'll find HTML reports in `/config/memray_reports/`. Open these files in a web browser to view interactive flamegraphs showing memory usage patterns.
+
+### Manual Report Generation
+
+If a process crashes or you want to generate a report from an existing binary file, you can manually create the HTML report:
+
+```bash
+memray flamegraph /config/memray_reports/<module_name>.bin
+```
+
+This will generate an HTML file that you can open in your browser.
+
+## Understanding the Reports
+
+Memray flamegraphs show:
+
+- **Memory allocations over time**: See where memory is being allocated in your code
+- **Call stacks**: Understand the full call chain leading to allocations
+- **Memory hotspots**: Identify functions or code paths that allocate the most memory
+- **Memory leaks**: Spot patterns where memory is allocated but not freed
+
+The interactive HTML reports allow you to:
+
+- Zoom into specific time ranges
+- Filter by function names
+- View detailed allocation information
+- Export data for further analysis
+
+## Best Practices
+
+1. **Profile During Issues**: Enable profiling when you're experiencing memory issues, not all the time, as it adds some overhead.
+
+2. **Profile Specific Modules**: Instead of profiling everything, focus on the modules you suspect are causing issues.
+
+3. **Let Processes Run**: Allow processes to run for a meaningful duration to capture representative memory usage patterns.
+
+4. **Check Binary Files**: If HTML reports aren't generated automatically (e.g., after a crash), check for `.bin` files in `/config/memray_reports/` and generate reports manually.
+
+5. **Compare Reports**: Generate reports at different times to compare memory usage patterns and identify trends.
+
+## Troubleshooting
+
+### No Reports Generated
+
+- Check that the environment variable is set correctly
+- Verify the module name matches exactly (case-sensitive)
+- Check logs for memray-related errors
+- Ensure `/config/memray_reports/` directory exists and is writable
+
+### Process Crashed Before Report Generation
+
+- Look for `.bin` files in `/config/memray_reports/`
+- Manually generate HTML reports using: `memray flamegraph <file>.bin`
+- The binary file contains all data up to the crash point
+
+### Reports Show No Data
+
+- Ensure the process ran long enough to generate meaningful data
+- Check that memray is properly installed (included by default in Frigate)
+- Verify the process actually started and ran (check process logs)
+
+## Example Usage
+
+```bash
+# Enable profiling for review and capture modules
+export FRIGATE_MEMRAY_MODULES="frigate.review_segment_manager,frigate.capture"
+
+# Start Frigate
+# ... let it run for a while ...
+
+# Check for reports
+ls -lh /config/memray_reports/
+
+# If a process crashed, manually generate report
+memray flamegraph /config/memray_reports/frigate_capture_front_door.bin
+```
+
+For more information about memray and interpreting reports, see the [official memray documentation](https://bloomberg.github.io/memray/).

docs/sidebars.ts

@@ -131,6 +131,7 @@ const sidebars: SidebarsConfig = {
       "troubleshooting/recordings",
       "troubleshooting/gpu",
       "troubleshooting/edgetpu",
+      "troubleshooting/memory",
     ],
     Development: [
       "development/contributing",

frigate/detectors/plugins/axengine.py

@@ -0,0 +1,87 @@
+import logging
+import os.path
+import re
+import urllib.request
+from typing import Literal
+
+import axengine as axe
+
+from frigate.const import MODEL_CACHE_DIR
+from frigate.detectors.detection_api import DetectionApi
+from frigate.detectors.detector_config import BaseDetectorConfig, ModelTypeEnum
+from frigate.util.model import post_process_yolo
+
+logger = logging.getLogger(__name__)
+
+DETECTOR_KEY = "axengine"
+
+supported_models = {
+    ModelTypeEnum.yologeneric: "frigate-yolov9-.*$",
+}
+
+model_cache_dir = os.path.join(MODEL_CACHE_DIR, "axengine_cache/")
+
+
+class AxengineDetectorConfig(BaseDetectorConfig):
+    type: Literal[DETECTOR_KEY]
+
+class Axengine(DetectionApi):
+    type_key = DETECTOR_KEY
+    def __init__(self, config: AxengineDetectorConfig):
+        logger.info("__init__ axengine")
+        super().__init__(config)
+        self.height = config.model.height
+        self.width = config.model.width
+        model_path = config.model.path or "frigate-yolov9-tiny"
+        model_props = self.parse_model_input(model_path)
+        self.session = axe.InferenceSession(model_props["path"])
+
+    def __del__(self):
+        pass
+
+    def parse_model_input(self, model_path):
+        model_props = {}
+        model_props["preset"] = True
+
+        model_matched = False
+
+        for model_type, pattern in supported_models.items():
+            if re.match(pattern, model_path):
+                model_matched = True
+                model_props["model_type"] = model_type
+
+        if model_matched:
+            model_props["filename"] = model_path + ".axmodel"
+            model_props["path"] = model_cache_dir + model_props["filename"]
+
+            if not os.path.isfile(model_props["path"]):
+                self.download_model(model_props["filename"])
+        else:
+            supported_models_str = ", ".join(
+                model[1:-1] for model in supported_models
+            )
+            raise Exception(
+                f"Model {model_path} is unsupported. Provide your own model or choose one of the following: {supported_models_str}"
+            )
+        return model_props
+
+    def download_model(self, filename):
+        if not os.path.isdir(model_cache_dir):
+            os.mkdir(model_cache_dir)
+
+        GITHUB_ENDPOINT = os.environ.get("GITHUB_ENDPOINT", "https://github.com")
+        urllib.request.urlretrieve(
+            f"{GITHUB_ENDPOINT}/ivanshi1108/assets/releases/download/v0.16.2/{filename}",
+            model_cache_dir + filename,
+        )
+
+    def detect_raw(self, tensor_input):
+        results = None
+        results = self.session.run(None, {"images": tensor_input})
+        if self.detector_config.model.model_type == ModelTypeEnum.yologeneric:
+            return post_process_yolo(results, self.width, self.height)
+        else:
+            raise ValueError(
+                f'Model type "{self.detector_config.model.model_type}" is currently not supported.'
+            )
+

frigate/util/process.py

@@ -1,7 +1,10 @@
+import atexit
 import faulthandler
 import logging
 import multiprocessing as mp
 import os
+import pathlib
+import subprocess
 import threading
 from logging.handlers import QueueHandler
 from multiprocessing.synchronize import Event as MpEvent
@@ -48,6 +51,7 @@ class FrigateProcess(BaseProcess):
 
     def before_start(self) -> None:
         self.__log_queue = frigate.log.log_listener.queue
+        self.__memray_tracker = None
 
     def pre_run_setup(self, logConfig: LoggerConfig | None = None) -> None:
         os.nice(self.priority)
@@ -64,3 +68,86 @@ class FrigateProcess(BaseProcess):
             frigate.log.apply_log_levels(
                 logConfig.default.value.upper(), logConfig.logs
             )
+
+        self._setup_memray()
+
+    def _setup_memray(self) -> None:
+        """Setup memray profiling if enabled via environment variable."""
+        memray_modules = os.environ.get("FRIGATE_MEMRAY_MODULES", "")
+
+        if not memray_modules:
+            return
+
+        # Extract module name from process name (e.g., "frigate.capture:camera" -> "frigate.capture")
+        process_name = self.name
+        module_name = (
+            process_name.split(":")[0] if ":" in process_name else process_name
+        )
+
+        enabled_modules = [m.strip() for m in memray_modules.split(",")]
+
+        if module_name not in enabled_modules and process_name not in enabled_modules:
+            return
+
+        try:
+            import memray
+
+            reports_dir = pathlib.Path("/config/memray_reports")
+            reports_dir.mkdir(parents=True, exist_ok=True)
+            safe_name = (
+                process_name.replace(":", "_").replace("/", "_").replace("\\", "_")
+            )
+
+            binary_file = reports_dir / f"{safe_name}.bin"
+
+            self.__memray_tracker = memray.Tracker(str(binary_file))
+            self.__memray_tracker.__enter__()
+
+            # Register cleanup handler to stop tracking and generate HTML report
+            # atexit runs on normal exits and most signal-based terminations (SIGTERM, SIGINT)
+            # For hard kills (SIGKILL) or segfaults, the binary file is preserved for manual generation
+            atexit.register(self._cleanup_memray, safe_name, binary_file)
+
+            self.logger.info(
+                f"Memray profiling enabled for module {module_name} (process: {self.name}). "
+                f"Binary file (updated continuously): {binary_file}. "
+                f"HTML report will be generated on exit: {reports_dir}/{safe_name}.html. "
+                f"If process crashes, manually generate with: memray flamegraph {binary_file}"
+            )
+        except Exception as e:
+            self.logger.error(f"Failed to setup memray profiling: {e}", exc_info=True)
+
+    def _cleanup_memray(self, safe_name: str, binary_file: pathlib.Path) -> None:
+        """Stop memray tracking and generate HTML report."""
+        if self.__memray_tracker is None:
+            return
+
+        try:
+            self.__memray_tracker.__exit__(None, None, None)
+            self.__memray_tracker = None
+
+            reports_dir = pathlib.Path("/config/memray_reports")
+            html_file = reports_dir / f"{safe_name}.html"
+
+            result = subprocess.run(
+                ["memray", "flamegraph", "--output", str(html_file), str(binary_file)],
+                capture_output=True,
+                text=True,
+                timeout=10,
+            )
+
+            if result.returncode == 0:
+                self.logger.info(f"Memray report generated: {html_file}")
+            else:
+                self.logger.error(
+                    f"Failed to generate memray report: {result.stderr}. "
+                    f"Binary file preserved at {binary_file} for manual generation."
+                )
+
+            # Keep the binary file for manual report generation if needed
+            # Users can run: memray flamegraph {binary_file}
+
+        except subprocess.TimeoutExpired:
+            self.logger.error("Memray report generation timed out")
+        except Exception as e:
+            self.logger.error(f"Failed to cleanup memray profiling: {e}", exc_info=True)