diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 89acd8a9b..f053abe3f 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,3 +1,385 @@ -- For Frigate NVR, never write strings in the frontend directly. Since the project uses `react-i18next`, use `t()` and write the English string in the relevant translations file in `web/public/locales/en`. -- Always conform new and refactored code to the existing coding style in the project. -- Always have a way to test your work and confirm your changes. When running backend tests, use `python3 -u -m unittest`. +# GitHub Copilot Instructions for Frigate NVR + +This document provides coding guidelines and best practices for contributing to Frigate NVR, a complete and local NVR designed for Home Assistant with AI object detection. + +## Project Overview + +Frigate NVR is a realtime object detection system for IP cameras that uses: + +- **Backend**: Python 3.13+ with FastAPI, OpenCV, TensorFlow/ONNX +- **Frontend**: React with TypeScript, Vite, TailwindCSS +- **Architecture**: Multiprocessing design with ZMQ and MQTT communication +- **Focus**: Minimal resource usage with maximum performance + +## Code Review Guidelines + +When reviewing code, do NOT comment on: + +- Missing imports - Static analysis tooling catches these +- Code formatting - Ruff (Python) and Prettier (TypeScript/React) handle formatting +- Minor style inconsistencies already enforced by linters + +## Python Backend Standards + +### Python Requirements + +- **Compatibility**: Python 3.13+ +- **Language Features**: Use modern Python features: + - Pattern matching + - Type hints (comprehensive typing preferred) + - f-strings (preferred over `%` or `.format()`) + - Dataclasses + - Async/await patterns + +### Code Quality Standards + +- **Formatting**: Ruff (configured in `pyproject.toml`) +- **Linting**: Ruff with rules defined in project config +- **Type Checking**: Use type hints consistently +- **Testing**: unittest framework - use `python3 -u -m unittest` to run tests +- **Language**: American English for all code, comments, and documentation + +### Logging Standards + +- **Logger Pattern**: Use module-level logger + + ```python + import logging + + logger = logging.getLogger(__name__) + ``` + +- **Format Guidelines**: + - No periods at end of log messages + - No sensitive data (keys, tokens, passwords) + - Use lazy logging: `logger.debug("Message with %s", variable)` +- **Log Levels**: + - `debug`: Development and troubleshooting information + - `info`: Important runtime events (startup, shutdown, state changes) + - `warning`: Recoverable issues that should be addressed + - `error`: Errors that affect functionality but don't crash the app + - `exception`: Use in except blocks to include traceback + +### Error Handling + +- **Exception Types**: Choose most specific exception available +- **Try/Catch Best Practices**: + - Only wrap code that can throw exceptions + - Keep try blocks minimal - process data after the try/except + - Avoid bare exceptions except in background tasks + + Bad pattern: + + ```python + try: + data = await device.get_data() # Can throw + # ❌ Don't process data inside try block + processed = data.get("value", 0) * 100 + result = processed + except DeviceError: + logger.error("Failed to get data") + ``` + + Good pattern: + + ```python + try: + data = await device.get_data() # Can throw + except DeviceError: + logger.error("Failed to get data") + return + + # ✅ Process data outside try block + processed = data.get("value", 0) * 100 + result = processed + ``` + +### Async Programming + +- **External I/O**: All external I/O operations must be async +- **Best Practices**: + - Avoid sleeping in loops - use `asyncio.sleep()` not `time.sleep()` + - Avoid awaiting in loops - use `asyncio.gather()` instead + - No blocking calls in async functions + - Use `asyncio.create_task()` for background operations +- **Thread Safety**: Use proper synchronization for shared state + +### Documentation Standards + +- **Module Docstrings**: Concise descriptions at top of files + ```python + """Utilities for motion detection and analysis.""" + ``` +- **Function Docstrings**: Required for public functions and methods + + ```python + async def process_frame(frame: ndarray, config: Config) -> Detection: + """Process a video frame for object detection. + + Args: + frame: The video frame as numpy array + config: Detection configuration + + Returns: + Detection results with bounding boxes + """ + ``` + +- **Comment Style**: + - Explain the "why" not just the "what" + - Keep lines under 88 characters when possible + - Use clear, descriptive comments + +### File Organization + +- **API Endpoints**: `frigate/api/` - FastAPI route handlers +- **Configuration**: `frigate/config/` - Configuration parsing and validation +- **Detectors**: `frigate/detectors/` - Object detection backends +- **Events**: `frigate/events/` - Event management and storage +- **Utilities**: `frigate/util/` - Shared utility functions + +## Frontend (React/TypeScript) Standards + +### Internationalization (i18n) + +- **CRITICAL**: Never write user-facing strings directly in components +- **Always use react-i18next**: Import and use the `t()` function + + ```tsx + import { useTranslation } from "react-i18next"; + + function MyComponent() { + const { t } = useTranslation(["views/live"]); + return
{t("camera_not_found")}
; + } + ``` + +- **Translation Files**: Add English strings to the appropriate json files in `web/public/locales/en` +- **Namespaces**: Organize translations by feature/view (e.g., `views/live`, `common`, `views/system`) + +### Code Quality + +- **Linting**: ESLint (see `web/.eslintrc.cjs`) +- **Formatting**: Prettier with Tailwind CSS plugin +- **Type Safety**: TypeScript strict mode enabled +- **Testing**: Vitest for unit tests + +### Component Patterns + +- **UI Components**: Use Radix UI primitives (in `web/src/components/ui/`) +- **Styling**: TailwindCSS with `cn()` utility for class merging +- **State Management**: React hooks (useState, useEffect, useCallback, useMemo) +- **Data Fetching**: Custom hooks with proper loading and error states + +### ESLint Rules + +Key rules enforced: + +- `react-hooks/rules-of-hooks`: error +- `react-hooks/exhaustive-deps`: error +- `no-console`: error (use proper logging or remove) +- `@typescript-eslint/no-explicit-any`: warn (always use proper types instead of `any`) +- Unused variables must be prefixed with `_` +- Comma dangles required for multiline objects/arrays + +### File Organization + +- **Pages**: `web/src/pages/` - Route components +- **Views**: `web/src/views/` - Complex view components +- **Components**: `web/src/components/` - Reusable components +- **Hooks**: `web/src/hooks/` - Custom React hooks +- **API**: `web/src/api/` - API client functions +- **Types**: `web/src/types/` - TypeScript type definitions + +## Testing Requirements + +### Backend Testing + +- **Framework**: Python unittest +- **Run Command**: `python3 -u -m unittest` +- **Location**: `frigate/test/` +- **Coverage**: Aim for comprehensive test coverage of core functionality +- **Pattern**: Use `TestCase` classes with descriptive test method names + ```python + class TestMotionDetection(unittest.TestCase): + def test_detects_motion_above_threshold(self): + # Test implementation + ``` + +### Test Best Practices + +- Always have a way to test your work and confirm your changes +- Write tests for bug fixes to prevent regressions +- Test edge cases and error conditions +- Mock external dependencies (cameras, APIs, hardware) +- Use fixtures for test data + +## Development Commands + +### Python Backend + +```bash +# Run all tests +python3 -u -m unittest + +# Run specific test file +python3 -u -m unittest frigate.test.test_ffmpeg_presets + +# Check formatting (Ruff) +ruff format --check frigate/ + +# Apply formatting +ruff format frigate/ + +# Run linter +ruff check frigate/ +``` + +### Frontend (from web/ directory) + +```bash +# Start dev server (AI agents should never run this directly unless asked) +npm run dev + +# Build for production +npm run build + +# Run linter +npm run lint + +# Fix linting issues +npm run lint:fix + +# Format code +npm run prettier:write +``` + +### Docker Development + +AI agents should never run these commands directly unless instructed. + +```bash +# Build local image +make local + +# Build debug image +make debug +``` + +## Common Patterns + +### API Endpoint Pattern + +```python +from fastapi import APIRouter, Request +from frigate.api.defs.tags import Tags + +router = APIRouter(tags=[Tags.Events]) + +@router.get("/events") +async def get_events(request: Request, limit: int = 100): + """Retrieve events from the database.""" + # Implementation +``` + +### Configuration Access + +```python +# Access Frigate configuration +config: FrigateConfig = request.app.frigate_config +camera_config = config.cameras["front_door"] +``` + +### Database Queries + +```python +from frigate.models import Event + +# Use Peewee ORM for database access +events = ( + Event.select() + .where(Event.camera == camera_name) + .order_by(Event.start_time.desc()) + .limit(limit) +) +``` + +## Common Anti-Patterns to Avoid + +### ❌ Avoid These + +```python +# Blocking operations in async functions +data = requests.get(url) # ❌ Use async HTTP client +time.sleep(5) # ❌ Use asyncio.sleep() + +# Hardcoded strings in React components +
Camera not found
# ❌ Use t("camera_not_found") + +# Missing error handling +data = await api.get_data() # ❌ No exception handling + +# Bare exceptions in regular code +try: + value = await sensor.read() +except Exception: # ❌ Too broad + logger.error("Failed") +``` + +### ✅ Use These Instead + +```python +# Async operations +import aiohttp +async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + data = await response.json() + +await asyncio.sleep(5) # ✅ Non-blocking + +# Translatable strings in React +const { t } = useTranslation(); +
{t("camera_not_found")}
# ✅ Translatable + +# Proper error handling +try: + data = await api.get_data() +except ApiException as err: + logger.error("API error: %s", err) + raise + +# Specific exceptions +try: + value = await sensor.read() +except SensorException as err: # ✅ Specific + logger.exception("Failed to read sensor") +``` + +## Project-Specific Conventions + +### Configuration Files + +- Main config: `config/config.yml` + +### Directory Structure + +- Backend code: `frigate/` +- Frontend code: `web/` +- Docker files: `docker/` +- Documentation: `docs/` +- Database migrations: `migrations/` + +### Code Style Conformance + +Always conform new and refactored code to the existing coding style in the project: + +- Follow established patterns in similar files +- Match indentation and formatting of surrounding code +- Use consistent naming conventions (snake_case for Python, camelCase for TypeScript) +- Maintain the same level of verbosity in comments and docstrings + +## Additional Resources + +- Documentation: https://docs.frigate.video +- Main Repository: https://github.com/blakeblackshear/frigate +- Home Assistant Integration: https://github.com/blakeblackshear/frigate-hass-integration diff --git a/docs/docs/frigate/hardware.md b/docs/docs/frigate/hardware.md index f7294042a..48ab2c647 100644 --- a/docs/docs/frigate/hardware.md +++ b/docs/docs/frigate/hardware.md @@ -167,7 +167,7 @@ Inference speeds vary greatly depending on the CPU or GPU used, some known examp | Intel N100 | ~ 15 ms | s-320: 30 ms | 320: ~ 25 ms | | Can only run one detector instance | | Intel N150 | ~ 15 ms | t-320: 16 ms s-320: 24 ms | | | | | Intel Iris XE | ~ 10 ms | t-320: 6 ms t-640: 14 ms s-320: 8 ms s-640: 16 ms | 320: ~ 10 ms 640: ~ 20 ms | 320-n: 33 ms | | -| Intel NPU | ~ 6 ms | s-320: 11 ms | 320: ~ 14 ms 640: ~ 34 ms | 320-n: 40 ms | | +| Intel NPU | ~ 6 ms | s-320: 11 ms s-640: 30 ms | 320: ~ 14 ms 640: ~ 34 ms | 320-n: 40 ms | | | Intel Arc A310 | ~ 5 ms | t-320: 7 ms t-640: 11 ms s-320: 8 ms s-640: 15 ms | 320: ~ 8 ms 640: ~ 14 ms | | | | Intel Arc A380 | ~ 6 ms | | 320: ~ 10 ms 640: ~ 22 ms | 336: 20 ms 448: 27 ms | | | Intel Arc A750 | ~ 4 ms | | 320: ~ 8 ms | | | diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index 7f0192912..821b449de 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -140,7 +140,12 @@ Each line represents a detection state, not necessarily unique individuals. Pare ) as f: f.write(context_prompt) - response = self._send(context_prompt, thumbnails) + json_schema = { + "name": "review_metadata", + "schema": ReviewMetadata.model_json_schema(), + "strict": True, + } + response = self._send(context_prompt, thumbnails, json_schema=json_schema) if debug_save and response: with open( @@ -152,6 +157,8 @@ Each line represents a detection state, not necessarily unique individuals. Pare f.write(response) if response: + # With JSON schema, response should already be valid JSON + # But keep regex cleanup as fallback for providers without schema support clean_json = re.sub( r"\n?```$", "", re.sub(r"^```[a-zA-Z0-9]*\n?", "", response) ) @@ -284,8 +291,16 @@ Guidelines: """Initialize the client.""" return None - def _send(self, prompt: str, images: list[bytes]) -> Optional[str]: - """Submit a request to the provider.""" + def _send( + self, prompt: str, images: list[bytes], json_schema: Optional[dict] = None + ) -> Optional[str]: + """Submit a request to the provider. + + Args: + prompt: The text prompt to send + images: List of image bytes to include + json_schema: Optional JSON schema for structured output (provider-specific support) + """ return None def get_context_size(self) -> int: diff --git a/frigate/genai/azure-openai.py b/frigate/genai/azure-openai.py index eb08f7786..9cc03fe75 100644 --- a/frigate/genai/azure-openai.py +++ b/frigate/genai/azure-openai.py @@ -41,29 +41,46 @@ class OpenAIClient(GenAIClient): azure_endpoint=azure_endpoint, ) - def _send(self, prompt: str, images: list[bytes]) -> Optional[str]: + def _send( + self, prompt: str, images: list[bytes], json_schema: Optional[dict] = None + ) -> Optional[str]: """Submit a request to Azure OpenAI.""" encoded_images = [base64.b64encode(image).decode("utf-8") for image in images] + + request_params = { + "model": self.genai_config.model, + "messages": [ + { + "role": "user", + "content": [{"type": "text", "text": prompt}] + + [ + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{image}", + "detail": "low", + }, + } + for image in encoded_images + ], + }, + ], + "timeout": self.timeout, + } + + if json_schema: + request_params["response_format"] = { + "type": "json_schema", + "json_schema": { + "name": json_schema.get("name", "response"), + "schema": json_schema.get("schema", {}), + "strict": json_schema.get("strict", True), + }, + } + try: result = self.provider.chat.completions.create( - model=self.genai_config.model, - messages=[ - { - "role": "user", - "content": [{"type": "text", "text": prompt}] - + [ - { - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{image}", - "detail": "low", - }, - } - for image in encoded_images - ], - }, - ], - timeout=self.timeout, + **request_params, **self.genai_config.runtime_options, ) except Exception as e: diff --git a/frigate/genai/gemini.py b/frigate/genai/gemini.py index b700c33a4..1212f15ad 100644 --- a/frigate/genai/gemini.py +++ b/frigate/genai/gemini.py @@ -41,7 +41,9 @@ class GeminiClient(GenAIClient): http_options=types.HttpOptions(**http_options_dict), ) - def _send(self, prompt: str, images: list[bytes]) -> Optional[str]: + def _send( + self, prompt: str, images: list[bytes], json_schema: Optional[dict] = None + ) -> Optional[str]: """Submit a request to Gemini.""" contents = [ types.Part.from_bytes(data=img, mime_type="image/jpeg") for img in images @@ -51,6 +53,12 @@ class GeminiClient(GenAIClient): generation_config_dict = {"candidate_count": 1} generation_config_dict.update(self.genai_config.runtime_options) + if json_schema and "schema" in json_schema: + generation_config_dict["response_mime_type"] = "application/json" + generation_config_dict["response_schema"] = types.Schema( + json_schema=json_schema["schema"] + ) + response = self.provider.models.generate_content( model=self.genai_config.model, contents=contents, diff --git a/frigate/genai/ollama.py b/frigate/genai/ollama.py index ab6d3c0b3..f798cbd19 100644 --- a/frigate/genai/ollama.py +++ b/frigate/genai/ollama.py @@ -50,7 +50,9 @@ class OllamaClient(GenAIClient): logger.warning("Error initializing Ollama: %s", str(e)) return None - def _send(self, prompt: str, images: list[bytes]) -> Optional[str]: + def _send( + self, prompt: str, images: list[bytes], json_schema: Optional[dict] = None + ) -> Optional[str]: """Submit a request to Ollama""" if self.provider is None: logger.warning( @@ -62,6 +64,10 @@ class OllamaClient(GenAIClient): **self.provider_options, **self.genai_config.runtime_options, } + + if json_schema and "schema" in json_schema: + ollama_options["format"] = json_schema["schema"] + result = self.provider.generate( self.genai_config.model, prompt, diff --git a/frigate/genai/openai.py b/frigate/genai/openai.py index 1fb0dd852..a5ee6455e 100644 --- a/frigate/genai/openai.py +++ b/frigate/genai/openai.py @@ -31,7 +31,9 @@ class OpenAIClient(GenAIClient): } return OpenAI(api_key=self.genai_config.api_key, **provider_opts) - def _send(self, prompt: str, images: list[bytes]) -> Optional[str]: + def _send( + self, prompt: str, images: list[bytes], json_schema: Optional[dict] = None + ) -> Optional[str]: """Submit a request to OpenAI.""" encoded_images = [base64.b64encode(image).decode("utf-8") for image in images] messages_content = [] @@ -51,16 +53,31 @@ class OpenAIClient(GenAIClient): "text": prompt, } ) + + request_params = { + "model": self.genai_config.model, + "messages": [ + { + "role": "user", + "content": messages_content, + }, + ], + "timeout": self.timeout, + } + + if json_schema: + request_params["response_format"] = { + "type": "json_schema", + "json_schema": { + "name": json_schema.get("name", "response"), + "schema": json_schema.get("schema", {}), + "strict": json_schema.get("strict", True), + }, + } + try: result = self.provider.chat.completions.create( - model=self.genai_config.model, - messages=[ - { - "role": "user", - "content": messages_content, - }, - ], - timeout=self.timeout, + **request_params, **self.genai_config.runtime_options, ) if ( diff --git a/web/src/components/card/AnimatedEventCard.tsx b/web/src/components/card/AnimatedEventCard.tsx index a67dd8305..c5da99aa2 100644 --- a/web/src/components/card/AnimatedEventCard.tsx +++ b/web/src/components/card/AnimatedEventCard.tsx @@ -12,7 +12,7 @@ import { useCameraPreviews } from "@/hooks/use-camera-previews"; import { baseUrl } from "@/api/baseUrl"; import { VideoPreview } from "../preview/ScrubbablePreview"; import { useApiHost } from "@/api"; -import { isDesktop, isSafari } from "react-device-detect"; +import { isSafari } from "react-device-detect"; import { useUserPersistence } from "@/hooks/use-user-persistence"; import { Skeleton } from "../ui/skeleton"; import { Button } from "../ui/button"; @@ -87,7 +87,6 @@ export function AnimatedEventCard({ }, [visibilityListener]); const [isLoaded, setIsLoaded] = useState(false); - const [isHovered, setIsHovered] = useState(false); // interaction @@ -134,31 +133,27 @@ export function AnimatedEventCard({
setIsHovered(true) : undefined} - onMouseLeave={isDesktop ? () => setIsHovered(false) : undefined} > - {isHovered && ( - - - - - {t("markAsReviewed")} - - )} + + + + + {t("markAsReviewed")} + {previews != undefined && alertVideosLoaded && (