add ffmpeg backend for linux compatibility
This commit is contained in:
TPP-G6
+143
-16
@@ -7,9 +7,11 @@ and Helmholtz-Hodge decomposition.
|
||||
"""
|
||||
|
||||
import os
|
||||
import platform
|
||||
import numpy as np
|
||||
import cv2
|
||||
import time
|
||||
import threading
|
||||
from enum import Enum, auto
|
||||
from typing import Tuple
|
||||
# from threading import Thread, Event
|
||||
@@ -21,6 +23,10 @@ from .HHD import nHHD
|
||||
from . import configs
|
||||
from .get_depth import Contact3D
|
||||
from .backend_availability import (check_cupy_available, check_opencl_available)
|
||||
try:
|
||||
import av # Optional: used for Linux camera capture path.
|
||||
except ImportError:
|
||||
av = None
|
||||
|
||||
class Sensor:
|
||||
"""
|
||||
@@ -128,6 +134,18 @@ class Sensor:
|
||||
self.__contact3d = Contact3D(config_contact=self.__config_contact, verbose=self.verbose)
|
||||
self.__hhd = None
|
||||
self.poisson_backend = poisson_backend
|
||||
self._use_pyav = False
|
||||
self._av_container = None
|
||||
self._av_stream = None
|
||||
self._av_decoder = None
|
||||
self.cap = None
|
||||
self._is_live_camera = isinstance(self.vid_src, int)
|
||||
self._capture_thread = None
|
||||
self._capture_running = False
|
||||
self._latest_frame = None
|
||||
self._latest_frame_seq = 0
|
||||
self._last_consumed_seq = -1
|
||||
self._latest_frame_lock = threading.Lock()
|
||||
|
||||
# Setup video capture
|
||||
self._open_video_source(override, box)
|
||||
@@ -157,7 +175,7 @@ class Sensor:
|
||||
# Simple direct read - buffer size of 1 should minimize frame accumulation
|
||||
# Timing measurement helps identify if capture or processing is the bottleneck
|
||||
t_capture_start = time.perf_counter()
|
||||
ret, frame = self.cap.read()
|
||||
ret, frame = self._read_frame()
|
||||
t_capture_end = time.perf_counter()
|
||||
self._last_capture_time = t_capture_end - t_capture_start
|
||||
|
||||
@@ -350,7 +368,20 @@ class Sensor:
|
||||
|
||||
def disconnect(self):
|
||||
"""Release resources and stop async threads."""
|
||||
if self.cap.isOpened():
|
||||
self._capture_running = False
|
||||
if self._capture_thread is not None:
|
||||
self._capture_thread.join(timeout=1.0)
|
||||
self._capture_thread = None
|
||||
|
||||
if self._use_pyav:
|
||||
if self._av_container is not None:
|
||||
self._av_container.close()
|
||||
self._av_container = None
|
||||
self._av_stream = None
|
||||
self._av_decoder = None
|
||||
if self.verbose:
|
||||
print("Sensor disconnected.")
|
||||
elif self.cap is not None and self.cap.isOpened():
|
||||
self.cap.release()
|
||||
if self.verbose:
|
||||
print("Sensor disconnected.")
|
||||
@@ -420,29 +451,57 @@ class Sensor:
|
||||
|
||||
def _open_video_source(self, override, box):
|
||||
"""Initialize video capture and calibration."""
|
||||
use_linux_camera_pyav = (
|
||||
platform.system() == "Linux"
|
||||
and isinstance(self.vid_src, int)
|
||||
and av is not None
|
||||
)
|
||||
|
||||
self.cap = cv2.VideoCapture(self.vid_src)
|
||||
# Set buffer size to 1 to reduce latency and avoid frame accumulation
|
||||
# Note: Not all backends support this, but it helps when available
|
||||
try:
|
||||
self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
|
||||
except:
|
||||
pass # Some backends don't support this property
|
||||
self.cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter.fourcc('M', 'J', 'P', 'G'))
|
||||
self.cap.set(cv2.CAP_PROP_FPS, self._config_fps)
|
||||
self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, self._config_resolution_raw[0])
|
||||
self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self._config_resolution_raw[1])
|
||||
if use_linux_camera_pyav:
|
||||
device = f"/dev/video{self.vid_src}"
|
||||
try:
|
||||
self._av_container = av.open(
|
||||
device,
|
||||
format="v4l2",
|
||||
options={
|
||||
"input_format": "mjpeg",
|
||||
"video_size": f"{self._config_resolution_raw[0]}x{self._config_resolution_raw[1]}",
|
||||
"framerate": str(self._config_fps),
|
||||
},
|
||||
)
|
||||
self._av_stream = self._av_container.streams.video[0]
|
||||
self._av_decoder = self._av_container.decode(self._av_stream)
|
||||
self._use_pyav = True
|
||||
if self.verbose:
|
||||
print(f"Using PyAV Linux camera backend: {device}")
|
||||
except Exception as exc:
|
||||
if self.verbose:
|
||||
print(f"PyAV camera open failed ({exc}), fallback to OpenCV backend.")
|
||||
self._use_pyav = False
|
||||
|
||||
if not self._use_pyav:
|
||||
self.cap = cv2.VideoCapture(self.vid_src)
|
||||
# Set buffer size to 1 to reduce latency and avoid frame accumulation
|
||||
# Note: Not all backends support this, but it helps when available
|
||||
try:
|
||||
self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
|
||||
except Exception:
|
||||
pass # Some backends don't support this property
|
||||
self.cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter.fourcc('M', 'J', 'P', 'G'))
|
||||
self.cap.set(cv2.CAP_PROP_FPS, self._config_fps)
|
||||
self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, self._config_resolution_raw[0])
|
||||
self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self._config_resolution_raw[1])
|
||||
|
||||
if self.verbose:
|
||||
print("Connecting to video source...")
|
||||
|
||||
if not self.cap.isOpened():
|
||||
if not self._is_source_open():
|
||||
raise Exception("Failed to open video source.")
|
||||
|
||||
# Wait for first valid frame
|
||||
frame = None
|
||||
while frame is None:
|
||||
ret, frame = self.cap.read()
|
||||
ret, frame = self._read_frame_raw()
|
||||
if not ret:
|
||||
time.sleep(0.1)
|
||||
|
||||
@@ -463,6 +522,10 @@ class Sensor:
|
||||
self.initial_img = cv2.flip(self.initial_img, 1)
|
||||
self.initial_img = self.initial_img.astype(np.uint8)
|
||||
|
||||
# For live cameras, start a latest-frame capture thread to avoid latency buildup.
|
||||
if self._is_live_camera:
|
||||
self._start_capture_thread()
|
||||
|
||||
|
||||
def _find_stitch_boxes(self, img_src, show=False):
|
||||
"""Detect vision tile boxes with camera stabilization."""
|
||||
@@ -470,7 +533,9 @@ class Sensor:
|
||||
start_time = cv2.getTickCount()
|
||||
|
||||
while n_boxes < self._config_units:
|
||||
ret, frame = self.cap.read()
|
||||
ret, frame = self._read_frame_raw()
|
||||
if not ret:
|
||||
continue
|
||||
v_mean = np.sum(cv2.mean(frame))
|
||||
|
||||
if abs(v_mean - v_mean0) < 5:
|
||||
@@ -496,6 +561,68 @@ class Sensor:
|
||||
self.stitcher.extract_tiles()
|
||||
return True
|
||||
|
||||
def _is_source_open(self):
|
||||
if self._use_pyav:
|
||||
return self._av_container is not None
|
||||
return self.cap is not None and self.cap.isOpened()
|
||||
|
||||
def _read_frame(self):
|
||||
if self._capture_running:
|
||||
# Startup race guard: wait briefly until capture thread publishes first frame.
|
||||
if self._latest_frame is None:
|
||||
deadline = time.perf_counter() + 1.0
|
||||
while time.perf_counter() < deadline:
|
||||
with self._latest_frame_lock:
|
||||
if self._latest_frame is not None:
|
||||
break
|
||||
time.sleep(0.001)
|
||||
|
||||
# Return latest frame immediately (do not wait for next camera tick).
|
||||
# This keeps end-to-end latency minimal; caller may receive repeated
|
||||
# frames if processing runs faster than camera FPS.
|
||||
with self._latest_frame_lock:
|
||||
frame = self._latest_frame
|
||||
seq = self._latest_frame_seq
|
||||
if frame is not None:
|
||||
self._last_consumed_seq = seq
|
||||
return True, frame.copy()
|
||||
return False, None
|
||||
return self._read_frame_raw()
|
||||
|
||||
def _read_frame_raw(self):
|
||||
if self._use_pyav:
|
||||
if self._av_decoder is None:
|
||||
return False, None
|
||||
try:
|
||||
frame = next(self._av_decoder)
|
||||
except StopIteration:
|
||||
return False, None
|
||||
except Exception:
|
||||
return False, None
|
||||
return True, frame.to_ndarray(format="bgr24")
|
||||
if self.cap is None:
|
||||
return False, None
|
||||
return self.cap.read()
|
||||
|
||||
def _start_capture_thread(self):
|
||||
if self._capture_thread is not None:
|
||||
return
|
||||
self._capture_running = True
|
||||
self._capture_thread = threading.Thread(target=self._capture_loop, daemon=True)
|
||||
self._capture_thread.start()
|
||||
if self.verbose:
|
||||
print("Latest-frame capture thread started.")
|
||||
|
||||
def _capture_loop(self):
|
||||
while self._capture_running:
|
||||
ret, frame = self._read_frame_raw()
|
||||
if not ret:
|
||||
time.sleep(0.001)
|
||||
continue
|
||||
with self._latest_frame_lock:
|
||||
self._latest_frame = frame
|
||||
self._latest_frame_seq += 1
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# Private - Utilities
|
||||
|
||||
Reference in New Issue
Block a user