add ffmpeg backend for linux compatibility

src/orisys/sensor.py

@@ -7,9 +7,11 @@ and Helmholtz-Hodge decomposition.
 """
 
 import os
+import platform
 import numpy as np
 import cv2
 import time
+import threading
 from enum import Enum, auto
 from typing import Tuple
 # from threading import Thread, Event
@@ -21,6 +23,10 @@ from .HHD import nHHD
 from . import configs 
 from .get_depth import Contact3D
 from .backend_availability import (check_cupy_available, check_opencl_available)
+try:
+    import av  # Optional: used for Linux camera capture path.
+except ImportError:
+    av = None
 
 class Sensor:
     """
@@ -128,6 +134,18 @@ class Sensor:
         self.__contact3d = Contact3D(config_contact=self.__config_contact, verbose=self.verbose)
         self.__hhd = None
         self.poisson_backend = poisson_backend
+        self._use_pyav = False
+        self._av_container = None
+        self._av_stream = None
+        self._av_decoder = None
+        self.cap = None
+        self._is_live_camera = isinstance(self.vid_src, int)
+        self._capture_thread = None
+        self._capture_running = False
+        self._latest_frame = None
+        self._latest_frame_seq = 0
+        self._last_consumed_seq = -1
+        self._latest_frame_lock = threading.Lock()
         
         # Setup video capture
         self._open_video_source(override, box)
@@ -157,7 +175,7 @@ class Sensor:
         # Simple direct read - buffer size of 1 should minimize frame accumulation
         # Timing measurement helps identify if capture or processing is the bottleneck
         t_capture_start = time.perf_counter()
-        ret, frame = self.cap.read()
+        ret, frame = self._read_frame()
         t_capture_end = time.perf_counter()
         self._last_capture_time = t_capture_end - t_capture_start
         
@@ -350,7 +368,20 @@ class Sensor:
 
     def disconnect(self):
         """Release resources and stop async threads."""
-        if self.cap.isOpened():
+        self._capture_running = False
+        if self._capture_thread is not None:
+            self._capture_thread.join(timeout=1.0)
+            self._capture_thread = None
+
+        if self._use_pyav:
+            if self._av_container is not None:
+                self._av_container.close()
+                self._av_container = None
+                self._av_stream = None
+                self._av_decoder = None
+                if self.verbose:
+                    print("Sensor disconnected.")
+        elif self.cap is not None and self.cap.isOpened():
             self.cap.release()
             if self.verbose:
                 print("Sensor disconnected.")
@@ -420,29 +451,57 @@ class Sensor:
 
     def _open_video_source(self, override, box):
         """Initialize video capture and calibration."""
+        use_linux_camera_pyav = (
+            platform.system() == "Linux"
+            and isinstance(self.vid_src, int)
+            and av is not None
+        )
 
-        self.cap = cv2.VideoCapture(self.vid_src)        
+        if use_linux_camera_pyav:
-        # Set buffer size to 1 to reduce latency and avoid frame accumulation
+            device = f"/dev/video{self.vid_src}"
-        # Note: Not all backends support this, but it helps when available
+            try:
-        try:
+                self._av_container = av.open(
-            self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
+                    device,
-        except:
+                    format="v4l2",
-            pass  # Some backends don't support this property
+                    options={
-        self.cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter.fourcc('M', 'J', 'P', 'G'))
+                        "input_format": "mjpeg",
-        self.cap.set(cv2.CAP_PROP_FPS, self._config_fps)
+                        "video_size": f"{self._config_resolution_raw[0]}x{self._config_resolution_raw[1]}",
-        self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, self._config_resolution_raw[0])
+                        "framerate": str(self._config_fps),
-        self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self._config_resolution_raw[1])
+                    },
+                )
+                self._av_stream = self._av_container.streams.video[0]
+                self._av_decoder = self._av_container.decode(self._av_stream)
+                self._use_pyav = True
+                if self.verbose:
+                    print(f"Using PyAV Linux camera backend: {device}")
+            except Exception as exc:
+                if self.verbose:
+                    print(f"PyAV camera open failed ({exc}), fallback to OpenCV backend.")
+                self._use_pyav = False
+
+        if not self._use_pyav:
+            self.cap = cv2.VideoCapture(self.vid_src)
+            # Set buffer size to 1 to reduce latency and avoid frame accumulation
+            # Note: Not all backends support this, but it helps when available
+            try:
+                self.cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
+            except Exception:
+                pass  # Some backends don't support this property
+            self.cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter.fourcc('M', 'J', 'P', 'G'))
+            self.cap.set(cv2.CAP_PROP_FPS, self._config_fps)
+            self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, self._config_resolution_raw[0])
+            self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self._config_resolution_raw[1])
         
         if self.verbose:
             print("Connecting to video source...")
         
-        if not self.cap.isOpened():
+        if not self._is_source_open():
             raise Exception("Failed to open video source.")
         
         # Wait for first valid frame
         frame = None
         while frame is None:
-            ret, frame = self.cap.read()
+            ret, frame = self._read_frame_raw()
             if not ret:
                 time.sleep(0.1)
 
@@ -463,6 +522,10 @@ class Sensor:
                 self.initial_img = cv2.flip(self.initial_img, 1)
             self.initial_img = self.initial_img.astype(np.uint8)
 
+        # For live cameras, start a latest-frame capture thread to avoid latency buildup.
+        if self._is_live_camera:
+            self._start_capture_thread()
+
 
     def _find_stitch_boxes(self, img_src, show=False):
         """Detect vision tile boxes with camera stabilization."""
@@ -470,7 +533,9 @@ class Sensor:
         start_time = cv2.getTickCount()
         
         while n_boxes < self._config_units:
-            ret, frame = self.cap.read()
+            ret, frame = self._read_frame_raw()
+            if not ret:
+                continue
             v_mean = np.sum(cv2.mean(frame))
             
             if abs(v_mean - v_mean0) < 5:
@@ -496,6 +561,68 @@ class Sensor:
         self.stitcher.extract_tiles()
         return True
 
+    def _is_source_open(self):
+        if self._use_pyav:
+            return self._av_container is not None
+        return self.cap is not None and self.cap.isOpened()
+
+    def _read_frame(self):
+        if self._capture_running:
+            # Startup race guard: wait briefly until capture thread publishes first frame.
+            if self._latest_frame is None:
+                deadline = time.perf_counter() + 1.0
+                while time.perf_counter() < deadline:
+                    with self._latest_frame_lock:
+                        if self._latest_frame is not None:
+                            break
+                    time.sleep(0.001)
+
+            # Return latest frame immediately (do not wait for next camera tick).
+            # This keeps end-to-end latency minimal; caller may receive repeated
+            # frames if processing runs faster than camera FPS.
+            with self._latest_frame_lock:
+                frame = self._latest_frame
+                seq = self._latest_frame_seq
+            if frame is not None:
+                self._last_consumed_seq = seq
+                return True, frame.copy()
+            return False, None
+        return self._read_frame_raw()
+
+    def _read_frame_raw(self):
+        if self._use_pyav:
+            if self._av_decoder is None:
+                return False, None
+            try:
+                frame = next(self._av_decoder)
+            except StopIteration:
+                return False, None
+            except Exception:
+                return False, None
+            return True, frame.to_ndarray(format="bgr24")
+        if self.cap is None:
+            return False, None
+        return self.cap.read()
+
+    def _start_capture_thread(self):
+        if self._capture_thread is not None:
+            return
+        self._capture_running = True
+        self._capture_thread = threading.Thread(target=self._capture_loop, daemon=True)
+        self._capture_thread.start()
+        if self.verbose:
+            print("Latest-frame capture thread started.")
+
+    def _capture_loop(self):
+        while self._capture_running:
+            ret, frame = self._read_frame_raw()
+            if not ret:
+                time.sleep(0.001)
+                continue
+            with self._latest_frame_lock:
+                self._latest_frame = frame
+                self._latest_frame_seq += 1
+
 
     # =========================================================================
     # Private - Utilities

test_cam.py

@@ -0,0 +1,75 @@
+import cv2
+import time
+
+# Toggle these to test whether exposure is limiting delivered FPS.
+USE_MANUAL_EXPOSURE = False
+MANUAL_EXPOSURE = -6  # Typical V4L2/OpenCV manual range is negative values.
+
+cap = cv2.VideoCapture(4, cv2.CAP_V4L2)
+cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter.fourcc('M', 'J', 'P', 'G'))
+cap.set(cv2.CAP_PROP_FPS, 30)
+cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1920)
+cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 1080)
+cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
+
+# For V4L2 in OpenCV: auto-exposure is usually 3, manual is usually 1.
+if USE_MANUAL_EXPOSURE:
+    ok_auto = cap.set(cv2.CAP_PROP_AUTO_EXPOSURE, 1)
+    ok_exp = cap.set(cv2.CAP_PROP_EXPOSURE, MANUAL_EXPOSURE)
+else:
+    ok_auto = cap.set(cv2.CAP_PROP_AUTO_EXPOSURE, 3)
+    ok_exp = True
+
+actual_fourcc = int(cap.get(cv2.CAP_PROP_FOURCC))
+print(f"Backend: {cap.getBackendName()}")
+print(
+    f"FOURCC: {chr(actual_fourcc & 0xFF)}"
+    f"{chr((actual_fourcc >> 8) & 0xFF)}"
+    f"{chr((actual_fourcc >> 16) & 0xFF)}"
+    f"{chr((actual_fourcc >> 24) & 0xFF)}"
+)
+print(f"Resolution: {int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))}x{int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))}")
+print(f"FPS: {cap.get(cv2.CAP_PROP_FPS)}")
+print(
+    f"Exposure mode: {'manual' if USE_MANUAL_EXPOSURE else 'auto'} "
+    f"(set_auto_ok={ok_auto}, set_exp_ok={ok_exp})"
+)
+print(f"AutoExposure(readback): {cap.get(cv2.CAP_PROP_AUTO_EXPOSURE)}")
+print(f"Exposure(readback): {cap.get(cv2.CAP_PROP_EXPOSURE)}")
+
+frame_count = 0
+start = time.perf_counter()
+while cap.isOpened():
+    t0 = time.perf_counter()
+    ok = cap.grab()
+    t1 = time.perf_counter()
+    if not ok:
+        break
+
+    ret, frame = cap.retrieve()
+    t2 = time.perf_counter()
+    if not ret:
+        continue
+
+    cv2.imshow("webcam", frame)
+    t3 = time.perf_counter()
+    key = cv2.waitKey(1) & 0xFF
+    t4 = time.perf_counter()
+
+    frame_count += 1
+    if frame_count % 30 == 0:
+        elapsed = t4 - start
+        print(
+            f"Frame {frame_count}: "
+            f"grab={1000*(t1-t0):.1f}ms, "
+            f"retrieve={1000*(t2-t1):.1f}ms, "
+            f"imshow={1000*(t3-t2):.1f}ms, "
+            f"waitKey={1000*(t4-t3):.1f}ms, "
+            f"avg_fps={frame_count/elapsed:.1f}"
+        )
+
+    if key == ord('q'):
+        break
+
+cap.release()
+cv2.destroyAllWindows()

test_ffmpeg.py

@@ -0,0 +1,46 @@
+import av
+import cv2
+import time
+
+DEVICE = "/dev/video4"
+TARGET_FPS = 30
+WIDTH = 1280
+HEIGHT = 720
+
+# Open v4l2 device with same style options as ffmpeg CLI
+container = av.open(
+    DEVICE,
+    format="v4l2",
+    options={
+        "input_format": "mjpeg",
+        "video_size": f"{WIDTH}x{HEIGHT}",
+        "framerate": str(TARGET_FPS),
+    },
+)
+
+stream = container.streams.video[0]
+frame_count = 0
+t_start = time.perf_counter()
+
+for frame in container.decode(stream):
+    t0 = time.perf_counter()
+    img = frame.to_ndarray(format="bgr24")  # OpenCV-compatible
+    t1 = time.perf_counter()
+
+    cv2.imshow("pyav_cam", img)
+    key = cv2.waitKey(1) & 0xFF
+    t2 = time.perf_counter()
+
+    frame_count += 1
+    if frame_count % 30 == 0:
+        elapsed = t2 - t_start
+        print(
+            f"Frame {frame_count}: convert={1000*(t1-t0):.1f}ms, "
+            f"imshow+waitKey={1000*(t2-t1):.1f}ms, avg_fps={frame_count/elapsed:.1f}"
+        )
+
+    if key == ord("q"):
+        break
+
+container.close()
+cv2.destroyAllWindows()