sw: add generic software device driver (broken JPEG decoding)

Makefile

@@ -23,6 +23,7 @@ USE_HW_H264 ?= 1
 USE_FFMPEG ?= $(shell pkg-config libavutil libavformat libavcodec && echo 1)
 USE_LIBCAMERA ?= $(shell pkg-config libcamera && echo 1)
 USE_RTSP ?= $(shell pkg-config live555 && echo 1)
+USE_LIBJPEG ?= $(shell pkg-config libjpeg && echo 1)
 USE_LIBDATACHANNEL ?= $(shell [ -e $(LIBDATACHANNEL_PATH)/CMakeLists.txt ] && echo 1)
 
 ifeq (1,$(DEBUG))
@@ -48,6 +49,11 @@ CFLAGS += -DUSE_RTSP $(shell pkg-config --cflags live555)
 LDLIBS += $(shell pkg-config --libs live555)
 endif
 
+ifeq (1,$(USE_LIBJPEG))
+CFLAGS += -DUSE_LIBJPEG $(shell pkg-config --cflags libjpeg)
+LDLIBS += $(shell pkg-config --libs libjpeg)
+endif
+
 ifeq (1,$(USE_LIBDATACHANNEL))
 CFLAGS += -DUSE_LIBDATACHANNEL
 CFLAGS += -I$(LIBDATACHANNEL_PATH)/include

device/buffer.h

@@ -28,6 +28,7 @@ typedef struct buffer_s {
     struct buffer_v4l2_s *v4l2;
     struct buffer_dummy_s *dummy;
     struct buffer_libcamera_s *libcamera;
+    struct buffer_sw_s *sw;
   };
 
   // State

device/buffer_list.c

@@ -139,10 +139,12 @@ int buffer_list_set_stream(buffer_list_t *buf_list, bool do_on)
     return 0;
   }
 
+  buf_list->streaming = do_on;
+
   if (buf_list->dev->hw->buffer_list_set_stream(buf_list, do_on) < 0) {
+    buf_list->streaming = !do_on;
     goto error;
   }
-  buf_list->streaming = do_on;
 
   if (do_on) {
     buf_list->last_enqueued_us = get_monotonic_time_us(NULL, NULL);

device/buffer_list.h

@@ -51,6 +51,7 @@ typedef struct buffer_list_s {
   union {
     struct buffer_list_v4l2_s *v4l2;
     struct buffer_list_dummy_s *dummy;
+    struct buffer_list_sw_s *sw;
     struct buffer_list_libcamera_s *libcamera;
   };
 

device/camera/camera_decoder.c

@@ -30,17 +30,26 @@ static unsigned decoder_formats[] =
 buffer_list_t *camera_configure_decoder(camera_t *camera, buffer_list_t *src_capture)
 {
   unsigned chosen_format = 0;
-  device_info_t *device = device_list_find_m2m_formats(camera->device_list, src_capture->fmt.format, decoder_formats, &chosen_format);
 
-  if (!device) {
+  if (!getenv("CAMERA_FORCE_SWDECODER")) {
+    device_info_t *device = device_list_find_m2m_formats(camera->device_list, src_capture->fmt.format, decoder_formats, &chosen_format);
+    if (device) {
+      camera->decoder = device_v4l2_open("DECODER", device->path);
+    }
+  }
+
+  if (!camera->decoder) {
+    camera->decoder = device_sw_open("SWDECODER");
+    chosen_format = V4L2_PIX_FMT_YUYV;
+  }
+
+  if (!camera->decoder) {
     LOG_INFO(camera, "Cannot find '%s' decoder", fourcc_to_string(src_capture->fmt.format).buf);
     return NULL;
   }
 
   device_video_force_key(camera->camera);
 
-  camera->decoder = device_v4l2_open("DECODER", device->path);
-
   buffer_list_t *decoder_output = device_open_buffer_list_output(
     camera->decoder, src_capture);
   buffer_list_t *decoder_capture = device_open_buffer_list_capture2(

device/device.h

@@ -56,6 +56,7 @@ typedef struct device_s {
     struct device_v4l2_s *v4l2;
     struct device_dummy_s *dummy;
     struct device_libcamera_s *libcamera;
+    struct device_sw_t *sw;
   };
 
   bool paused;
@@ -123,3 +124,4 @@ int device_capture_enqueued(device_t *dev, int *max);
 device_t *device_v4l2_open(const char *name, const char *path);
 device_t *device_libcamera_open(const char *name, const char *path);
 device_t *device_dummy_open(const char *name, const char *path);
+device_t *device_sw_open(const char *name);

device/links.c

@@ -360,8 +360,14 @@ static int links_step(link_t *all_links, bool force_active, int timeout_now_ms,
       buf_list->dev->paused);
 
     if (pool.fds[i].revents & POLLIN) {
-      if (links_enqueue_from_capture_list(capture_list, link) < 0) {
-        return -1;
+      if (capture_list) {
+        if (links_enqueue_from_capture_list(capture_list, link) < 0) {
+          return -1;
+        }
+      } else if (output_list) {
+        if (links_dequeue_from_output_list(output_list) < 0) {
+          return -1;
+        }
       }
     }
 

device/sw/buffer.c

@@ -0,0 +1,92 @@
+#include "sw.h"
+#include "device/buffer.h"
+#include "device/buffer_list.h"
+#include "util/opts/log.h"
+#include "util/opts/fourcc.h"
+
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+// Taken from https://github.com/raspberrypi/linux/blob/bb63dc31e48948bc2649357758c7a152210109c4/drivers/staging/vc04_services/bcm2835-codec/bcm2835-v4l2-codec.c#L148
+#define DEF_COMP_BUF_SIZE_JPEG (4096 << 10)
+
+int sw_buffer_open(buffer_t *buf)
+{
+  buf->sw = calloc(1, sizeof(buffer_sw_t));
+
+  switch (buf->buf_list->fmt.format) {
+  case V4L2_PIX_FMT_MJPEG:
+    buf->length = DEF_COMP_BUF_SIZE_JPEG;
+    break;
+
+  case V4L2_PIX_FMT_YUYV:
+    buf->length = buf->buf_list->fmt.width * buf->buf_list->fmt.height * 2;
+    break;
+
+  default:
+    LOG_INFO(buf->buf_list, "The format is not supported '%s'",
+      fourcc_to_string(buf->buf_list->fmt.format).buf);
+    return -1;
+  }
+
+  if (buf->buf_list->do_mmap) {
+    buf->start = malloc(buf->length);
+    if (!buf->start) {
+      LOG_INFO(buf, "Failed to allocate %lu bytes for '%s'",
+        buf->length,
+        fourcc_to_string(buf->buf_list->fmt.format).buf);
+      return -1;
+    }
+  }
+
+  buf->used = 0;
+  return 0;
+}
+
+void sw_buffer_close(buffer_t *buf)
+{
+  if (!buf->buf_list->do_mmap) {
+    free(buf->start);
+  }
+  free(buf->sw);
+}
+
+int sw_buffer_enqueue(buffer_t *buf, const char *who)
+{
+  unsigned index = buf->index;
+  if (write(buf->buf_list->sw->send_fds[1], &index, sizeof(index)) != sizeof(index)) {
+    return -1;
+  }
+  return 0;
+}
+
+int sw_buffer_list_dequeue(buffer_list_t *buf_list, buffer_t **bufp)
+{
+  unsigned index = 0;
+  int n = read(buf_list->sw->recv_fds[0], &index, sizeof(index));
+  if (n != sizeof(index)) {
+    LOG_INFO(buf_list, "Received invalid result from `read`: %d", n);
+    return -1;
+  }
+
+  if (index >= (unsigned)buf_list->nbufs) {
+    LOG_INFO(buf_list, "Received invalid index from `read`: %d >= %d", index, buf_list->nbufs);
+    return -1;
+  }
+
+  *bufp = buf_list->bufs[index];
+  return 0;
+}
+
+int sw_buffer_list_pollfd(buffer_list_t *buf_list, struct pollfd *pollfd, bool can_dequeue)
+{
+  int count_enqueued = buffer_list_count_enqueued(buf_list);
+  pollfd->fd = buf_list->sw->recv_fds[0]; // write end
+  pollfd->events = POLLHUP;
+  if (can_dequeue && count_enqueued > 0) {
+    pollfd->events |= POLLIN;
+  }
+  pollfd->revents = 0;
+  return 0;
+}

device/sw/buffer_list.c

@@ -0,0 +1,80 @@
+#include "sw.h"
+#include "device/buffer_list.h"
+#include "device/device.h"
+#include "util/opts/log.h"
+#include "util/opts/fourcc.h"
+
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+int sw_buffer_list_open(buffer_list_t *buf_list)
+{
+  buf_list->do_mmap = true;
+
+  if (buf_list->do_capture) {
+    switch (buf_list->fmt.format) {
+    case V4L2_PIX_FMT_YUYV:
+      break;
+
+    default:
+      LOG_INFO(buf_list, "The format is not supported '%s'",
+        fourcc_to_string(buf_list->fmt.format).buf);
+      return -1;
+    }
+  } else {
+    switch (buf_list->fmt.format) {
+#ifdef USE_LIBJPEG
+    case V4L2_PIX_FMT_JPEG:
+    case V4L2_PIX_FMT_MJPEG:
+      break;
+#endif // USE_LIBJPEG
+
+    default:
+      LOG_INFO(buf_list, "The format is not supported '%s'",
+        fourcc_to_string(buf_list->fmt.format).buf);
+      return -1;
+    }
+  }
+
+  buf_list->sw = calloc(1, sizeof(buffer_list_sw_t));
+  buf_list->sw->send_fds[0] = -1;
+  buf_list->sw->send_fds[1] = -1;
+  buf_list->sw->recv_fds[0] = -1;
+  buf_list->sw->recv_fds[1] = -1;
+
+  if (pipe2(buf_list->sw->send_fds, O_DIRECT|O_CLOEXEC) < 0) {
+    LOG_INFO(buf_list, "Cannot open `pipe2`.");
+    return -1;
+  }
+
+  if (pipe2(buf_list->sw->recv_fds, O_DIRECT|O_CLOEXEC) < 0) {
+    LOG_INFO(buf_list, "Cannot open `pipe2`.");
+    close(buf_list->sw->send_fds[0]);
+    close(buf_list->sw->send_fds[1]);
+    return -1;
+  }
+
+  return buf_list->fmt.nbufs;
+}
+
+void sw_buffer_list_close(buffer_list_t *buf_list)
+{
+  if (buf_list->sw) {
+    close(buf_list->sw->send_fds[0]);
+    close(buf_list->sw->send_fds[1]);
+    close(buf_list->sw->recv_fds[0]);
+    close(buf_list->sw->recv_fds[1]);
+    free(buf_list->sw->data);
+  }
+
+  free(buf_list->sw);
+}
+
+int sw_buffer_list_set_stream(buffer_list_t *buf_list, bool do_on)
+{
+  if (do_on && !buf_list->do_capture) {
+    pthread_create(&buf_list->sw->thread, NULL, sw_device_thread, buf_list);
+  }
+  return 0;
+}

device/sw/device.c

@@ -0,0 +1,16 @@
+#include "sw.h"
+#include "device/device.h"
+
+#include <stdlib.h>
+
+int sw_device_open(device_t *dev)
+{
+  dev->opts.allow_dma = false;
+  dev->sw = calloc(1, sizeof(device_sw_t));
+  return 0;
+}
+
+void sw_device_close(device_t *dev)
+{
+  free(dev->sw);
+}

device/sw/jpeg.c

@@ -0,0 +1,73 @@
+#include "sw.h"
+
+#include "device/device.h"
+#include "device/buffer.h"
+#include "device/buffer_list.h"
+#include "util/opts/log.h"
+#include "util/opts/fourcc.h"
+
+#ifdef USE_LIBJPEG
+#include <jpeglib.h>
+
+bool sw_device_process_jpeg_capture_buf(buffer_t *output_buf, buffer_t *capture_buf)
+{
+  struct jpeg_decompress_struct cinfo;
+  struct jpeg_error_mgr jerr;
+  int ret;
+
+  LOG_VERBOSE(output_buf, "JPEG parsing: %ld", output_buf->used);
+
+  cinfo.err = jpeg_std_error(&jerr);
+  jpeg_create_decompress(&cinfo);
+  jpeg_mem_src(&cinfo, output_buf->start, output_buf->used);
+
+  ret = jpeg_read_header(&cinfo, TRUE);
+  if (ret != JPEG_HEADER_OK) {
+    LOG_VERBOSE(output_buf, "JPEG invalid header: %d", ret);
+    goto error;
+  }
+
+  if (capture_buf->buf_list->fmt.width != cinfo.image_width &&
+    capture_buf->buf_list->fmt.height != cinfo.image_height) {
+    LOG_VERBOSE(output_buf, "JPEG invalid image size: %dx%d, expected %dx%d",
+      cinfo.image_width, cinfo.image_height,
+      capture_buf->buf_list->fmt.width, capture_buf->buf_list->fmt.height);
+    goto error;
+  }
+
+  cinfo.scale_num = 1;
+  cinfo.scale_denom = 1;
+  cinfo.dct_method = JDCT_FASTEST; // JDCT_DEFAULT
+  cinfo.out_color_space = JCS_YCbCr;
+
+  if (!jpeg_start_decompress(&cinfo)) {
+    LOG_VERBOSE(output_buf, "Failed to start decompress");
+    goto error;
+  }
+
+  capture_buf->length = 0;
+
+  while (cinfo.output_scanline < cinfo.output_height) {
+    JSAMPROW row = (unsigned char *)capture_buf->start + capture_buf->length;
+    ret = jpeg_read_scanlines(&cinfo, &row, 1);
+    if (!ret) {
+      LOG_VERBOSE(output_buf, "Failed to read scanline");
+      goto error;
+    }
+    capture_buf->length += ret * cinfo.output_width * 2;
+  }
+
+  jpeg_finish_decompress(&cinfo);
+  jpeg_destroy_decompress(&cinfo);
+  return true;
+
+error:
+  jpeg_destroy_decompress(&cinfo);
+  return false;
+}
+#else // USE_LIBJPEG
+bool sw_device_process_jpeg_capture_buf(buffer_t *output_buf, buffer_t *capture_buf)
+{
+  return false;
+}
+#endif // USE_LIBJPEG

device/sw/sw.c

@@ -0,0 +1,23 @@
+#include "sw.h"
+
+#include "device/device.h"
+
+device_hw_t sw_device_hw = {
+  .device_open = sw_device_open,
+  .device_close = sw_device_close,
+
+  .buffer_open = sw_buffer_open,
+  .buffer_close = sw_buffer_close,
+  .buffer_enqueue = sw_buffer_enqueue,
+
+  .buffer_list_dequeue = sw_buffer_list_dequeue,
+  .buffer_list_pollfd = sw_buffer_list_pollfd,
+  .buffer_list_open = sw_buffer_list_open,
+  .buffer_list_close = sw_buffer_list_close,
+  .buffer_list_set_stream = sw_buffer_list_set_stream
+};
+
+device_t *device_sw_open(const char *name)
+{
+  return device_open(name, "/dev/null", &sw_device_hw);
+}