From 9f715c08c1737a0246e14b4fe6318758321b807a Mon Sep 17 00:00:00 2001 From: Christopher Nadler <147471517+cnadler86@users.noreply.github.com> Date: Tue, 29 Oct 2024 21:07:55 +0100 Subject: [PATCH] Img converter to develop (#10) Add image conversion feature to capture method (experimental feature) --- .../{ESP32cam.html => CameraSettings.html} | 14 ++- examples/CameraSettings.py | 9 +- examples/{WebCam.py => SimpleWebCam.py} | 0 examples/benchmark.py | 98 ++++++++++++++++ src/modcamera.c | 111 ++++++++++++++---- src/modcamera.h | 7 +- src/modcamera_api.c | 29 ++++- 7 files changed, 231 insertions(+), 37 deletions(-) rename examples/{ESP32cam.html => CameraSettings.html} (95%) rename examples/{WebCam.py => SimpleWebCam.py} (100%) create mode 100644 examples/benchmark.py diff --git a/examples/ESP32cam.html b/examples/CameraSettings.html similarity index 95% rename from examples/ESP32cam.html rename to examples/CameraSettings.html index 5e03966..27e7447 100644 --- a/examples/ESP32cam.html +++ b/examples/CameraSettings.html @@ -140,6 +140,16 @@ .catch(error => { console.error('Error fetching sensor name:', error); }); + + fetch('/get_pixel_format') + .then(response => response.text()) + .then(pixelFormat => { + const showJpegQuality = (pixelFormat === '4'); + document.getElementById('quality').parentElement.classList.toggle('hidden', !showJpegQuality); + }) + .catch(error => { + console.error('Error fetching pixel format:', error); + }); } document.addEventListener("DOMContentLoaded", () => { @@ -151,7 +161,7 @@
-

ESP32 Camera Stream

+

Micropython Camera Stream

@@ -299,7 +309,7 @@

ESP32 Camera Stream

- + Loading stream...
diff --git a/examples/CameraSettings.py b/examples/CameraSettings.py index 2ce802f..c8f61fd 100644 --- a/examples/CameraSettings.py +++ b/examples/CameraSettings.py @@ -17,12 +17,14 @@ print(f'Connected! IP: {station.ifconfig()[0]}. Open this IP in your browser') -with open("ESP32cam.html", 'r') as file: +with open("CameraSettings.html", 'r') as file: html = file.read() async def stream_camera(writer): try: cam.init() + if not cam.get_bmp_out() and cam.get_pixel_format() != PixelFormat.JPEG: + cam.set_bmp_out(True) writer.write(b'HTTP/1.1 200 OK\r\nContent-Type: multipart/x-mixed-replace; boundary=frame\r\n\r\n') await writer.drain() @@ -30,7 +32,10 @@ async def stream_camera(writer): while True: frame = cam.capture() if frame: - writer.write(b'--frame\r\nContent-Type: image/jpeg\r\n\r\n') + if cam.get_pixel_format() == PixelFormat.JPEG: + writer.write(b'--frame\r\nContent-Type: image/jpeg\r\n\r\n') + else: + writer.write(b'--frame\r\nContent-Type: image/bmp\r\n\r\n') writer.write(frame) await writer.drain() diff --git a/examples/WebCam.py b/examples/SimpleWebCam.py similarity index 100% rename from examples/WebCam.py rename to examples/SimpleWebCam.py diff --git a/examples/benchmark.py b/examples/benchmark.py new file mode 100644 index 0000000..f038b67 --- /dev/null +++ b/examples/benchmark.py @@ -0,0 +1,98 @@ +from camera import Camera, FrameSize, PixelFormat +import time +import gc +import os +gc.enable() + +def measure_fps(duration=2): + start_time = time.time() + while time.time() - start_time < 0.5: + cam.capture() + + start_time = time.time() + frame_count = 0 + + while time.time() - start_time < duration: + cam.capture() + frame_count += 1 + + end_time = time.time() + fps = frame_count / (end_time - start_time) + return fps + +def print_summary_table(results, cam): + (_,_,_,_,target) = os.uname() + print(f"\nBenchmark {target} with {cam.get_sensor_name()}, fb_count: {cam.get_fb_count()}, GrabMode: {cam.get_grab_mode()}:") + + pixel_formats = list(results.keys()) + print(f"{'Frame Size':<15}", end="") + for p in pixel_formats: + print(f"{p:<15}", end="") + print() + + frame_size_names = {getattr(FrameSize, f): f for f in dir(FrameSize) if not f.startswith('_')} + frame_sizes = list(next(iter(results.values())).keys()) + + for f in frame_sizes: + frame_size_name = frame_size_names.get(f, str(f)) + print(f"{frame_size_name:<15}", end="") + for p in pixel_formats: + fps = results[p].get(f, "N/A") + print(f"{fps:<15}", end="") + print() + +if __name__ == "__main__": + cam = Camera() + results = {} + + try: + for p in dir(PixelFormat): + if not p.startswith('_'): + p_value = getattr(PixelFormat, p) + if p_value == PixelFormat.RGB888 and cam.get_sensor_name() == "OV2640": + continue + try: + cam.reconfigure(pixel_format=p_value) + results[p] = {} + except Exception as e: + print('ERR:', e) + continue + + for f in dir(FrameSize): + if not f.startswith('_'): + f_value = getattr(FrameSize, f) + if f_value > cam.get_max_frame_size(): + continue + gc.collect() + print('Set', p, f, ':') + + try: + cam.reconfigure(frame_size=f_value) + time.sleep_ms(10) + img = cam.capture() + + if img: + print('---> Image size:', len(img)) + fps = measure_fps(2) + print(f"---> FPS: {fps}") + results[p][f_value] = fps # FPS in Dictionary speichern + else: + print('No image captured') + results[p][f_value] = 'No image' + + print(f"---> Free Memory: {gc.mem_free()}") + except Exception as e: + print('ERR:', e) + results[p][f_value] = 'ERR' + finally: + time.sleep_ms(250) + gc.collect() + print('') + + except KeyboardInterrupt: + print("\nScript interrupted by user.") + + finally: + cam.deinit() + print_summary_table(results) # Tabelle am Ende ausgeben + diff --git a/src/modcamera.c b/src/modcamera.c index e90c3b2..13897b1 100644 --- a/src/modcamera.c +++ b/src/modcamera.c @@ -28,6 +28,7 @@ #include "modcamera.h" #include "esp_err.h" #include "esp_log.h" +#include "img_converters.h" #define TAG "ESP32_MPY_CAMERA" @@ -42,7 +43,7 @@ #endif // Supporting functions -void raise_micropython_error_from_esp_err(esp_err_t err) { +static void raise_micropython_error_from_esp_err(esp_err_t err) { switch (err) { case ESP_OK: return; @@ -72,8 +73,8 @@ void raise_micropython_error_from_esp_err(esp_err_t err) { break; default: - mp_raise_msg_varg(&mp_type_RuntimeError, MP_ERROR_TEXT("Unknown error 0x%04x"), err); - // mp_raise_msg_varg(&mp_type_RuntimeError, MP_ERROR_TEXT("Unknown error")); + // mp_raise_msg_varg(&mp_type_RuntimeError, MP_ERROR_TEXT("Unknown error 0x%04x"), err); + mp_raise_msg_varg(&mp_type_RuntimeError, MP_ERROR_TEXT("Unknown error")); break; } } @@ -85,7 +86,7 @@ static int map(int value, int fromLow, int fromHigh, int toLow, int toHigh) { return (int)((int32_t)(value - fromLow) * (toHigh - toLow) / (fromHigh - fromLow) + toLow); } -static int get_mapped_jpeg_quality(int8_t quality) { +static inline int get_mapped_jpeg_quality(int8_t quality) { return map(quality, 0, 100, 63, 0); } @@ -246,8 +247,7 @@ void mp_camera_hal_reconfigure(mp_camera_obj_t *self, mp_camera_framesize_t fram } } -mp_obj_t mp_camera_hal_capture(mp_camera_obj_t *self, int timeout_ms) { - // Timeout not used at the moment +mp_obj_t mp_camera_hal_capture(mp_camera_obj_t *self, int8_t out_format) { if (!self->initialized) { mp_raise_msg(&mp_type_OSError, MP_ERROR_TEXT("Failed to capture image: Camera not initialized")); } @@ -255,28 +255,82 @@ mp_obj_t mp_camera_hal_capture(mp_camera_obj_t *self, int timeout_ms) { esp_camera_fb_return(self->captured_buffer); self->captured_buffer = NULL; } + + static size_t out_len = 0; + static uint8_t *out_buf = NULL; + if (out_len>0 || out_buf) { + free(out_buf); + out_len = 0; + out_buf = NULL; + } + ESP_LOGI(TAG, "Capturing image"); self->captured_buffer = esp_camera_fb_get(); - if (self->captured_buffer) { - if (self->camera_config.pixel_format == PIXFORMAT_JPEG) { - ESP_LOGI(TAG, "Captured image in JPEG format"); - return mp_obj_new_memoryview('b', self->captured_buffer->len, self->captured_buffer->buf); - } else { - ESP_LOGI(TAG, "Captured image in raw format"); - return mp_obj_new_memoryview('b', self->captured_buffer->len, self->captured_buffer->buf); - // TODO: Stub at the moment in order to return raw data, but it sould be implemented to return a Bitmap, see following circuitpython example: - // - // int width = common_hal_espcamera_camera_get_width(self); - // int height = common_hal_espcamera_camera_get_height(self); - // displayio_bitmap_t *bitmap = m_new_obj(displayio_bitmap_t); - // bitmap->base.type = &displayio_bitmap_type; - // common_hal_displayio_bitmap_construct_from_buffer(bitmap, width, height, (format == PIXFORMAT_RGB565) ? 16 : 8, (uint32_t *)(void *)result->buf, true); - // return bitmap; + if (!self->captured_buffer) { + ESP_LOGE(TAG, "Failed to capture image"); + return mp_const_none; + } + + if (out_format >= 0 && (mp_camera_pixformat_t)out_format != self->camera_config.pixel_format) { + switch (out_format) { + case PIXFORMAT_JPEG: + if (frame2jpg(self->captured_buffer, self->camera_config.jpeg_quality, &out_buf, &out_len)) { + esp_camera_fb_return(self->captured_buffer); + mp_obj_t result = mp_obj_new_memoryview('b', out_len, out_buf); + return result; + } else { + return mp_const_none; + } + + case PIXFORMAT_RGB888: + out_len = self->captured_buffer->width * self->captured_buffer->height * 3; + out_buf = (uint8_t *)malloc(out_len); + if (!out_buf) { + ESP_LOGE(TAG, "out_buf malloc failed"); + return mp_const_none; + } + if (fmt2rgb888(self->captured_buffer->buf, self->captured_buffer->len, self->captured_buffer->format, out_buf)) { + esp_camera_fb_return(self->captured_buffer); + mp_obj_t result = mp_obj_new_memoryview('b', out_len, out_buf); + return result; + } else { + return mp_const_none; + } + + case PIXFORMAT_RGB565: + if(self->camera_config.pixel_format == PIXFORMAT_JPEG){ + if (jpg2rgb565(self->captured_buffer->buf, self->captured_buffer->len, out_buf, JPG_SCALE_NONE)) { + esp_camera_fb_return(self->captured_buffer); + mp_obj_t result = mp_obj_new_memoryview('b', out_len, out_buf); + return result; + } else { + return mp_const_none; + } + } else { + mp_raise_msg(&mp_type_OSError, MP_ERROR_TEXT("Can only convert JPEG to RGB565")); + return mp_const_none; + } + + default: + mp_raise_msg(&mp_type_OSError, MP_ERROR_TEXT("Unsupported pixel format for conversion")); + return mp_const_none; + } + } + + if (self->bmp_out == false) { + ESP_LOGI(TAG, "Returning imgae without conversion"); + return mp_obj_new_memoryview('b', self->captured_buffer->len, self->captured_buffer->buf); } else { - esp_camera_fb_return(self->captured_buffer); - self->captured_buffer = NULL; - return mp_const_none; + ESP_LOGI(TAG, "Returning image as bitmap"); + if (frame2bmp(self->captured_buffer, &out_buf, &out_len)) { + esp_camera_fb_return(self->captured_buffer); + mp_obj_t result = mp_obj_new_memoryview('b', out_len, out_buf); + return result; + } else { + mp_raise_msg(&mp_type_OSError, MP_ERROR_TEXT("Failed to convert image to BMP")); + return mp_const_none; + } } } @@ -289,6 +343,7 @@ const mp_rom_map_elem_t mp_camera_hal_pixel_format_table[] = { { MP_ROM_QSTR(MP_QSTR_YUV422), MP_ROM_INT(PIXFORMAT_YUV422) }, { MP_ROM_QSTR(MP_QSTR_GRAYSCALE), MP_ROM_INT(PIXFORMAT_GRAYSCALE) }, { MP_ROM_QSTR(MP_QSTR_RGB565), MP_ROM_INT(PIXFORMAT_RGB565) }, + { MP_ROM_QSTR(MP_QSTR_RGB888), MP_ROM_INT(PIXFORMAT_RGB888) }, }; const mp_rom_map_elem_t mp_camera_hal_frame_size_table[] = { @@ -418,16 +473,24 @@ void mp_camera_hal_set_frame_size(mp_camera_obj_t * self, framesize_t value) { if (!self->initialized) { mp_raise_ValueError(MP_ERROR_TEXT("Camera not initialized")); } + sensor_t *sensor = esp_camera_sensor_get(); if (!sensor->set_framesize) { mp_raise_ValueError(MP_ERROR_TEXT("No attribute frame_size")); } + + if (self->captured_buffer) { + esp_camera_return_all(); + self->captured_buffer = NULL; + } + if (sensor->set_framesize(sensor, value) < 0) { mp_raise_ValueError(MP_ERROR_TEXT("Invalid setting for frame_size")); } else { self->camera_config.frame_size = value; } } + int mp_camera_hal_get_quality(mp_camera_obj_t * self) { if (!self->initialized) { mp_raise_ValueError(MP_ERROR_TEXT("Camera not initialized")); diff --git a/src/modcamera.h b/src/modcamera.h index a78d25a..1e1305b 100644 --- a/src/modcamera.h +++ b/src/modcamera.h @@ -106,6 +106,7 @@ typedef struct hal_camera_obj { camera_config_t camera_config; bool initialized; camera_fb_t *captured_buffer; + bool bmp_out; } hal_camera_obj_t; #endif // CONFIG_IDF_TARGET_ESP32 || CONFIG_IDF_TARGET_ESP32S2 || CONFIG_IDF_TARGET_ESP32S3 @@ -194,16 +195,16 @@ extern void mp_camera_hal_reconfigure(mp_camera_obj_t *self, mp_camera_framesize * @brief Captures an image and returns it as mp_obj_t (e.g. mp_obj_new_memoryview). * * @param self Pointer to the camera object. - * @param timeout_ms Timeout in milliseconds. + * @param out_format Output pixelformat format. * @return Captured image as micropython object. */ -extern mp_obj_t mp_camera_hal_capture(mp_camera_obj_t *self, int timeout_ms); +extern mp_obj_t mp_camera_hal_capture(mp_camera_obj_t *self, int8_t out_format); /** * @brief Table mapping pixel formats API to their corresponding values at HAL. * @details Needs to be defined in the port-specific implementation. */ -extern const mp_rom_map_elem_t mp_camera_hal_pixel_format_table[4]; +extern const mp_rom_map_elem_t mp_camera_hal_pixel_format_table[5]; /** * @brief Table mapping frame sizes API to their corresponding values at HAL. diff --git a/src/modcamera_api.c b/src/modcamera_api.c index 035b213..5f07665 100644 --- a/src/modcamera_api.c +++ b/src/modcamera_api.c @@ -40,7 +40,7 @@ const mp_obj_type_t camera_type; //Constructor static mp_obj_t mp_camera_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) { - enum { ARG_data_pins, ARG_pixel_clock_pin, ARG_vsync_pin, ARG_href_pin, ARG_sda_pin, ARG_scl_pin, ARG_xclock_pin, ARG_xclock_frequency, ARG_powerdown_pin, ARG_reset_pin, ARG_pixel_format, ARG_frame_size, ARG_jpeg_quality, ARG_fb_count, ARG_grab_mode, ARG_init, NUM_ARGS }; + enum { ARG_data_pins, ARG_pixel_clock_pin, ARG_vsync_pin, ARG_href_pin, ARG_sda_pin, ARG_scl_pin, ARG_xclock_pin, ARG_xclock_frequency, ARG_powerdown_pin, ARG_reset_pin, ARG_pixel_format, ARG_frame_size, ARG_jpeg_quality, ARG_fb_count, ARG_grab_mode, ARG_init, ARG_bmp_out, NUM_ARGS }; static const mp_arg_t allowed_args[] = { #ifdef MICROPY_CAMERA_ALL_REQ_PINS_DEFINED { MP_QSTR_data_pins, MP_ARG_OBJ | MP_ARG_KW_ONLY , { .u_obj = MP_ROM_NONE } }, @@ -68,6 +68,7 @@ static mp_obj_t mp_camera_make_new(const mp_obj_type_t *type, size_t n_args, siz { MP_QSTR_fb_count, MP_ARG_INT | MP_ARG_KW_ONLY, { .u_int = MICROPY_CAMERA_FB_COUNT } }, { MP_QSTR_grab_mode, MP_ARG_INT | MP_ARG_KW_ONLY, { .u_int = MICROPY_CAMERA_GRAB_MODE } }, { MP_QSTR_init, MP_ARG_BOOL | MP_ARG_KW_ONLY, { .u_bool = true } }, + { MP_QSTR_bmp_out, MP_ARG_BOOL | MP_ARG_KW_ONLY, { .u_bool = false } }, }; mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; @@ -128,13 +129,14 @@ static mp_obj_t mp_camera_make_new(const mp_obj_type_t *type, size_t n_args, siz mp_camera_obj_t *self = mp_obj_malloc_with_finaliser(mp_camera_obj_t, &camera_type); self->base.type = &camera_type; + self->bmp_out = args[ARG_bmp_out].u_bool; mp_camera_hal_construct(self, data_pins, xclock_pin, pixel_clock_pin, vsync_pin, href_pin, powerdown_pin, reset_pin, sda_pin, scl_pin, xclock_frequency, pixel_format, frame_size, jpeg_quality, fb_count, grab_mode); mp_camera_hal_init(self); - if (mp_camera_hal_capture(self, 100) == mp_const_none){ + if (mp_camera_hal_capture(self, -1) == mp_const_none){ mp_camera_hal_deinit(self); mp_raise_msg(&mp_type_OSError, MP_ERROR_TEXT("Failed to capture initial frame. \ Run reconfigure method or construct a new object with appropriate configuration (e.g. FrameSize).")); @@ -150,8 +152,8 @@ static mp_obj_t mp_camera_make_new(const mp_obj_type_t *type, size_t n_args, siz // Main methods static mp_obj_t camera_capture(size_t n_args, const mp_obj_t *args){ mp_camera_obj_t *self = MP_OBJ_TO_PTR(args[0]); - mp_float_t timeout = n_args < 2 ? MICROPY_FLOAT_CONST(0.25) : mp_obj_get_float(args[1]); - return mp_camera_hal_capture(self, timeout); + int8_t out_format = n_args < 2 ? -1 : mp_obj_get_int(args[1]); + return mp_camera_hal_capture(self, out_format); } static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(camera_capture_obj, 1, 2, camera_capture); @@ -205,14 +207,28 @@ static mp_obj_t mp_camera_deinit(mp_obj_t self_in) { } static MP_DEFINE_CONST_FUN_OBJ_1(mp_camera_deinit_obj, mp_camera_deinit); +static mp_obj_t camera_get_bmp_out(mp_obj_t self_in) { + mp_camera_obj_t *self = MP_OBJ_TO_PTR(self_in); + return mp_obj_new_bool(self->bmp_out); +} +static MP_DEFINE_CONST_FUN_OBJ_1(camera_get_bmp_out_obj, camera_get_bmp_out); + +static mp_obj_t camera_set_bmp_out(mp_obj_t self_in, mp_obj_t arg) { + mp_camera_obj_t *self = MP_OBJ_TO_PTR(self_in); + self->bmp_out = mp_obj_is_true(arg); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_2(camera_set_bmp_out_obj, camera_set_bmp_out); + +// Destructor static mp_obj_t mp_camera_obj___exit__(size_t n_args, const mp_obj_t *args) { (void)n_args; return mp_camera_deinit(args[0]); } static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_camera___exit___obj, 4, 4, mp_camera_obj___exit__); -// Camera propertiy functions -// Camera sensor propertiy functions +// Camera property functions +// Camera sensor property functions #define CREATE_GETTER(property, get_function) \ static mp_obj_t camera_get_##property(const mp_obj_t self_in) { \ mp_camera_obj_t *self = MP_OBJ_TO_PTR(self_in); \ @@ -316,6 +332,7 @@ static const mp_rom_map_elem_t camera_camera_locals_table[] = { ADD_PROPERTY_TO_TABLE(wpc), ADD_PROPERTY_TO_TABLE(raw_gma), ADD_PROPERTY_TO_TABLE(lenc), + ADD_PROPERTY_TO_TABLE(bmp_out), }; static MP_DEFINE_CONST_DICT(camera_camera_locals_dict, camera_camera_locals_table);