From add1200343e59774d2b168cc2ff1ab8ae5660489 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Tue, 27 Jun 2023 00:21:12 +1000 Subject: [PATCH 01/31] all: Remove the zlib module. This will be replaced with a new deflate module providing the same functionality, with an optional frozen Python wrapper providing a replacement zlib module. binascii.crc32 is temporarily disabled. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- examples/natmod/zlib/Makefile | 13 - examples/natmod/zlib/zlib.c | 35 --- extmod/extmod.cmake | 1 - extmod/extmod.mk | 1 - extmod/modbinascii.c | 6 +- extmod/modzlib.c | 239 ------------------ ports/cc3200/mpconfigport.h | 1 - ports/qemu-arm/mpconfigport.h | 1 - ports/samd/mpconfigport.h | 1 - .../boards/NUCLEO_G474RE/mpconfigboard.h | 1 - ports/windows/mpconfigport.h | 1 - ports/windows/msvc/sources.props | 1 - py/mpconfig.h | 4 - tests/extmod/zlib_decompio.py | 33 --- tests/extmod/zlib_decompio.py.exp | 12 - tests/extmod/zlib_decompio_gz.py | 60 ----- tests/extmod/zlib_decompio_gz.py.exp | 13 - tests/extmod/zlib_decompress.py | 57 ----- tests/run-natmodtests.py | 1 - tests/run-tests.py | 1 - tests/unix/extra_coverage.py.exp | 1 - tools/ci.sh | 3 +- 22 files changed, 4 insertions(+), 482 deletions(-) delete mode 100644 examples/natmod/zlib/Makefile delete mode 100644 examples/natmod/zlib/zlib.c delete mode 100644 extmod/modzlib.c delete mode 100644 tests/extmod/zlib_decompio.py delete mode 100644 tests/extmod/zlib_decompio.py.exp delete mode 100644 tests/extmod/zlib_decompio_gz.py delete mode 100644 tests/extmod/zlib_decompio_gz.py.exp delete mode 100644 tests/extmod/zlib_decompress.py diff --git a/examples/natmod/zlib/Makefile b/examples/natmod/zlib/Makefile deleted file mode 100644 index e4b4e0712ef2..000000000000 --- a/examples/natmod/zlib/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -# Location of top-level MicroPython directory -MPY_DIR = ../../.. - -# Name of module (different to built-in zlib so it can coexist) -MOD = zlib_$(ARCH) - -# Source files (.c or .py) -SRC = zlib.c - -# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) -ARCH = x64 - -include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/zlib/zlib.c b/examples/natmod/zlib/zlib.c deleted file mode 100644 index 2b0dbd5ca015..000000000000 --- a/examples/natmod/zlib/zlib.c +++ /dev/null @@ -1,35 +0,0 @@ -#define MICROPY_PY_ZLIB (1) - -#include "py/dynruntime.h" - -#if !defined(__linux__) -void *memset(void *s, int c, size_t n) { - return mp_fun_table.memset_(s, c, n); -} -#endif - -mp_obj_full_type_t decompio_type; - -#include "extmod/modzlib.c" - -mp_map_elem_t decompio_locals_dict_table[3]; -STATIC MP_DEFINE_CONST_DICT(decompio_locals_dict, decompio_locals_dict_table); - -mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { - MP_DYNRUNTIME_INIT_ENTRY - - decompio_type.base.type = mp_fun_table.type_type; - decompio_type.name = MP_QSTR_DecompIO; - MP_OBJ_TYPE_SET_SLOT(&decompio_type, make_new, &decompio_make_new, 0); - MP_OBJ_TYPE_SET_SLOT(&decompio_type, protocol, &decompio_stream_p, 1); - decompio_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_read), MP_OBJ_FROM_PTR(&mp_stream_read_obj) }; - decompio_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readinto), MP_OBJ_FROM_PTR(&mp_stream_readinto_obj) }; - decompio_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readline), MP_OBJ_FROM_PTR(&mp_stream_unbuffered_readline_obj) }; - MP_OBJ_TYPE_SET_SLOT(&decompio_type, locals_dict, (void*)&decompio_locals_dict, 2); - - mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_zlib)); - mp_store_global(MP_QSTR_decompress, MP_OBJ_FROM_PTR(&mod_zlib_decompress_obj)); - mp_store_global(MP_QSTR_DecompIO, MP_OBJ_FROM_PTR(&decompio_type)); - - MP_DYNRUNTIME_INIT_EXIT -} diff --git a/extmod/extmod.cmake b/extmod/extmod.cmake index 67f33cf91f89..29b4627614b4 100644 --- a/extmod/extmod.cmake +++ b/extmod/extmod.cmake @@ -36,7 +36,6 @@ set(MICROPY_SOURCE_EXTMOD ${MICROPY_EXTMOD_DIR}/modssl_mbedtls.c ${MICROPY_EXTMOD_DIR}/modtime.c ${MICROPY_EXTMOD_DIR}/modwebsocket.c - ${MICROPY_EXTMOD_DIR}/modzlib.c ${MICROPY_EXTMOD_DIR}/modwebrepl.c ${MICROPY_EXTMOD_DIR}/network_cyw43.c ${MICROPY_EXTMOD_DIR}/network_lwip.c diff --git a/extmod/extmod.mk b/extmod/extmod.mk index 41d200fce495..0745fb49ebc0 100644 --- a/extmod/extmod.mk +++ b/extmod/extmod.mk @@ -35,7 +35,6 @@ SRC_EXTMOD_C += \ extmod/moductypes.c \ extmod/modwebrepl.c \ extmod/modwebsocket.c \ - extmod/modzlib.c \ extmod/network_cyw43.c \ extmod/network_lwip.c \ extmod/network_ninaw10.c \ diff --git a/extmod/modbinascii.c b/extmod/modbinascii.c index 183cd3fc1789..0c562de7c0f0 100644 --- a/extmod/modbinascii.c +++ b/extmod/modbinascii.c @@ -170,8 +170,8 @@ STATIC mp_obj_t mod_binascii_b2a_base64(size_t n_args, const mp_obj_t *pos_args, } STATIC MP_DEFINE_CONST_FUN_OBJ_KW(mod_binascii_b2a_base64_obj, 1, mod_binascii_b2a_base64); -#if MICROPY_PY_BINASCII_CRC32 -#include "lib/uzlib/tinf.h" +#if 0 // MICROPY_PY_BINASCII_CRC32 +#include "lib/uzlib/crc32.c" STATIC mp_obj_t mod_binascii_crc32(size_t n_args, const mp_obj_t *args) { mp_buffer_info_t bufinfo; @@ -191,7 +191,7 @@ STATIC const mp_rom_map_elem_t mp_module_binascii_globals_table[] = { #endif { MP_ROM_QSTR(MP_QSTR_a2b_base64), MP_ROM_PTR(&mod_binascii_a2b_base64_obj) }, { MP_ROM_QSTR(MP_QSTR_b2a_base64), MP_ROM_PTR(&mod_binascii_b2a_base64_obj) }, - #if MICROPY_PY_BINASCII_CRC32 + #if 0 // MICROPY_PY_BINASCII_CRC32 { MP_ROM_QSTR(MP_QSTR_crc32), MP_ROM_PTR(&mod_binascii_crc32_obj) }, #endif }; diff --git a/extmod/modzlib.c b/extmod/modzlib.c deleted file mode 100644 index 31096cfeb9e9..000000000000 --- a/extmod/modzlib.c +++ /dev/null @@ -1,239 +0,0 @@ -/* - * This file is part of the MicroPython project, http://micropython.org/ - * - * The MIT License (MIT) - * - * Copyright (c) 2014-2016 Paul Sokolovsky - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#include -#include - -#include "py/runtime.h" -#include "py/stream.h" -#include "py/mperrno.h" - -#if MICROPY_PY_ZLIB - -#include "lib/uzlib/tinf.h" - -#if 0 // print debugging info -#define DEBUG_printf DEBUG_printf -#else // don't print debugging info -#define DEBUG_printf(...) (void)0 -#endif - -typedef struct _mp_obj_decompio_t { - mp_obj_base_t base; - mp_obj_t src_stream; - TINF_DATA decomp; - bool eof; -} mp_obj_decompio_t; - -STATIC int read_src_stream(TINF_DATA *data) { - byte *p = (void *)data; - p -= offsetof(mp_obj_decompio_t, decomp); - mp_obj_decompio_t *self = (mp_obj_decompio_t *)p; - - const mp_stream_p_t *stream = mp_get_stream(self->src_stream); - int err; - byte c; - mp_uint_t out_sz = stream->read(self->src_stream, &c, 1, &err); - if (out_sz == MP_STREAM_ERROR) { - mp_raise_OSError(err); - } - if (out_sz == 0) { - mp_raise_type(&mp_type_EOFError); - } - return c; -} - -STATIC mp_obj_t decompio_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { - mp_arg_check_num(n_args, n_kw, 1, 2, false); - mp_get_stream_raise(args[0], MP_STREAM_OP_READ); - mp_obj_decompio_t *o = mp_obj_malloc(mp_obj_decompio_t, type); - memset(&o->decomp, 0, sizeof(o->decomp)); - o->decomp.readSource = read_src_stream; - o->src_stream = args[0]; - o->eof = false; - - mp_int_t dict_opt = 0; - uint dict_sz; - if (n_args > 1) { - dict_opt = mp_obj_get_int(args[1]); - } - - if (dict_opt >= 16) { - int st = uzlib_gzip_parse_header(&o->decomp); - if (st != TINF_OK) { - goto header_error; - } - dict_sz = 1 << (dict_opt - 16); - } else if (dict_opt >= 0) { - dict_opt = uzlib_zlib_parse_header(&o->decomp); - if (dict_opt < 0) { - header_error: - mp_raise_ValueError(MP_ERROR_TEXT("compression header")); - } - // RFC 1950 section 2.2: - // CINFO is the base-2 logarithm of the LZ77 window size, - // minus eight (CINFO=7 indicates a 32K window size) - dict_sz = 1 << (dict_opt + 8); - } else { - dict_sz = 1 << -dict_opt; - } - - uzlib_uncompress_init(&o->decomp, m_new(byte, dict_sz), dict_sz); - return MP_OBJ_FROM_PTR(o); -} - -STATIC mp_uint_t decompio_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) { - mp_obj_decompio_t *o = MP_OBJ_TO_PTR(o_in); - if (o->eof) { - return 0; - } - - o->decomp.dest = buf; - o->decomp.dest_limit = (byte *)buf + size; - int st = uzlib_uncompress_chksum(&o->decomp); - if (st == TINF_DONE) { - o->eof = true; - } - if (st < 0) { - DEBUG_printf("uncompress error=" INT_FMT "\n", st); - *errcode = MP_EINVAL; - return MP_STREAM_ERROR; - } - return o->decomp.dest - (byte *)buf; -} - -#if !MICROPY_ENABLE_DYNRUNTIME -STATIC const mp_rom_map_elem_t decompio_locals_dict_table[] = { - { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) }, - { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) }, - { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) }, -}; - -STATIC MP_DEFINE_CONST_DICT(decompio_locals_dict, decompio_locals_dict_table); -#endif - -STATIC const mp_stream_p_t decompio_stream_p = { - .read = decompio_read, -}; - -#if !MICROPY_ENABLE_DYNRUNTIME -STATIC MP_DEFINE_CONST_OBJ_TYPE( - decompio_type, - MP_QSTR_DecompIO, - MP_TYPE_FLAG_NONE, - make_new, decompio_make_new, - protocol, &decompio_stream_p, - locals_dict, &decompio_locals_dict - ); -#endif - -STATIC mp_obj_t mod_zlib_decompress(size_t n_args, const mp_obj_t *args) { - mp_obj_t data = args[0]; - mp_buffer_info_t bufinfo; - mp_get_buffer_raise(data, &bufinfo, MP_BUFFER_READ); - - TINF_DATA *decomp = m_new_obj(TINF_DATA); - memset(decomp, 0, sizeof(*decomp)); - DEBUG_printf("sizeof(TINF_DATA)=" UINT_FMT "\n", sizeof(*decomp)); - uzlib_uncompress_init(decomp, NULL, 0); - mp_uint_t dest_buf_size = (bufinfo.len + 15) & ~15; - byte *dest_buf = m_new(byte, dest_buf_size); - - decomp->dest = dest_buf; - decomp->dest_limit = dest_buf + dest_buf_size; - DEBUG_printf("zlib: Initial out buffer: " UINT_FMT " bytes\n", dest_buf_size); - decomp->source = bufinfo.buf; - decomp->source_limit = (byte *)bufinfo.buf + bufinfo.len; - - int st; - bool is_zlib = true; - - if (n_args > 1 && MP_OBJ_SMALL_INT_VALUE(args[1]) < 0) { - is_zlib = false; - } - - if (is_zlib) { - st = uzlib_zlib_parse_header(decomp); - if (st < 0) { - goto error; - } - } - - while (1) { - st = uzlib_uncompress_chksum(decomp); - if (st < 0) { - goto error; - } - if (st == TINF_DONE) { - break; - } - size_t offset = decomp->dest - dest_buf; - dest_buf = m_renew(byte, dest_buf, dest_buf_size, dest_buf_size + 256); - dest_buf_size += 256; - decomp->dest = dest_buf + offset; - decomp->dest_limit = decomp->dest + 256; - } - - mp_uint_t final_sz = decomp->dest - dest_buf; - DEBUG_printf("zlib: Resizing from " UINT_FMT " to final size: " UINT_FMT " bytes\n", dest_buf_size, final_sz); - dest_buf = (byte *)m_renew(byte, dest_buf, dest_buf_size, final_sz); - mp_obj_t res = mp_obj_new_bytearray_by_ref(final_sz, dest_buf); - m_del_obj(TINF_DATA, decomp); - return res; - -error: - mp_raise_type_arg(&mp_type_ValueError, MP_OBJ_NEW_SMALL_INT(st)); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mod_zlib_decompress_obj, 1, 3, mod_zlib_decompress); - -#if !MICROPY_ENABLE_DYNRUNTIME -STATIC const mp_rom_map_elem_t mp_module_zlib_globals_table[] = { - { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_zlib) }, - { MP_ROM_QSTR(MP_QSTR_decompress), MP_ROM_PTR(&mod_zlib_decompress_obj) }, - { MP_ROM_QSTR(MP_QSTR_DecompIO), MP_ROM_PTR(&decompio_type) }, -}; - -STATIC MP_DEFINE_CONST_DICT(mp_module_zlib_globals, mp_module_zlib_globals_table); - -const mp_obj_module_t mp_module_zlib = { - .base = { &mp_type_module }, - .globals = (mp_obj_dict_t *)&mp_module_zlib_globals, -}; - - -MP_REGISTER_EXTENSIBLE_MODULE(MP_QSTR_zlib, mp_module_zlib); -#endif - -// Source files #include'd here to make sure they're compiled in -// only if module is enabled by config setting. - -#include "lib/uzlib/tinflate.c" -#include "lib/uzlib/tinfzlib.c" -#include "lib/uzlib/tinfgzip.c" -#include "lib/uzlib/adler32.c" -#include "lib/uzlib/crc32.c" - -#endif // MICROPY_PY_ZLIB diff --git a/ports/cc3200/mpconfigport.h b/ports/cc3200/mpconfigport.h index af10c63531ff..dbceabb9d86a 100644 --- a/ports/cc3200/mpconfigport.h +++ b/ports/cc3200/mpconfigport.h @@ -110,7 +110,6 @@ #define MICROPY_PY_THREAD_GIL (1) #define MICROPY_PY_BINASCII (1) #define MICROPY_PY_UCTYPES (0) -#define MICROPY_PY_ZLIB (0) #define MICROPY_PY_JSON (1) #define MICROPY_PY_RE (1) #define MICROPY_PY_HEAPQ (0) diff --git a/ports/qemu-arm/mpconfigport.h b/ports/qemu-arm/mpconfigport.h index 0eada5a529c5..d186f70b7eb3 100644 --- a/ports/qemu-arm/mpconfigport.h +++ b/ports/qemu-arm/mpconfigport.h @@ -40,7 +40,6 @@ #define MICROPY_PY_BINASCII (1) #define MICROPY_PY_RANDOM (1) #define MICROPY_PY_UCTYPES (1) -#define MICROPY_PY_ZLIB (1) #define MICROPY_PY_JSON (1) #define MICROPY_PY_OS (1) #define MICROPY_PY_RE (1) diff --git a/ports/samd/mpconfigport.h b/ports/samd/mpconfigport.h index 600c79da04fd..827e952b00de 100644 --- a/ports/samd/mpconfigport.h +++ b/ports/samd/mpconfigport.h @@ -91,7 +91,6 @@ #define MICROPY_PY_UCTYPES (1) #define MICROPY_PY_HEAPQ (1) #define MICROPY_PY_RANDOM (1) -#define MICROPY_PY_ZLIB (1) #define MICROPY_PY_ASYNCIO (1) #define MICROPY_PY_MACHINE_RTC (1) #ifndef MICROPY_PY_MACHINE_ADC diff --git a/ports/stm32/boards/NUCLEO_G474RE/mpconfigboard.h b/ports/stm32/boards/NUCLEO_G474RE/mpconfigboard.h index e10f63da7321..0093b2df39bb 100644 --- a/ports/stm32/boards/NUCLEO_G474RE/mpconfigboard.h +++ b/ports/stm32/boards/NUCLEO_G474RE/mpconfigboard.h @@ -10,7 +10,6 @@ #define MICROPY_HW_HAS_FLASH (0) // QSPI extflash not mounted #define MICROPY_PY_ASYNCIO (0) -#define MICROPY_PY_ZLIB (0) #define MICROPY_PY_BINASCII (0) #define MICROPY_PY_HASHLIB (0) #define MICROPY_PY_JSON (0) diff --git a/ports/windows/mpconfigport.h b/ports/windows/mpconfigport.h index dc9208f6f1dc..0658838075f6 100644 --- a/ports/windows/mpconfigport.h +++ b/ports/windows/mpconfigport.h @@ -144,7 +144,6 @@ #define MICROPY_PY_TIME_INCLUDEFILE "ports/unix/modtime.c" #define MICROPY_PY_ERRNO (1) #define MICROPY_PY_UCTYPES (1) -#define MICROPY_PY_ZLIB (1) #define MICROPY_PY_JSON (1) #define MICROPY_PY_RE (1) #define MICROPY_PY_HEAPQ (1) diff --git a/ports/windows/msvc/sources.props b/ports/windows/msvc/sources.props index 6277b74e20cc..6d438f3f0db0 100644 --- a/ports/windows/msvc/sources.props +++ b/ports/windows/msvc/sources.props @@ -18,7 +18,6 @@ - diff --git a/py/mpconfig.h b/py/mpconfig.h index 2bbe3849b051..122d0a70f2a7 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -1555,10 +1555,6 @@ typedef double mp_float_t; #define MICROPY_PY_UCTYPES_NATIVE_C_TYPES (1) #endif -#ifndef MICROPY_PY_ZLIB -#define MICROPY_PY_ZLIB (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) -#endif - #ifndef MICROPY_PY_JSON #define MICROPY_PY_JSON (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif diff --git a/tests/extmod/zlib_decompio.py b/tests/extmod/zlib_decompio.py deleted file mode 100644 index 9abbad43c60f..000000000000 --- a/tests/extmod/zlib_decompio.py +++ /dev/null @@ -1,33 +0,0 @@ -try: - import zlib - import io -except ImportError: - print("SKIP") - raise SystemExit - - -# Raw DEFLATE bitstream -buf = io.BytesIO(b"\xcbH\xcd\xc9\xc9\x07\x00") -inp = zlib.DecompIO(buf, -8) -print(buf.seek(0, 1)) -print(inp.read(1)) -print(buf.seek(0, 1)) -print(inp.read(2)) -print(inp.read()) -print(buf.seek(0, 1)) -print(inp.read(1)) -print(inp.read()) -print(buf.seek(0, 1)) - - -# zlib bitstream (with 256 byte window size) -inp = zlib.DecompIO(io.BytesIO(b"\x08\x9930\xa0=\x00\x00\xb3q\x12\xc1")) -print(inp.read(10)) -print(inp.read()) - -# zlib bitstream, wrong checksum -inp = zlib.DecompIO(io.BytesIO(b"\x08\x9930\xa0=\x00\x00\xb3q\x12\xc0")) -try: - print(inp.read()) -except OSError as e: - print(repr(e)) diff --git a/tests/extmod/zlib_decompio.py.exp b/tests/extmod/zlib_decompio.py.exp deleted file mode 100644 index 3f5f360fa3ec..000000000000 --- a/tests/extmod/zlib_decompio.py.exp +++ /dev/null @@ -1,12 +0,0 @@ -0 -b'h' -2 -b'el' -b'lo' -7 -b'' -b'' -7 -b'0000000000' -b'000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000' -OSError(22,) diff --git a/tests/extmod/zlib_decompio_gz.py b/tests/extmod/zlib_decompio_gz.py deleted file mode 100644 index 1407459304dc..000000000000 --- a/tests/extmod/zlib_decompio_gz.py +++ /dev/null @@ -1,60 +0,0 @@ -try: - import zlib - import io -except ImportError: - print("SKIP") - raise SystemExit - - -# gzip bitstream -buf = io.BytesIO( - b"\x1f\x8b\x08\x08\x99\x0c\xe5W\x00\x03hello\x00\xcbH\xcd\xc9\xc9\x07\x00\x86\xa6\x106\x05\x00\x00\x00" -) -inp = zlib.DecompIO(buf, 16 + 8) -print(buf.seek(0, 1)) -print(inp.read(1)) -print(buf.seek(0, 1)) -print(inp.read(2)) -print(inp.read()) -print(buf.seek(0, 1)) -print(inp.read(1)) -print(inp.read()) -print(buf.seek(0, 1)) - -# Check FHCRC field -buf = io.BytesIO( - b"\x1f\x8b\x08\x02\x99\x0c\xe5W\x00\x03\x00\x00\xcbH\xcd\xc9\xc9\x07\x00\x86\xa6\x106\x05\x00\x00\x00" -) -inp = zlib.DecompIO(buf, 16 + 8) -print(inp.read()) - -# Check FEXTRA field -buf = io.BytesIO( - b"\x1f\x8b\x08\x04\x99\x0c\xe5W\x00\x03\x01\x00X\xcbH\xcd\xc9\xc9\x07\x00\x86\xa6\x106\x05\x00\x00\x00" -) -inp = zlib.DecompIO(buf, 16 + 8) -print(inp.read()) - -# broken header -buf = io.BytesIO( - b"\x1f\x8c\x08\x08\x99\x0c\xe5W\x00\x03hello\x00\xcbH\xcd\xc9\xc9\x07\x00\x86\xa6\x106\x05\x00\x00\x00" -) -try: - inp = zlib.DecompIO(buf, 16 + 8) -except ValueError: - print("ValueError") - -# broken crc32 -buf = io.BytesIO( - b"\x1f\x8b\x08\x08\x99\x0c\xe5W\x00\x03hello\x00\xcbH\xcd\xc9\xc9\x07\x00\x86\xa7\x106\x05\x00\x00\x00" -) -inp = zlib.DecompIO(buf, 16 + 8) -try: - inp.read(6) -except OSError as e: - print(repr(e)) - -# broken uncompressed size - not checked so far -# buf = io.BytesIO(b'\x1f\x8b\x08\x08\x99\x0c\xe5W\x00\x03hello\x00\xcbH\xcd\xc9\xc9\x07\x00\x86\xa6\x106\x06\x00\x00\x00') -# inp = zlib.DecompIO(buf, 16 + 8) -# inp.read(6) diff --git a/tests/extmod/zlib_decompio_gz.py.exp b/tests/extmod/zlib_decompio_gz.py.exp deleted file mode 100644 index 20a30c82a3d5..000000000000 --- a/tests/extmod/zlib_decompio_gz.py.exp +++ /dev/null @@ -1,13 +0,0 @@ -16 -b'h' -18 -b'el' -b'lo' -31 -b'' -b'' -31 -b'hello' -b'hello' -ValueError -OSError(22,) diff --git a/tests/extmod/zlib_decompress.py b/tests/extmod/zlib_decompress.py deleted file mode 100644 index b72ee96ea88c..000000000000 --- a/tests/extmod/zlib_decompress.py +++ /dev/null @@ -1,57 +0,0 @@ -try: - import zlib -except ImportError: - print("SKIP") - raise SystemExit - -PATTERNS = [ - # Packed results produced by CPy's zlib.compress() - (b"0", b"x\x9c3\x00\x00\x001\x001"), - (b"a", b"x\x9cK\x04\x00\x00b\x00b"), - (b"0" * 100, b"x\x9c30\xa0=\x00\x00\xb3q\x12\xc1"), - ( - bytes(range(64)), - b"x\x9cc`dbfaec\xe7\xe0\xe4\xe2\xe6\xe1\xe5\xe3\x17\x10\x14\x12\x16\x11\x15\x13\x97\x90\x94\x92\x96\x91\x95\x93WPTRVQUS\xd7\xd0\xd4\xd2\xd6\xd1\xd5\xd370426153\xb7\xb0\xb4\xb2\xb6\xb1\xb5\xb3\x07\x00\xaa\xe0\x07\xe1", - ), - (b"hello", b"x\x01\x01\x05\x00\xfa\xffhello\x06,\x02\x15"), # compression level 0 - # adaptive/dynamic huffman tree - ( - b"13371813150|13764518736|12345678901", - b"x\x9c\x05\xc1\x81\x01\x000\x04\x04\xb1\x95\\\x1f\xcfn\x86o\x82d\x06Qq\xc8\x9d\xc5X}I}\x00\x951D>I}\x00\x951D>I}\x00\x951D>I}\x00\x951D", - b"x\x9c\x05\xc11\x01\x00\x00\x00\x010\x95\x14py\x84\x12C_\x9bR\x8cV\x8a\xd1J1Z)F\x1fw`\x089", - ), -] - -for unpacked, packed in PATTERNS: - assert zlib.decompress(packed) == unpacked - print(unpacked) - - -# Raw DEFLATE bitstream -v = b"\xcbH\xcd\xc9\xc9\x07\x00" -exp = b"hello" -out = zlib.decompress(v, -15) -assert out == exp -print(exp) -# Even when you ask CPython zlib.compress to produce Raw DEFLATE stream, -# it returns it with adler2 and oriignal size appended, as if it was a -# zlib stream. Make sure there're no random issues decompressing such. -v = b"\xcbH\xcd\xc9\xc9\x07\x00\x86\xa6\x106\x05\x00\x00\x00" -out = zlib.decompress(v, -15) -assert out == exp - -# this should error -try: - zlib.decompress(b"abc") -except Exception: - print("Exception") - -# invalid block type -try: - zlib.decompress(b"\x07", -15) # final-block, block-type=3 (invalid) -except Exception as er: - print("Exception") diff --git a/tests/run-natmodtests.py b/tests/run-natmodtests.py index 16f8fde967b3..ce4f94135063 100755 --- a/tests/run-natmodtests.py +++ b/tests/run-natmodtests.py @@ -25,7 +25,6 @@ "heapq": "heapq/heapq_$(ARCH).mpy", "random": "random/random_$(ARCH).mpy", "re": "re/re_$(ARCH).mpy", - "zlib": "zlib/zlib_$(ARCH).mpy", } # Code to allow a target MicroPython to import an .mpy from RAM diff --git a/tests/run-tests.py b/tests/run-tests.py index 6fee9d91834f..019b30189f93 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -567,7 +567,6 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1): for t in "bytearray le native_le ptr_le ptr_native_le sizeof sizeof_native array_assign_le array_assign_native_le".split() } ) # requires uctypes - skip_tests.add("extmod/zlibd_decompress.py") # requires zlib skip_tests.add("extmod/heapq1.py") # heapq not supported by WiPy skip_tests.add("extmod/random_basic.py") # requires random skip_tests.add("extmod/random_extra.py") # requires random diff --git a/tests/unix/extra_coverage.py.exp b/tests/unix/extra_coverage.py.exp index 013782ede5f1..9c7660de6dc6 100644 --- a/tests/unix/extra_coverage.py.exp +++ b/tests/unix/extra_coverage.py.exp @@ -59,7 +59,6 @@ io json machine math os random re select socket ssl struct sys termios time uctypes websocket -zlib me micropython machine math diff --git a/tools/ci.sh b/tools/ci.sh index 84ca83e9b8aa..bbff5359a1e1 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -430,7 +430,6 @@ function ci_native_mpy_modules_build { make -C examples/natmod/heapq ARCH=$arch make -C examples/natmod/random ARCH=$arch make -C examples/natmod/re ARCH=$arch - make -C examples/natmod/zlib ARCH=$arch } function ci_native_mpy_modules_32bit_build { @@ -496,7 +495,7 @@ function ci_unix_coverage_run_mpy_merge_tests { function ci_unix_coverage_run_native_mpy_tests { MICROPYPATH=examples/natmod/features2 ./ports/unix/build-coverage/micropython -m features2 - (cd tests && ./run-natmodtests.py "$@" extmod/{btree*,framebuf*,heapq*,random*,re*,zlib*}.py) + (cd tests && ./run-natmodtests.py "$@" extmod/{btree*,framebuf*,heapq*,random*,re*}.py) } function ci_unix_32bit_setup { From 198311c780e9e05a58c710d73a22abaf8347d4ee Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Mon, 26 Jun 2023 23:53:12 +1000 Subject: [PATCH 02/31] py/stream: Add mp_stream___exit___obj that calls mp_stream_close. There are enough places that implement __exit__ by forwarding directly to mp_stream_close that this saves code size. For the cases where __exit__ is a no-op, additionally make their MP_STREAM_CLOSE ioctl handled as a no-op. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- extmod/vfs_fat_file.c | 9 +-------- extmod/vfs_lfsx_file.c | 8 +------- extmod/vfs_posix_file.c | 8 +------- ports/stm32/usb.c | 9 +++------ py/objstringio.c | 8 +------- py/stream.c | 6 ++++++ py/stream.h | 1 + shared/runtime/sys_stdio_mphal.c | 9 +++------ 8 files changed, 17 insertions(+), 41 deletions(-) diff --git a/extmod/vfs_fat_file.c b/extmod/vfs_fat_file.c index 07e6df9bf959..f82fe4aaae7e 100644 --- a/extmod/vfs_fat_file.c +++ b/extmod/vfs_fat_file.c @@ -96,13 +96,6 @@ STATIC mp_uint_t file_obj_write(mp_obj_t self_in, const void *buf, mp_uint_t siz return sz_out; } - -STATIC mp_obj_t file_obj___exit__(size_t n_args, const mp_obj_t *args) { - (void)n_args; - return mp_stream_close(args[0]); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(file_obj___exit___obj, 4, 4, file_obj___exit__); - STATIC mp_uint_t file_obj_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, int *errcode) { pyb_file_obj_t *self = MP_OBJ_TO_PTR(o_in); @@ -165,7 +158,7 @@ STATIC const mp_rom_map_elem_t vfs_fat_rawfile_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_tell), MP_ROM_PTR(&mp_stream_tell_obj) }, { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, - { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&file_obj___exit___obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&mp_stream___exit___obj) }, }; STATIC MP_DEFINE_CONST_DICT(vfs_fat_rawfile_locals_dict, vfs_fat_rawfile_locals_dict_table); diff --git a/extmod/vfs_lfsx_file.c b/extmod/vfs_lfsx_file.c index 5f0155d3c5c0..879ba78973db 100644 --- a/extmod/vfs_lfsx_file.c +++ b/extmod/vfs_lfsx_file.c @@ -123,12 +123,6 @@ mp_obj_t MP_VFS_LFSx(file_open)(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mod return MP_OBJ_FROM_PTR(o); } -STATIC mp_obj_t MP_VFS_LFSx(file___exit__)(size_t n_args, const mp_obj_t *args) { - (void)n_args; - return mp_stream_close(args[0]); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(MP_VFS_LFSx(file___exit___obj), 4, 4, MP_VFS_LFSx(file___exit__)); - STATIC mp_uint_t MP_VFS_LFSx(file_read)(mp_obj_t self_in, void *buf, mp_uint_t size, int *errcode) { MP_OBJ_VFS_LFSx_FILE *self = MP_OBJ_TO_PTR(self_in); MP_VFS_LFSx(check_open)(self); @@ -213,7 +207,7 @@ STATIC const mp_rom_map_elem_t MP_VFS_LFSx(file_locals_dict_table)[] = { { MP_ROM_QSTR(MP_QSTR_tell), MP_ROM_PTR(&mp_stream_tell_obj) }, { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, - { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&MP_VFS_LFSx(file___exit___obj)) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&mp_stream___exit___obj) }, }; STATIC MP_DEFINE_CONST_DICT(MP_VFS_LFSx(file_locals_dict), MP_VFS_LFSx(file_locals_dict_table)); diff --git a/extmod/vfs_posix_file.c b/extmod/vfs_posix_file.c index 1d89e3ca0973..cfc56df99f10 100644 --- a/extmod/vfs_posix_file.c +++ b/extmod/vfs_posix_file.c @@ -115,12 +115,6 @@ STATIC mp_obj_t vfs_posix_file_fileno(mp_obj_t self_in) { } STATIC MP_DEFINE_CONST_FUN_OBJ_1(vfs_posix_file_fileno_obj, vfs_posix_file_fileno); -STATIC mp_obj_t vfs_posix_file___exit__(size_t n_args, const mp_obj_t *args) { - (void)n_args; - return mp_stream_close(args[0]); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(vfs_posix_file___exit___obj, 4, 4, vfs_posix_file___exit__); - STATIC mp_uint_t vfs_posix_file_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) { mp_obj_vfs_posix_file_t *o = MP_OBJ_TO_PTR(o_in); check_fd_is_open(o); @@ -239,7 +233,7 @@ STATIC const mp_rom_map_elem_t vfs_posix_rawfile_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, - { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&vfs_posix_file___exit___obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&mp_stream___exit___obj) }, }; STATIC MP_DEFINE_CONST_DICT(vfs_posix_rawfile_locals_dict, vfs_posix_rawfile_locals_dict_table); diff --git a/ports/stm32/usb.c b/ports/stm32/usb.c index c3b8544417d0..df6b2cf62a1f 100644 --- a/ports/stm32/usb.c +++ b/ports/stm32/usb.c @@ -858,11 +858,6 @@ STATIC mp_obj_t pyb_usb_vcp_irq(size_t n_args, const mp_obj_t *pos_args, mp_map_ } STATIC MP_DEFINE_CONST_FUN_OBJ_KW(pyb_usb_vcp_irq_obj, 1, pyb_usb_vcp_irq); -mp_obj_t pyb_usb_vcp___exit__(size_t n_args, const mp_obj_t *args) { - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(pyb_usb_vcp___exit___obj, 4, 4, pyb_usb_vcp___exit__); - STATIC const mp_rom_map_elem_t pyb_usb_vcp_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_init), MP_ROM_PTR(&pyb_usb_vcp_init_obj) }, { MP_ROM_QSTR(MP_QSTR_setinterrupt), MP_ROM_PTR(&pyb_usb_vcp_setinterrupt_obj) }, @@ -879,7 +874,7 @@ STATIC const mp_rom_map_elem_t pyb_usb_vcp_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_irq), MP_ROM_PTR(&pyb_usb_vcp_irq_obj) }, { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&mp_identity_obj) }, { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, - { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&pyb_usb_vcp___exit___obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&mp_stream___exit___obj) }, // class constants { MP_ROM_QSTR(MP_QSTR_RTS), MP_ROM_INT(USBD_CDC_FLOWCONTROL_RTS) }, @@ -923,6 +918,8 @@ STATIC mp_uint_t pyb_usb_vcp_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_ if ((flags & MP_STREAM_POLL_WR) && usbd_cdc_tx_half_empty(self->cdc_itf)) { ret |= MP_STREAM_POLL_WR; } + } else if (request == MP_STREAM_CLOSE) { + ret = 0; } else { *errcode = MP_EINVAL; ret = MP_STREAM_ERROR; diff --git a/py/objstringio.c b/py/objstringio.c index 1a083449bafa..a3c66ed010cc 100644 --- a/py/objstringio.c +++ b/py/objstringio.c @@ -170,12 +170,6 @@ STATIC mp_obj_t stringio_getvalue(mp_obj_t self_in) { } STATIC MP_DEFINE_CONST_FUN_OBJ_1(stringio_getvalue_obj, stringio_getvalue); -STATIC mp_obj_t stringio___exit__(size_t n_args, const mp_obj_t *args) { - (void)n_args; - return mp_stream_close(args[0]); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(stringio___exit___obj, 4, 4, stringio___exit__); - STATIC mp_obj_stringio_t *stringio_new(const mp_obj_type_t *type) { mp_obj_stringio_t *o = mp_obj_malloc(mp_obj_stringio_t, type); o->pos = 0; @@ -232,7 +226,7 @@ STATIC const mp_rom_map_elem_t stringio_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR_getvalue), MP_ROM_PTR(&stringio_getvalue_obj) }, { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, - { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&stringio___exit___obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&mp_stream___exit___obj) }, }; STATIC MP_DEFINE_CONST_DICT(stringio_locals_dict, stringio_locals_dict_table); diff --git a/py/stream.c b/py/stream.c index 36325e7d414a..ac0234ac1353 100644 --- a/py/stream.c +++ b/py/stream.c @@ -438,6 +438,12 @@ mp_obj_t mp_stream_close(mp_obj_t stream) { } MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_close_obj, mp_stream_close); +STATIC mp_obj_t mp_stream___exit__(size_t n_args, const mp_obj_t *args) { + (void)n_args; + return mp_stream_close(args[0]); +} +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream___exit___obj, 4, 4, mp_stream___exit__); + STATIC mp_obj_t stream_seek(size_t n_args, const mp_obj_t *args) { struct mp_stream_seek_t seek_s; // TODO: Could be uint64 diff --git a/py/stream.h b/py/stream.h index 4bc329b3e790..6b5c85ab15c2 100644 --- a/py/stream.h +++ b/py/stream.h @@ -83,6 +83,7 @@ MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_unbuffered_readlines_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_write_obj); MP_DECLARE_CONST_FUN_OBJ_2(mp_stream_write1_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_close_obj); +MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream___exit___obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_seek_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_tell_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_flush_obj); diff --git a/shared/runtime/sys_stdio_mphal.c b/shared/runtime/sys_stdio_mphal.c index ab5bdd34e73c..84ce5828ef04 100644 --- a/shared/runtime/sys_stdio_mphal.c +++ b/shared/runtime/sys_stdio_mphal.c @@ -89,17 +89,14 @@ STATIC mp_uint_t stdio_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_t arg, (void)self_in; if (request == MP_STREAM_POLL) { return mp_hal_stdio_poll(arg); + } else if (request == MP_STREAM_CLOSE) { + return 0; } else { *errcode = MP_EINVAL; return MP_STREAM_ERROR; } } -STATIC mp_obj_t stdio_obj___exit__(size_t n_args, const mp_obj_t *args) { - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(stdio_obj___exit___obj, 4, 4, stdio_obj___exit__); - STATIC const mp_rom_map_elem_t stdio_locals_dict_table[] = { #if MICROPY_PY_SYS_STDIO_BUFFER { MP_ROM_QSTR(MP_QSTR_buffer), MP_ROM_PTR(&stdio_buffer_obj) }, @@ -111,7 +108,7 @@ STATIC const mp_rom_map_elem_t stdio_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) }, { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_identity_obj) }, { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, - { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&stdio_obj___exit___obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&mp_stream___exit___obj) }, }; STATIC MP_DEFINE_CONST_DICT(stdio_locals_dict, stdio_locals_dict_table); From c4feb806e0df452273b3c19751d8dad39ef8295b Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 18 Jan 2023 15:46:23 +1100 Subject: [PATCH 03/31] lib/uzlib: Add memory-efficient, streaming LZ77 compression support. The compression algorithm implemented in this commit uses much less memory compared to the standard way of implementing it using a hash table and large look-back window. In particular the algorithm here doesn't allocate hash table to store indices into the history of the previously seen text. Instead it simply does a brute-force-search of the history text to find a match for the compressor. This is slower (linear search vs hash table lookup) but with a small enough history (eg 512 bytes) it's not that slow. And a small history does not impact the compression too much. To give some more concrete numbers comparing memory use between the approaches: - Standard approach: inplace compression, all text to compress must be in RAM (or at least memory addressable), and then an additional 16k bytes RAM of hash table pointers, pointing into the text - The approach in this commit: streaming compression, only a limited amount of previous text must be in RAM (user selectable, defaults to 512 bytes). To compress, say, 1k of data, the standard approach requires all that data to be in RAM, plus an additional 16k of RAM for the hash table pointers. With this commit, you only need the 1k of data in RAM. Or if it's streaming from a file (or elsewhere), you could get away with only 256 bytes of RAM for the sliding history and still get very decent compression. In summary: because compression takes such a large amount of RAM (in the standard algorithm) and it's not really suitable for microcontrollers, the approach taken in this commit is to minimise RAM usage as much as possible, and still have acceptable performance (speed and compression ratio). Signed-off-by: Damien George --- lib/uzlib/defl_static.c | 302 ++++++++++++++++++++++++++++++++++++++++ lib/uzlib/defl_static.h | 5 +- lib/uzlib/lz77.c | 107 ++++++++++++++ lib/uzlib/uzlib.h | 17 ++- 4 files changed, 419 insertions(+), 12 deletions(-) create mode 100644 lib/uzlib/defl_static.c create mode 100644 lib/uzlib/lz77.c diff --git a/lib/uzlib/defl_static.c b/lib/uzlib/defl_static.c new file mode 100644 index 000000000000..89b5e1aa5ff6 --- /dev/null +++ b/lib/uzlib/defl_static.c @@ -0,0 +1,302 @@ +/* + +Routines in this file are based on: +Zlib (RFC1950 / RFC1951) compression for PuTTY. + +PuTTY is copyright 1997-2014 Simon Tatham. + +Portions copyright Robert de Bath, Joris van Rantwijk, Delian +Delchev, Andreas Schultz, Jeroen Massar, Wez Furlong, Nicolas Barry, +Justin Bradford, Ben Harris, Malcolm Smith, Ahmad Khalifa, Markus +Kuhn, Colin Watson, and CORE SDI S.A. + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation files +(the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, +and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#include +#include +#include +#include + +/* ---------------------------------------------------------------------- + * Zlib compression. We always use the static Huffman tree option. + * Mostly this is because it's hard to scan a block in advance to + * work out better trees; dynamic trees are great when you're + * compressing a large file under no significant time constraint, + * but when you're compressing little bits in real time, things get + * hairier. + * + * I suppose it's possible that I could compute Huffman trees based + * on the frequencies in the _previous_ block, as a sort of + * heuristic, but I'm not confident that the gain would balance out + * having to transmit the trees. + */ + +static void outbits(struct Outbuf *out, unsigned long bits, int nbits) +{ + assert(out->noutbits + nbits <= 32); + out->outbits |= bits << out->noutbits; + out->noutbits += nbits; + while (out->noutbits >= 8) { + out->dest_write_cb(out, out->outbits & 0xFF); + out->outbits >>= 8; + out->noutbits -= 8; + } +} + +static const unsigned char mirrorbytes[256] = { + 0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, + 0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0, + 0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8, + 0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8, + 0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4, + 0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4, + 0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec, + 0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc, + 0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2, + 0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2, + 0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea, + 0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa, + 0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6, + 0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6, + 0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee, + 0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe, + 0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1, + 0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1, + 0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9, + 0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9, + 0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5, + 0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5, + 0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed, + 0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd, + 0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3, + 0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3, + 0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb, + 0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb, + 0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7, + 0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7, + 0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef, + 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff, +}; + +typedef struct { + uint8_t extrabits; + uint8_t min, max; +} len_coderecord; + +typedef struct { + uint8_t code, extrabits; + uint16_t min, max; +} dist_coderecord; + +#define TO_LCODE(x, y) x - 3, y - 3 +#define FROM_LCODE(x) (x + 3) + +static const len_coderecord lencodes[] = { + {0, TO_LCODE(3, 3)}, + {0, TO_LCODE(4, 4)}, + {0, TO_LCODE(5, 5)}, + {0, TO_LCODE(6, 6)}, + {0, TO_LCODE(7, 7)}, + {0, TO_LCODE(8, 8)}, + {0, TO_LCODE(9, 9)}, + {0, TO_LCODE(10, 10)}, + {1, TO_LCODE(11, 12)}, + {1, TO_LCODE(13, 14)}, + {1, TO_LCODE(15, 16)}, + {1, TO_LCODE(17, 18)}, + {2, TO_LCODE(19, 22)}, + {2, TO_LCODE(23, 26)}, + {2, TO_LCODE(27, 30)}, + {2, TO_LCODE(31, 34)}, + {3, TO_LCODE(35, 42)}, + {3, TO_LCODE(43, 50)}, + {3, TO_LCODE(51, 58)}, + {3, TO_LCODE(59, 66)}, + {4, TO_LCODE(67, 82)}, + {4, TO_LCODE(83, 98)}, + {4, TO_LCODE(99, 114)}, + {4, TO_LCODE(115, 130)}, + {5, TO_LCODE(131, 162)}, + {5, TO_LCODE(163, 194)}, + {5, TO_LCODE(195, 226)}, + {5, TO_LCODE(227, 257)}, + {0, TO_LCODE(258, 258)}, +}; + +static const dist_coderecord distcodes[] = { + {0, 0, 1, 1}, + {1, 0, 2, 2}, + {2, 0, 3, 3}, + {3, 0, 4, 4}, + {4, 1, 5, 6}, + {5, 1, 7, 8}, + {6, 2, 9, 12}, + {7, 2, 13, 16}, + {8, 3, 17, 24}, + {9, 3, 25, 32}, + {10, 4, 33, 48}, + {11, 4, 49, 64}, + {12, 5, 65, 96}, + {13, 5, 97, 128}, + {14, 6, 129, 192}, + {15, 6, 193, 256}, + {16, 7, 257, 384}, + {17, 7, 385, 512}, + {18, 8, 513, 768}, + {19, 8, 769, 1024}, + {20, 9, 1025, 1536}, + {21, 9, 1537, 2048}, + {22, 10, 2049, 3072}, + {23, 10, 3073, 4096}, + {24, 11, 4097, 6144}, + {25, 11, 6145, 8192}, + {26, 12, 8193, 12288}, + {27, 12, 12289, 16384}, + {28, 13, 16385, 24576}, + {29, 13, 24577, 32768}, +}; + +void zlib_literal(struct Outbuf *out, unsigned char c) +{ + if (out->comp_disabled) { + /* + * We're in an uncompressed block, so just output the byte. + */ + outbits(out, c, 8); + return; + } + + if (c <= 143) { + /* 0 through 143 are 8 bits long starting at 00110000. */ + outbits(out, mirrorbytes[0x30 + c], 8); + } else { + /* 144 through 255 are 9 bits long starting at 110010000. */ + outbits(out, 1 + 2 * mirrorbytes[0x90 - 144 + c], 9); + } +} + +void zlib_match(struct Outbuf *out, int distance, int len) +{ + const dist_coderecord *d; + const len_coderecord *l; + int i, j, k; + int lcode; + + assert(!out->comp_disabled); + + while (len > 0) { + int thislen; + + /* + * We can transmit matches of lengths 3 through 258 + * inclusive. So if len exceeds 258, we must transmit in + * several steps, with 258 or less in each step. + * + * Specifically: if len >= 261, we can transmit 258 and be + * sure of having at least 3 left for the next step. And if + * len <= 258, we can just transmit len. But if len == 259 + * or 260, we must transmit len-3. + */ + thislen = (len > 260 ? 258 : len <= 258 ? len : len - 3); + len -= thislen; + + /* + * Binary-search to find which length code we're + * transmitting. + */ + i = -1; + j = sizeof(lencodes) / sizeof(*lencodes); + while (1) { + assert(j - i >= 2); + k = (j + i) / 2; + if (thislen < FROM_LCODE(lencodes[k].min)) + j = k; + else if (thislen > FROM_LCODE(lencodes[k].max)) + i = k; + else { + l = &lencodes[k]; + break; /* found it! */ + } + } + + lcode = l - lencodes + 257; + + /* + * Transmit the length code. 256-279 are seven bits + * starting at 0000000; 280-287 are eight bits starting at + * 11000000. + */ + if (lcode <= 279) { + outbits(out, mirrorbytes[(lcode - 256) * 2], 7); + } else { + outbits(out, mirrorbytes[0xc0 - 280 + lcode], 8); + } + + /* + * Transmit the extra bits. + */ + if (l->extrabits) + outbits(out, thislen - FROM_LCODE(l->min), l->extrabits); + + /* + * Binary-search to find which distance code we're + * transmitting. + */ + i = -1; + j = sizeof(distcodes) / sizeof(*distcodes); + while (1) { + assert(j - i >= 2); + k = (j + i) / 2; + if (distance < distcodes[k].min) + j = k; + else if (distance > distcodes[k].max) + i = k; + else { + d = &distcodes[k]; + break; /* found it! */ + } + } + + /* + * Transmit the distance code. Five bits starting at 00000. + */ + outbits(out, mirrorbytes[d->code * 8], 5); + + /* + * Transmit the extra bits. + */ + if (d->extrabits) + outbits(out, distance - d->min, d->extrabits); + } +} + +void zlib_start_block(struct Outbuf *out) +{ +// outbits(out, 0x9C78, 16); + outbits(out, 1, 1); /* Final block */ + outbits(out, 1, 2); /* Static huffman block */ +} + +void zlib_finish_block(struct Outbuf *out) +{ + outbits(out, 0, 7); /* close block */ + outbits(out, 0, 7); /* Make sure all bits are flushed */ +} diff --git a/lib/uzlib/defl_static.h b/lib/uzlib/defl_static.h index 292734d77324..94741c97fda3 100644 --- a/lib/uzlib/defl_static.h +++ b/lib/uzlib/defl_static.h @@ -31,14 +31,13 @@ code. */ struct Outbuf { - unsigned char *outbuf; - int outlen, outsize; + void *dest_write_data; + void (*dest_write_cb)(struct Outbuf *outbuf, uint8_t byte); unsigned long outbits; int noutbits; int comp_disabled; }; -void outbits(struct Outbuf *out, unsigned long bits, int nbits); void zlib_start_block(struct Outbuf *ctx); void zlib_finish_block(struct Outbuf *ctx); void zlib_literal(struct Outbuf *ectx, unsigned char c); diff --git a/lib/uzlib/lz77.c b/lib/uzlib/lz77.c new file mode 100644 index 000000000000..285b4ddf91c8 --- /dev/null +++ b/lib/uzlib/lz77.c @@ -0,0 +1,107 @@ +/* + * Simple LZ77 streaming compressor. + * + * The scheme implemented here doesn't use a hash table and instead does a brute + * force search in the history for a previous string. It is relatively slow + * (but still O(N)) but gives good compression and minimal memory usage. For a + * small history window (eg 256 bytes) it's not too slow and compresses well. + * + * MIT license; Copyright (c) 2021 Damien P. George + */ + +#include "uzlib.h" + +#define MATCH_LEN_MIN (3) +#define MATCH_LEN_MAX (258) + +// hist should be a preallocated buffer of hist_max size bytes. +// hist_max should be greater than 0 a power of 2 (ie 1, 2, 4, 8, ...). +// It's possible to pass in hist=NULL, and then the history window will be taken from the +// src passed in to uzlib_lz77_compress (this is useful when not doing streaming compression). +void uzlib_lz77_init(struct uzlib_lz77_state *state, uint8_t *hist, size_t hist_max) { + memset(&state->outbuf, 0, sizeof(state->outbuf)); + state->hist_buf = hist; + state->hist_max = hist_max; + state->hist_start = 0; + state->hist_len = 0; +} + +// Push the given byte to the history. +// Search back in the history for the maximum match of the given src data, +// with support for searching beyond the end of the history and into the src buffer +// (effectively the history and src buffer are concatenated). +static size_t uzlib_lz77_search_max_match(struct uzlib_lz77_state *state, const uint8_t *src, size_t len, size_t *longest_offset) { + size_t longest_len = 0; + for (size_t hist_search = 0; hist_search < state->hist_len; ++hist_search) { + size_t match_len; + for (match_len = 0; match_len <= MATCH_LEN_MAX && match_len < len; ++match_len) { + uint8_t hist; + if (hist_search + match_len < state->hist_len) { + hist = state->hist_buf[(state->hist_start + hist_search + match_len) & (state->hist_max - 1)]; + } else { + hist = src[hist_search + match_len - state->hist_len]; + } + if (src[match_len] != hist) { + break; + } + } + if (match_len >= MATCH_LEN_MIN && match_len > longest_len) { + longest_len = match_len; + *longest_offset = state->hist_len - hist_search; + } + } + + return longest_len; +} + +// Compress the given chunk of data. +void uzlib_lz77_compress(struct uzlib_lz77_state *state, const uint8_t *src, unsigned len) { + bool use_src_as_history = false; + if (state->hist_buf == NULL) { + use_src_as_history = true; + state->hist_buf = (uint8_t *)src; + state->hist_len = 0; + } + + const uint8_t *top = src + len; + while (src < top) { + // Look for a match in the history window. + size_t match_offset = 0; + size_t match_len = uzlib_lz77_search_max_match(state, src, top - src, &match_offset); + + // Encode the literal byte or the match. + if (match_len == 0) { + zlib_literal(&state->outbuf, *src); + match_len = 1; + } else { + zlib_match(&state->outbuf, match_offset, match_len); + } + + // Advance the history window. + if (use_src_as_history) { + // Use src as the history, so advance it. + state->hist_len += match_len; + if (state->hist_len > state->hist_max) { + state->hist_buf += state->hist_len - state->hist_max; + state->hist_len = state->hist_max; + } + src += match_len; + } else { + // Push the bytes into the history buffer. + size_t mask = state->hist_max - 1; + while (match_len--) { + uint8_t b = *src++; + state->hist_buf[(state->hist_start + state->hist_len) & mask] = b; + if (state->hist_len == state->hist_max) { + state->hist_start = (state->hist_start + 1) & mask; + } else { + ++state->hist_len; + } + } + } + } + + if (use_src_as_history) { + state->hist_buf = NULL; + } +} diff --git a/lib/uzlib/uzlib.h b/lib/uzlib/uzlib.h index 3a4a1ad160da..83dddcd47776 100644 --- a/lib/uzlib/uzlib.h +++ b/lib/uzlib/uzlib.h @@ -143,17 +143,16 @@ int TINFCC uzlib_gzip_parse_header(TINF_DATA *d); /* Compression API */ -typedef const uint8_t *uzlib_hash_entry_t; - -struct uzlib_comp { - struct Outbuf out; - - uzlib_hash_entry_t *hash_table; - unsigned int hash_bits; - unsigned int dict_size; +struct uzlib_lz77_state { + struct Outbuf outbuf; + uint8_t *hist_buf; + size_t hist_max; + size_t hist_start; + size_t hist_len; }; -void TINFCC uzlib_compress(struct uzlib_comp *c, const uint8_t *src, unsigned slen); +void TINFCC uzlib_lz77_init(struct uzlib_lz77_state *state, uint8_t *hist, size_t hist_max); +void TINFCC uzlib_lz77_compress(struct uzlib_lz77_state *state, const uint8_t *src, unsigned len); /* Checksum API */ From 82db9926ed3546fa98e56b8c06936fed21f492aa Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Tue, 27 Jun 2023 22:03:52 +1000 Subject: [PATCH 04/31] lib/uzlib/lz77: Always use separate history buffer. Because we only use the streaming source, this is just extra code size. Saves 64 bytes on PYBV11. Signed-off-by: Jim Mussared --- lib/uzlib/lz77.c | 40 +++++++++------------------------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/lib/uzlib/lz77.c b/lib/uzlib/lz77.c index 285b4ddf91c8..1c7a8442ca85 100644 --- a/lib/uzlib/lz77.c +++ b/lib/uzlib/lz77.c @@ -56,13 +56,6 @@ static size_t uzlib_lz77_search_max_match(struct uzlib_lz77_state *state, const // Compress the given chunk of data. void uzlib_lz77_compress(struct uzlib_lz77_state *state, const uint8_t *src, unsigned len) { - bool use_src_as_history = false; - if (state->hist_buf == NULL) { - use_src_as_history = true; - state->hist_buf = (uint8_t *)src; - state->hist_len = 0; - } - const uint8_t *top = src + len; while (src < top) { // Look for a match in the history window. @@ -77,31 +70,16 @@ void uzlib_lz77_compress(struct uzlib_lz77_state *state, const uint8_t *src, uns zlib_match(&state->outbuf, match_offset, match_len); } - // Advance the history window. - if (use_src_as_history) { - // Use src as the history, so advance it. - state->hist_len += match_len; - if (state->hist_len > state->hist_max) { - state->hist_buf += state->hist_len - state->hist_max; - state->hist_len = state->hist_max; - } - src += match_len; - } else { - // Push the bytes into the history buffer. - size_t mask = state->hist_max - 1; - while (match_len--) { - uint8_t b = *src++; - state->hist_buf[(state->hist_start + state->hist_len) & mask] = b; - if (state->hist_len == state->hist_max) { - state->hist_start = (state->hist_start + 1) & mask; - } else { - ++state->hist_len; - } + // Push the bytes into the history buffer. + size_t mask = state->hist_max - 1; + while (match_len--) { + uint8_t b = *src++; + state->hist_buf[(state->hist_start + state->hist_len) & mask] = b; + if (state->hist_len == state->hist_max) { + state->hist_start = (state->hist_start + 1) & mask; + } else { + ++state->hist_len; } } } - - if (use_src_as_history) { - state->hist_buf = NULL; - } } From 0900976384f2f881dc727f7b936c2b232e1ee6be Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Thu, 22 Jun 2023 14:24:42 +1000 Subject: [PATCH 05/31] lib/uzlib/defl_static: Implement some code size improvements. This commit makes the following changes: - Replace 256-byte reverse-bits-in-byte lookup table with computation. - Replace length and distance code lookup tables with computation. - Remove comp_disabled check (it's unused). - Make the dest_write_cb take the data pointer directly, rather than the Outbuf. Saves 500 bytes on PYBV11. Signed-off-by: Jim Mussared --- lib/uzlib/defl_static.c | 218 +++++++++------------------------------- lib/uzlib/defl_static.h | 3 +- 2 files changed, 49 insertions(+), 172 deletions(-) diff --git a/lib/uzlib/defl_static.c b/lib/uzlib/defl_static.c index 89b5e1aa5ff6..529055ae847d 100644 --- a/lib/uzlib/defl_static.c +++ b/lib/uzlib/defl_static.c @@ -10,6 +10,9 @@ Delchev, Andreas Schultz, Jeroen Massar, Wez Furlong, Nicolas Barry, Justin Bradford, Ben Harris, Malcolm Smith, Ahmad Khalifa, Markus Kuhn, Colin Watson, and CORE SDI S.A. +Optimised for MicroPython: +Copyright (c) 2023 by Jim Mussared + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, @@ -55,152 +58,44 @@ static void outbits(struct Outbuf *out, unsigned long bits, int nbits) out->outbits |= bits << out->noutbits; out->noutbits += nbits; while (out->noutbits >= 8) { - out->dest_write_cb(out, out->outbits & 0xFF); + out->dest_write_cb(out->dest_write_data, out->outbits & 0xFF); out->outbits >>= 8; out->noutbits -= 8; } } -static const unsigned char mirrorbytes[256] = { - 0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, - 0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0, - 0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8, - 0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8, - 0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4, - 0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4, - 0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec, - 0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc, - 0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2, - 0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2, - 0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea, - 0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa, - 0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6, - 0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6, - 0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee, - 0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe, - 0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1, - 0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1, - 0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9, - 0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9, - 0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5, - 0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5, - 0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed, - 0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd, - 0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3, - 0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3, - 0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb, - 0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb, - 0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7, - 0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7, - 0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef, - 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff, +static const unsigned char mirrornibbles[16] = { + 0x0, 0x8, 0x4, 0xc, 0x2, 0xa, 0x6, 0xe, + 0x1, 0x9, 0x5, 0xd, 0x3, 0xb, 0x7, 0xf, }; -typedef struct { - uint8_t extrabits; - uint8_t min, max; -} len_coderecord; - -typedef struct { - uint8_t code, extrabits; - uint16_t min, max; -} dist_coderecord; - -#define TO_LCODE(x, y) x - 3, y - 3 -#define FROM_LCODE(x) (x + 3) - -static const len_coderecord lencodes[] = { - {0, TO_LCODE(3, 3)}, - {0, TO_LCODE(4, 4)}, - {0, TO_LCODE(5, 5)}, - {0, TO_LCODE(6, 6)}, - {0, TO_LCODE(7, 7)}, - {0, TO_LCODE(8, 8)}, - {0, TO_LCODE(9, 9)}, - {0, TO_LCODE(10, 10)}, - {1, TO_LCODE(11, 12)}, - {1, TO_LCODE(13, 14)}, - {1, TO_LCODE(15, 16)}, - {1, TO_LCODE(17, 18)}, - {2, TO_LCODE(19, 22)}, - {2, TO_LCODE(23, 26)}, - {2, TO_LCODE(27, 30)}, - {2, TO_LCODE(31, 34)}, - {3, TO_LCODE(35, 42)}, - {3, TO_LCODE(43, 50)}, - {3, TO_LCODE(51, 58)}, - {3, TO_LCODE(59, 66)}, - {4, TO_LCODE(67, 82)}, - {4, TO_LCODE(83, 98)}, - {4, TO_LCODE(99, 114)}, - {4, TO_LCODE(115, 130)}, - {5, TO_LCODE(131, 162)}, - {5, TO_LCODE(163, 194)}, - {5, TO_LCODE(195, 226)}, - {5, TO_LCODE(227, 257)}, - {0, TO_LCODE(258, 258)}, -}; +static unsigned int mirrorbyte(unsigned int b) { + return mirrornibbles[b & 0xf] << 4 | mirrornibbles[b >> 4]; +} -static const dist_coderecord distcodes[] = { - {0, 0, 1, 1}, - {1, 0, 2, 2}, - {2, 0, 3, 3}, - {3, 0, 4, 4}, - {4, 1, 5, 6}, - {5, 1, 7, 8}, - {6, 2, 9, 12}, - {7, 2, 13, 16}, - {8, 3, 17, 24}, - {9, 3, 25, 32}, - {10, 4, 33, 48}, - {11, 4, 49, 64}, - {12, 5, 65, 96}, - {13, 5, 97, 128}, - {14, 6, 129, 192}, - {15, 6, 193, 256}, - {16, 7, 257, 384}, - {17, 7, 385, 512}, - {18, 8, 513, 768}, - {19, 8, 769, 1024}, - {20, 9, 1025, 1536}, - {21, 9, 1537, 2048}, - {22, 10, 2049, 3072}, - {23, 10, 3073, 4096}, - {24, 11, 4097, 6144}, - {25, 11, 6145, 8192}, - {26, 12, 8193, 12288}, - {27, 12, 12289, 16384}, - {28, 13, 16385, 24576}, - {29, 13, 24577, 32768}, -}; +static int int_log2(int x) { + int r = 0; + while (x >>= 1) { + ++r; + } + return r; +} void zlib_literal(struct Outbuf *out, unsigned char c) { - if (out->comp_disabled) { - /* - * We're in an uncompressed block, so just output the byte. - */ - outbits(out, c, 8); - return; - } - if (c <= 143) { /* 0 through 143 are 8 bits long starting at 00110000. */ - outbits(out, mirrorbytes[0x30 + c], 8); + outbits(out, mirrorbyte(0x30 + c), 8); } else { /* 144 through 255 are 9 bits long starting at 110010000. */ - outbits(out, 1 + 2 * mirrorbytes[0x90 - 144 + c], 9); + outbits(out, 1 + 2 * mirrorbyte(0x90 - 144 + c), 9); } } void zlib_match(struct Outbuf *out, int distance, int len) { - const dist_coderecord *d; - const len_coderecord *l; - int i, j, k; - int lcode; - - assert(!out->comp_disabled); + assert(distance >= 1 && distance <= 32768); + distance -= 1; while (len > 0) { int thislen; @@ -218,73 +113,56 @@ void zlib_match(struct Outbuf *out, int distance, int len) thislen = (len > 260 ? 258 : len <= 258 ? len : len - 3); len -= thislen; - /* - * Binary-search to find which length code we're - * transmitting. - */ - i = -1; - j = sizeof(lencodes) / sizeof(*lencodes); - while (1) { - assert(j - i >= 2); - k = (j + i) / 2; - if (thislen < FROM_LCODE(lencodes[k].min)) - j = k; - else if (thislen > FROM_LCODE(lencodes[k].max)) - i = k; - else { - l = &lencodes[k]; - break; /* found it! */ + assert(thislen >= 3 && thislen <= 258); + + thislen -= 3; + int lcode = 28; + int x = int_log2(thislen); + int y; + if (thislen < 255) { + if (x) { + --x; } + y = (thislen >> (x ? x - 1 : 0)) & 3; + lcode = x * 4 + y; } - lcode = l - lencodes + 257; - /* * Transmit the length code. 256-279 are seven bits * starting at 0000000; 280-287 are eight bits starting at * 11000000. */ - if (lcode <= 279) { - outbits(out, mirrorbytes[(lcode - 256) * 2], 7); + if (lcode <= 22) { + outbits(out, mirrorbyte((lcode + 1) * 2), 7); } else { - outbits(out, mirrorbytes[0xc0 - 280 + lcode], 8); + outbits(out, mirrorbyte(lcode + 169), 8); } /* * Transmit the extra bits. */ - if (l->extrabits) - outbits(out, thislen - FROM_LCODE(l->min), l->extrabits); - - /* - * Binary-search to find which distance code we're - * transmitting. - */ - i = -1; - j = sizeof(distcodes) / sizeof(*distcodes); - while (1) { - assert(j - i >= 2); - k = (j + i) / 2; - if (distance < distcodes[k].min) - j = k; - else if (distance > distcodes[k].max) - i = k; - else { - d = &distcodes[k]; - break; /* found it! */ - } + if (thislen < 255 && x > 1) { + int extrabits = x - 1; + int lmin = (thislen >> extrabits) << extrabits; + outbits(out, thislen - lmin, extrabits); } + x = int_log2(distance); + y = (distance >> (x ? x - 1 : 0)) & 1; + /* * Transmit the distance code. Five bits starting at 00000. */ - outbits(out, mirrorbytes[d->code * 8], 5); + outbits(out, mirrorbyte((x * 2 + y) * 8), 5); /* * Transmit the extra bits. */ - if (d->extrabits) - outbits(out, distance - d->min, d->extrabits); + if (x > 1) { + int dextrabits = x - 1; + int dmin = (distance >> dextrabits) << dextrabits; + outbits(out, distance - dmin, dextrabits); + } } } diff --git a/lib/uzlib/defl_static.h b/lib/uzlib/defl_static.h index 94741c97fda3..7329de36efa0 100644 --- a/lib/uzlib/defl_static.h +++ b/lib/uzlib/defl_static.h @@ -32,10 +32,9 @@ struct Outbuf { void *dest_write_data; - void (*dest_write_cb)(struct Outbuf *outbuf, uint8_t byte); + void (*dest_write_cb)(void *data, uint8_t byte); unsigned long outbits; int noutbits; - int comp_disabled; }; void zlib_start_block(struct Outbuf *ctx); From c2b8e6e5d685ce0ef7cd7186f326cac8a39eb2a3 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Tue, 27 Jun 2023 00:50:05 +1000 Subject: [PATCH 06/31] lib/uzlib: Clean up tinf -> uzlib rename. This library used a mix of "tinf" and "uzlib" to refer to itself. Remove all use of "tinf" in the public API. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- lib/uzlib/adler32.c | 2 +- lib/uzlib/crc32.c | 2 +- lib/uzlib/tinf.h | 3 -- lib/uzlib/tinf_compat.h | 9 ---- lib/uzlib/tinfgzip.c | 24 ++++----- lib/uzlib/tinflate.c | 98 ++++++++++++++++++------------------ lib/uzlib/tinfzlib.c | 14 +++--- lib/uzlib/uzlib.h | 58 +++++++++------------ ports/stm32/mboot/fsload.c | 2 +- ports/stm32/mboot/gzstream.c | 42 ++++++++-------- 10 files changed, 117 insertions(+), 137 deletions(-) delete mode 100644 lib/uzlib/tinf.h delete mode 100644 lib/uzlib/tinf_compat.h diff --git a/lib/uzlib/adler32.c b/lib/uzlib/adler32.c index 1f1759493bba..65fe1181b860 100644 --- a/lib/uzlib/adler32.c +++ b/lib/uzlib/adler32.c @@ -36,7 +36,7 @@ * Copyright (C) 1995-1998 Jean-loup Gailly and Mark Adler */ -#include "tinf.h" +#include "uzlib.h" #define A32_BASE 65521 #define A32_NMAX 5552 diff --git a/lib/uzlib/crc32.c b/lib/uzlib/crc32.c index e24c643b6acf..1e3b1756b383 100644 --- a/lib/uzlib/crc32.c +++ b/lib/uzlib/crc32.c @@ -36,7 +36,7 @@ * Copyright (C) 1995-1998 Jean-loup Gailly and Mark Adler */ -#include "tinf.h" +#include "uzlib.h" static const unsigned int tinf_crc32tab[16] = { 0x00000000, 0x1db71064, 0x3b6e20c8, 0x26d930ac, 0x76dc4190, diff --git a/lib/uzlib/tinf.h b/lib/uzlib/tinf.h deleted file mode 100644 index ae6e1c4073eb..000000000000 --- a/lib/uzlib/tinf.h +++ /dev/null @@ -1,3 +0,0 @@ -/* Compatibility header for the original tinf lib/older versions of uzlib. - Note: may be removed in the future, please migrate to uzlib.h. */ -#include "uzlib.h" diff --git a/lib/uzlib/tinf_compat.h b/lib/uzlib/tinf_compat.h deleted file mode 100644 index f763804bd94b..000000000000 --- a/lib/uzlib/tinf_compat.h +++ /dev/null @@ -1,9 +0,0 @@ -/* This header contains compatibility defines for the original tinf API - and uzlib 2.x and below API. These defines are deprecated and going - to be removed in the future, so applications should migrate to new - uzlib API. */ -#define TINF_DATA struct uzlib_uncomp - -#define destSize dest_size -#define destStart dest_start -#define readSource source_read_cb diff --git a/lib/uzlib/tinfgzip.c b/lib/uzlib/tinfgzip.c index 22b000df9aa2..3b0feb532122 100644 --- a/lib/uzlib/tinfgzip.c +++ b/lib/uzlib/tinfgzip.c @@ -33,7 +33,7 @@ * any source distribution. */ -#include "tinf.h" +#include "uzlib.h" #define FTEXT 1 #define FHCRC 2 @@ -41,38 +41,38 @@ #define FNAME 8 #define FCOMMENT 16 -void tinf_skip_bytes(TINF_DATA *d, int num); -uint16_t tinf_get_uint16(TINF_DATA *d); +void tinf_skip_bytes(uzlib_uncomp_t *d, int num); +uint16_t tinf_get_uint16(uzlib_uncomp_t *d); -void tinf_skip_bytes(TINF_DATA *d, int num) +void tinf_skip_bytes(uzlib_uncomp_t *d, int num) { while (num--) uzlib_get_byte(d); } -uint16_t tinf_get_uint16(TINF_DATA *d) +uint16_t tinf_get_uint16(uzlib_uncomp_t *d) { unsigned int v = uzlib_get_byte(d); v = (uzlib_get_byte(d) << 8) | v; return v; } -int uzlib_gzip_parse_header(TINF_DATA *d) +int uzlib_gzip_parse_header(uzlib_uncomp_t *d) { unsigned char flg; /* -- check format -- */ /* check id bytes */ - if (uzlib_get_byte(d) != 0x1f || uzlib_get_byte(d) != 0x8b) return TINF_DATA_ERROR; + if (uzlib_get_byte(d) != 0x1f || uzlib_get_byte(d) != 0x8b) return UZLIB_DATA_ERROR; /* check method is deflate */ - if (uzlib_get_byte(d) != 8) return TINF_DATA_ERROR; + if (uzlib_get_byte(d) != 8) return UZLIB_DATA_ERROR; /* get flag byte */ flg = uzlib_get_byte(d); /* check that reserved bits are zero */ - if (flg & 0xe0) return TINF_DATA_ERROR; + if (flg & 0xe0) return UZLIB_DATA_ERROR; /* -- find start of compressed data -- */ @@ -99,12 +99,12 @@ int uzlib_gzip_parse_header(TINF_DATA *d) // TODO: Check! // if (hcrc != (tinf_crc32(src, start - src) & 0x0000ffff)) -// return TINF_DATA_ERROR; +// return UZLIB_DATA_ERROR; } /* initialize for crc32 checksum */ - d->checksum_type = TINF_CHKSUM_CRC; + d->checksum_type = UZLIB_CHKSUM_CRC; d->checksum = ~0; - return TINF_OK; + return UZLIB_OK; } diff --git a/lib/uzlib/tinflate.c b/lib/uzlib/tinflate.c index 045952c7553a..e844d7faa491 100644 --- a/lib/uzlib/tinflate.c +++ b/lib/uzlib/tinflate.c @@ -33,7 +33,7 @@ */ #include -#include "tinf.h" +#include "uzlib.h" #define UZLIB_DUMP_ARRAY(heading, arr, size) \ { \ @@ -44,8 +44,8 @@ printf("\n"); \ } -uint32_t tinf_get_le_uint32(TINF_DATA *d); -uint32_t tinf_get_be_uint32(TINF_DATA *d); +uint32_t tinf_get_le_uint32(uzlib_uncomp_t *d); +uint32_t tinf_get_be_uint32(uzlib_uncomp_t *d); /* --------------------------------------------------- * * -- uninitialized global data (static structures) -- * @@ -189,7 +189,7 @@ static void tinf_build_tree(TINF_TREE *t, const unsigned char *lengths, unsigned * -- decode functions -- * * ---------------------- */ -unsigned char uzlib_get_byte(TINF_DATA *d) +unsigned char uzlib_get_byte(uzlib_uncomp_t *d) { /* If end of source buffer is not reached, return next byte from source buffer. */ @@ -200,14 +200,14 @@ unsigned char uzlib_get_byte(TINF_DATA *d) /* Otherwise if there's callback and we haven't seen EOF yet, try to read next byte using it. (Note: the callback can also update ->source and ->source_limit). */ - if (d->readSource && !d->eof) { - int val = d->readSource(d); + if (d->source_read_cb && !d->eof) { + int val = d->source_read_cb(d); if (val >= 0) { return (unsigned char)val; } } - /* Otherwise, we hit EOF (either from ->readSource() or from exhaustion + /* Otherwise, we hit EOF (either from ->source_read_cb() or from exhaustion of the buffer), and it will be "sticky", i.e. further calls to this function will end up here too. */ d->eof = true; @@ -215,7 +215,7 @@ unsigned char uzlib_get_byte(TINF_DATA *d) return 0; } -uint32_t tinf_get_le_uint32(TINF_DATA *d) +uint32_t tinf_get_le_uint32(uzlib_uncomp_t *d) { uint32_t val = 0; int i; @@ -225,7 +225,7 @@ uint32_t tinf_get_le_uint32(TINF_DATA *d) return val; } -uint32_t tinf_get_be_uint32(TINF_DATA *d) +uint32_t tinf_get_be_uint32(uzlib_uncomp_t *d) { uint32_t val = 0; int i; @@ -236,7 +236,7 @@ uint32_t tinf_get_be_uint32(TINF_DATA *d) } /* get one bit from source stream */ -static int tinf_getbit(TINF_DATA *d) +static int tinf_getbit(uzlib_uncomp_t *d) { unsigned int bit; @@ -256,7 +256,7 @@ static int tinf_getbit(TINF_DATA *d) } /* read a num bit value from a stream and add base */ -static unsigned int tinf_read_bits(TINF_DATA *d, int num, int base) +static unsigned int tinf_read_bits(uzlib_uncomp_t *d, int num, int base) { unsigned int val = 0; @@ -274,7 +274,7 @@ static unsigned int tinf_read_bits(TINF_DATA *d, int num, int base) } /* given a data stream and a tree, decode a symbol */ -static int tinf_decode_symbol(TINF_DATA *d, TINF_TREE *t) +static int tinf_decode_symbol(uzlib_uncomp_t *d, TINF_TREE *t) { int sum = 0, cur = 0, len = 0; @@ -284,7 +284,7 @@ static int tinf_decode_symbol(TINF_DATA *d, TINF_TREE *t) cur = 2*cur + tinf_getbit(d); if (++len == TINF_ARRAY_SIZE(t->table)) { - return TINF_DATA_ERROR; + return UZLIB_DATA_ERROR; } sum += t->table[len]; @@ -295,7 +295,7 @@ static int tinf_decode_symbol(TINF_DATA *d, TINF_TREE *t) sum += cur; #if UZLIB_CONF_PARANOID_CHECKS if (sum < 0 || sum >= TINF_ARRAY_SIZE(t->trans)) { - return TINF_DATA_ERROR; + return UZLIB_DATA_ERROR; } #endif @@ -303,7 +303,7 @@ static int tinf_decode_symbol(TINF_DATA *d, TINF_TREE *t) } /* given a data stream, decode dynamic trees from it */ -static int tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) +static int tinf_decode_trees(uzlib_uncomp_t *d, TINF_TREE *lt, TINF_TREE *dt) { /* code lengths for 288 literal/len symbols and 32 dist symbols */ unsigned char lengths[288+32]; @@ -348,7 +348,7 @@ static int tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) { case 16: /* copy previous code length 3-6 times (read 2 bits) */ - if (num == 0) return TINF_DATA_ERROR; + if (num == 0) return UZLIB_DATA_ERROR; fill_value = lengths[num - 1]; lbits = 2; break; @@ -370,7 +370,7 @@ static int tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) /* special code length 16-18 are handled here */ length = tinf_read_bits(d, lbits, lbase); - if (num + length > hlimit) return TINF_DATA_ERROR; + if (num + length > hlimit) return UZLIB_DATA_ERROR; for (; length; --length) { lengths[num++] = fill_value; @@ -387,7 +387,7 @@ static int tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) #if UZLIB_CONF_PARANOID_CHECKS /* Check that there's "end of block" symbol */ if (lengths[256] == 0) { - return TINF_DATA_ERROR; + return UZLIB_DATA_ERROR; } #endif @@ -395,7 +395,7 @@ static int tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) tinf_build_tree(lt, lengths, hlit); tinf_build_tree(dt, lengths + hlit, hdist); - return TINF_OK; + return UZLIB_OK; } /* ----------------------------- * @@ -403,7 +403,7 @@ static int tinf_decode_trees(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) * ----------------------------- */ /* given a stream and two trees, inflate next byte of output */ -static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) +static int tinf_inflate_block_data(uzlib_uncomp_t *d, TINF_TREE *lt, TINF_TREE *dt) { if (d->curlen == 0) { unsigned int offs; @@ -412,24 +412,24 @@ static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) //printf("huff sym: %02x\n", sym); if (d->eof) { - return TINF_DATA_ERROR; + return UZLIB_DATA_ERROR; } /* literal byte */ if (sym < 256) { TINF_PUT(d, sym); - return TINF_OK; + return UZLIB_OK; } /* end of block */ if (sym == 256) { - return TINF_DONE; + return UZLIB_DONE; } /* substring from sliding dictionary */ sym -= 257; if (sym >= 29) { - return TINF_DATA_ERROR; + return UZLIB_DATA_ERROR; } /* possibly get more bits from length code */ @@ -437,7 +437,7 @@ static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) dist = tinf_decode_symbol(d, dt); if (dist >= 30) { - return TINF_DATA_ERROR; + return UZLIB_DATA_ERROR; } /* possibly get more bits from distance code */ @@ -446,7 +446,7 @@ static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) /* calculate and validate actual LZ offset to use */ if (d->dict_ring) { if (offs > d->dict_size) { - return TINF_DICT_ERROR; + return UZLIB_DICT_ERROR; } /* Note: unlike full-dest-in-memory case below, we don't try to catch offset which points to not yet filled @@ -464,8 +464,8 @@ static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) } } else { /* catch trying to point before the start of dest buffer */ - if (offs > (unsigned int)(d->dest - d->destStart)) { - return TINF_DATA_ERROR; + if (offs > (unsigned int)(d->dest - d->dest_start)) { + return UZLIB_DATA_ERROR; } d->lzOff = -offs; } @@ -482,11 +482,11 @@ static int tinf_inflate_block_data(TINF_DATA *d, TINF_TREE *lt, TINF_TREE *dt) d->dest++; } d->curlen--; - return TINF_OK; + return UZLIB_OK; } /* inflate next byte from uncompressed block of data */ -static int tinf_inflate_uncompressed_block(TINF_DATA *d) +static int tinf_inflate_uncompressed_block(uzlib_uncomp_t *d) { if (d->curlen == 0) { unsigned int length, invlength; @@ -498,9 +498,9 @@ static int tinf_inflate_uncompressed_block(TINF_DATA *d) invlength = uzlib_get_byte(d); invlength += 256 * uzlib_get_byte(d); /* check length */ - if (length != (~invlength & 0x0000ffff)) return TINF_DATA_ERROR; + if (length != (~invlength & 0x0000ffff)) return UZLIB_DATA_ERROR; - /* increment length to properly return TINF_DONE below, without + /* increment length to properly return UZLIB_DONE below, without producing data at the same time */ d->curlen = length + 1; @@ -509,12 +509,12 @@ static int tinf_inflate_uncompressed_block(TINF_DATA *d) } if (--d->curlen == 0) { - return TINF_DONE; + return UZLIB_DONE; } unsigned char c = uzlib_get_byte(d); TINF_PUT(d, c); - return TINF_OK; + return UZLIB_OK; } /* ---------------------- * @@ -536,7 +536,7 @@ void uzlib_init(void) } /* initialize decompression structure */ -void uzlib_uncompress_init(TINF_DATA *d, void *dict, unsigned int dictLen) +void uzlib_uncompress_init(uzlib_uncomp_t *d, void *dict, unsigned int dictLen) { d->eof = 0; d->bitcount = 0; @@ -549,7 +549,7 @@ void uzlib_uncompress_init(TINF_DATA *d, void *dict, unsigned int dictLen) } /* inflate next output bytes from compressed stream */ -int uzlib_uncompress(TINF_DATA *d) +int uzlib_uncompress(uzlib_uncomp_t *d) { do { int res; @@ -572,7 +572,7 @@ int uzlib_uncompress(TINF_DATA *d) } else if (d->btype == 2) { /* decode trees from stream */ res = tinf_decode_trees(d, &d->ltree, &d->dtree); - if (res != TINF_OK) { + if (res != UZLIB_OK) { return res; } } @@ -592,27 +592,27 @@ int uzlib_uncompress(TINF_DATA *d) res = tinf_inflate_block_data(d, &d->ltree, &d->dtree); break; default: - return TINF_DATA_ERROR; + return UZLIB_DATA_ERROR; } - if (res == TINF_DONE && !d->bfinal) { + if (res == UZLIB_DONE && !d->bfinal) { /* the block has ended (without producing more data), but we can't return without data, so start procesing next block */ goto next_blk; } - if (res != TINF_OK) { + if (res != UZLIB_OK) { return res; } } while (d->dest < d->dest_limit); - return TINF_OK; + return UZLIB_OK; } /* inflate next output bytes from compressed stream, updating checksum, and at the end of stream, verify it */ -int uzlib_uncompress_chksum(TINF_DATA *d) +int uzlib_uncompress_chksum(uzlib_uncomp_t *d) { int res; unsigned char *data = d->dest; @@ -623,31 +623,31 @@ int uzlib_uncompress_chksum(TINF_DATA *d) switch (d->checksum_type) { - case TINF_CHKSUM_ADLER: + case UZLIB_CHKSUM_ADLER: d->checksum = uzlib_adler32(data, d->dest - data, d->checksum); break; - case TINF_CHKSUM_CRC: + case UZLIB_CHKSUM_CRC: d->checksum = uzlib_crc32(data, d->dest - data, d->checksum); break; } - if (res == TINF_DONE) { + if (res == UZLIB_DONE) { unsigned int val; switch (d->checksum_type) { - case TINF_CHKSUM_ADLER: + case UZLIB_CHKSUM_ADLER: val = tinf_get_be_uint32(d); if (d->checksum != val) { - return TINF_CHKSUM_ERROR; + return UZLIB_CHKSUM_ERROR; } break; - case TINF_CHKSUM_CRC: + case UZLIB_CHKSUM_CRC: val = tinf_get_le_uint32(d); if (~d->checksum != val) { - return TINF_CHKSUM_ERROR; + return UZLIB_CHKSUM_ERROR; } // Uncompressed size. TODO: Check val = tinf_get_le_uint32(d); diff --git a/lib/uzlib/tinfzlib.c b/lib/uzlib/tinfzlib.c index 5cb8852fcc4f..03954cefaeba 100644 --- a/lib/uzlib/tinfzlib.c +++ b/lib/uzlib/tinfzlib.c @@ -33,9 +33,9 @@ * any source distribution. */ -#include "tinf.h" +#include "uzlib.h" -int uzlib_zlib_parse_header(TINF_DATA *d) +int uzlib_zlib_parse_header(uzlib_uncomp_t *d) { unsigned char cmf, flg; @@ -47,19 +47,19 @@ int uzlib_zlib_parse_header(TINF_DATA *d) /* -- check format -- */ /* check checksum */ - if ((256*cmf + flg) % 31) return TINF_DATA_ERROR; + if ((256*cmf + flg) % 31) return UZLIB_DATA_ERROR; /* check method is deflate */ - if ((cmf & 0x0f) != 8) return TINF_DATA_ERROR; + if ((cmf & 0x0f) != 8) return UZLIB_DATA_ERROR; /* check window size is valid */ - if ((cmf >> 4) > 7) return TINF_DATA_ERROR; + if ((cmf >> 4) > 7) return UZLIB_DATA_ERROR; /* check there is no preset dictionary */ - if (flg & 0x20) return TINF_DATA_ERROR; + if (flg & 0x20) return UZLIB_DATA_ERROR; /* initialize for adler32 checksum */ - d->checksum_type = TINF_CHKSUM_ADLER; + d->checksum_type = UZLIB_CHKSUM_ADLER; d->checksum = 1; return cmf >> 4; diff --git a/lib/uzlib/uzlib.h b/lib/uzlib/uzlib.h index 83dddcd47776..ca58299fc3ec 100644 --- a/lib/uzlib/uzlib.h +++ b/lib/uzlib/uzlib.h @@ -7,6 +7,9 @@ * * Copyright (c) 2014-2018 by Paul Sokolovsky * + * Optimised for MicroPython: + * Copyright (c) 2023 by Jim Mussared + * * This software is provided 'as-is', without any express * or implied warranty. In no event will the authors be * held liable for any damages arising from the use of @@ -46,31 +49,22 @@ #include #endif -/* calling convention */ -#ifndef TINFCC - #ifdef __WATCOMC__ - #define TINFCC __cdecl - #else - #define TINFCC - #endif -#endif - #ifdef __cplusplus extern "C" { #endif /* ok status, more data produced */ -#define TINF_OK 0 +#define UZLIB_OK 0 /* end of compressed stream reached */ -#define TINF_DONE 1 -#define TINF_DATA_ERROR (-3) -#define TINF_CHKSUM_ERROR (-4) -#define TINF_DICT_ERROR (-5) +#define UZLIB_DONE 1 +#define UZLIB_DATA_ERROR (-3) +#define UZLIB_CHKSUM_ERROR (-4) +#define UZLIB_DICT_ERROR (-5) /* checksum types */ -#define TINF_CHKSUM_NONE 0 -#define TINF_CHKSUM_ADLER 1 -#define TINF_CHKSUM_CRC 2 +#define UZLIB_CHKSUM_NONE 0 +#define UZLIB_CHKSUM_ADLER 1 +#define UZLIB_CHKSUM_CRC 2 /* helper macros */ #define TINF_ARRAY_SIZE(arr) (sizeof(arr) / sizeof(*(arr))) @@ -82,7 +76,7 @@ typedef struct { unsigned short trans[288]; /* code -> symbol translation table */ } TINF_TREE; -struct uzlib_uncomp { +typedef struct _uzlib_uncomp_t { /* Pointer to the next byte in the input buffer */ const unsigned char *source; /* Pointer to the next byte past the input buffer (source_limit = source + len) */ @@ -92,7 +86,7 @@ struct uzlib_uncomp { also return -1 in case of EOF (or irrecoverable error). Note that besides returning the next byte, it may also update source and source_limit fields, thus allowing for buffered operation. */ - int (*source_read_cb)(struct uzlib_uncomp *uncomp); + int (*source_read_cb)(struct _uzlib_uncomp_t *uncomp); unsigned int tag; unsigned int bitcount; @@ -119,9 +113,7 @@ struct uzlib_uncomp { TINF_TREE ltree; /* dynamic length/symbol tree */ TINF_TREE dtree; /* dynamic distance tree */ -}; - -#include "tinf_compat.h" +} uzlib_uncomp_t; #define TINF_PUT(d, c) \ { \ @@ -129,17 +121,17 @@ struct uzlib_uncomp { if (d->dict_ring) { d->dict_ring[d->dict_idx++] = c; if (d->dict_idx == d->dict_size) d->dict_idx = 0; } \ } -unsigned char TINFCC uzlib_get_byte(TINF_DATA *d); +unsigned char uzlib_get_byte(uzlib_uncomp_t *d); /* Decompression API */ -void TINFCC uzlib_init(void); -void TINFCC uzlib_uncompress_init(TINF_DATA *d, void *dict, unsigned int dictLen); -int TINFCC uzlib_uncompress(TINF_DATA *d); -int TINFCC uzlib_uncompress_chksum(TINF_DATA *d); +void uzlib_init(void); +void uzlib_uncompress_init(uzlib_uncomp_t *d, void *dict, unsigned int dictLen); +int uzlib_uncompress(uzlib_uncomp_t *d); +int uzlib_uncompress_chksum(uzlib_uncomp_t *d); -int TINFCC uzlib_zlib_parse_header(TINF_DATA *d); -int TINFCC uzlib_gzip_parse_header(TINF_DATA *d); +int uzlib_zlib_parse_header(uzlib_uncomp_t *d); +int uzlib_gzip_parse_header(uzlib_uncomp_t *d); /* Compression API */ @@ -151,15 +143,15 @@ struct uzlib_lz77_state { size_t hist_len; }; -void TINFCC uzlib_lz77_init(struct uzlib_lz77_state *state, uint8_t *hist, size_t hist_max); -void TINFCC uzlib_lz77_compress(struct uzlib_lz77_state *state, const uint8_t *src, unsigned len); +void uzlib_lz77_init(struct uzlib_lz77_state *state, uint8_t *hist, size_t hist_max); +void uzlib_lz77_compress(struct uzlib_lz77_state *state, const uint8_t *src, unsigned len); /* Checksum API */ /* prev_sum is previous value for incremental computation, 1 initially */ -uint32_t TINFCC uzlib_adler32(const void *data, unsigned int length, uint32_t prev_sum); +uint32_t uzlib_adler32(const void *data, unsigned int length, uint32_t prev_sum); /* crc is previous value for incremental computation, 0xffffffff initially */ -uint32_t TINFCC uzlib_crc32(const void *data, unsigned int length, uint32_t crc); +uint32_t uzlib_crc32(const void *data, unsigned int length, uint32_t crc); #ifdef __cplusplus } /* extern "C" */ diff --git a/ports/stm32/mboot/fsload.c b/ports/stm32/mboot/fsload.c index f3ea57c89fda..abc3514ed146 100644 --- a/ports/stm32/mboot/fsload.c +++ b/ports/stm32/mboot/fsload.c @@ -27,7 +27,7 @@ #include #include "py/mphal.h" -#include "lib/uzlib/tinf.h" +#include "lib/uzlib/uzlib.h" #include "mboot.h" #include "pack.h" #include "vfs.h" diff --git a/ports/stm32/mboot/gzstream.c b/ports/stm32/mboot/gzstream.c index 6ed8a21d9949..f008eca5cd79 100644 --- a/ports/stm32/mboot/gzstream.c +++ b/ports/stm32/mboot/gzstream.c @@ -38,14 +38,14 @@ typedef struct _gz_stream_t { void *stream_data; stream_read_t stream_read; - struct uzlib_uncomp tinf; + uzlib_uncomp_t decomp; uint8_t buf[512]; uint8_t dict[DICT_SIZE]; } gz_stream_t; static gz_stream_t gz_stream SECTION_NOZERO_BSS; -static int gz_stream_read_src(TINF_DATA *tinf) { +static int gz_stream_read_src(uzlib_uncomp_t *decomp) { int n = gz_stream.stream_read(gz_stream.stream_data, gz_stream.buf, sizeof(gz_stream.buf)); if (n < 0) { // Stream error @@ -56,17 +56,17 @@ static int gz_stream_read_src(TINF_DATA *tinf) { return -1; } - tinf->source = gz_stream.buf + 1; - tinf->source_limit = gz_stream.buf + n; + decomp->source = gz_stream.buf + 1; + decomp->source_limit = gz_stream.buf + n; return gz_stream.buf[0]; } int gz_stream_init_from_raw_data(const uint8_t *data, size_t len) { - memset(&gz_stream.tinf, 0, sizeof(gz_stream.tinf)); - gz_stream.tinf.source = data; - gz_stream.tinf.source_limit = data + len; + memset(&gz_stream.decomp, 0, sizeof(gz_stream.decomp)); + gz_stream.decomp.source = data; + gz_stream.decomp.source_limit = data + len; - uzlib_uncompress_init(&gz_stream.tinf, gz_stream.dict, DICT_SIZE); + uzlib_uncompress_init(&gz_stream.decomp, gz_stream.dict, DICT_SIZE); return 0; } @@ -75,36 +75,36 @@ int gz_stream_init_from_stream(void *stream_data, stream_read_t stream_read) { gz_stream.stream_data = stream_data; gz_stream.stream_read = stream_read; - memset(&gz_stream.tinf, 0, sizeof(gz_stream.tinf)); - gz_stream.tinf.source_read_cb = gz_stream_read_src; + memset(&gz_stream.decomp, 0, sizeof(gz_stream.decomp)); + gz_stream.decomp.source_read_cb = gz_stream_read_src; - int st = uzlib_gzip_parse_header(&gz_stream.tinf); - if (st != TINF_OK) { + int st = uzlib_gzip_parse_header(&gz_stream.decomp); + if (st != UZLIB_OK) { return -MBOOT_ERRNO_GUNZIP_FAILED; } - uzlib_uncompress_init(&gz_stream.tinf, gz_stream.dict, DICT_SIZE); + uzlib_uncompress_init(&gz_stream.decomp, gz_stream.dict, DICT_SIZE); return 0; } int gz_stream_read(size_t len, uint8_t *buf) { - if (gz_stream.tinf.source == NULL && gz_stream.tinf.source_read_cb == NULL) { + if (gz_stream.decomp.source == NULL && gz_stream.decomp.source_read_cb == NULL) { // End of stream. return 0; } - gz_stream.tinf.dest = buf; - gz_stream.tinf.dest_limit = buf + len; - int st = uzlib_uncompress_chksum(&gz_stream.tinf); + gz_stream.decomp.dest = buf; + gz_stream.decomp.dest_limit = buf + len; + int st = uzlib_uncompress_chksum(&gz_stream.decomp); if (st < 0) { return st; } - if (st == TINF_DONE) { + if (st == UZLIB_DONE) { // Indicate end-of-stream for subsequent calls. - gz_stream.tinf.source = NULL; - gz_stream.tinf.source_read_cb = NULL; + gz_stream.decomp.source = NULL; + gz_stream.decomp.source_read_cb = NULL; } - return gz_stream.tinf.dest - buf; + return gz_stream.decomp.dest - buf; } #endif // MBOOT_FSLOAD || MBOOT_ENABLE_PACKING From d75a3cd8611c421d03784ebc459c3faa878bf77b Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Tue, 27 Jun 2023 01:34:31 +1000 Subject: [PATCH 07/31] lib/uzlib: Combine zlib/gzip header parsing to allow auto-detect. This supports `wbits` values between +40 to +47. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- lib/uzlib/header.c | 133 +++++++++++++++++++++++++++++++++++ lib/uzlib/tinfgzip.c | 110 ----------------------------- lib/uzlib/tinfzlib.c | 66 ----------------- lib/uzlib/uzlib.h | 5 +- ports/stm32/mboot/Makefile | 2 +- ports/stm32/mboot/gzstream.c | 5 +- 6 files changed, 140 insertions(+), 181 deletions(-) create mode 100644 lib/uzlib/header.c delete mode 100644 lib/uzlib/tinfgzip.c delete mode 100644 lib/uzlib/tinfzlib.c diff --git a/lib/uzlib/header.c b/lib/uzlib/header.c new file mode 100644 index 000000000000..edd2b08ab5a4 --- /dev/null +++ b/lib/uzlib/header.c @@ -0,0 +1,133 @@ +/* + * uzlib - tiny deflate/inflate library (deflate, gzip, zlib) + * + * Copyright (c) 2003 by Joergen Ibsen / Jibz + * All Rights Reserved + * + * http://www.ibsensoftware.com/ + * + * Copyright (c) 2014-2018 by Paul Sokolovsky + * + * Optimised for MicroPython: + * Copyright (c) 2023 by Jim Mussared + * + * This software is provided 'as-is', without any express + * or implied warranty. In no event will the authors be + * held liable for any damages arising from the use of + * this software. + * + * Permission is granted to anyone to use this software + * for any purpose, including commercial applications, + * and to alter it and redistribute it freely, subject to + * the following restrictions: + * + * 1. The origin of this software must not be + * misrepresented; you must not claim that you + * wrote the original software. If you use this + * software in a product, an acknowledgment in + * the product documentation would be appreciated + * but is not required. + * + * 2. Altered source versions must be plainly marked + * as such, and must not be misrepresented as + * being the original software. + * + * 3. This notice may not be removed or altered from + * any source distribution. + */ + +#include "uzlib.h" + +#define FTEXT 1 +#define FHCRC 2 +#define FEXTRA 4 +#define FNAME 8 +#define FCOMMENT 16 + +void tinf_skip_bytes(uzlib_uncomp_t *d, int num); +uint16_t tinf_get_uint16(uzlib_uncomp_t *d); + +void tinf_skip_bytes(uzlib_uncomp_t *d, int num) +{ + while (num--) uzlib_get_byte(d); +} + +uint16_t tinf_get_uint16(uzlib_uncomp_t *d) +{ + unsigned int v = uzlib_get_byte(d); + v = (uzlib_get_byte(d) << 8) | v; + return v; +} + +int uzlib_parse_zlib_gzip_header(uzlib_uncomp_t *d, int *wbits) +{ + /* -- check format -- */ + unsigned char cmf = uzlib_get_byte(d); + unsigned char flg = uzlib_get_byte(d); + + /* check for gzip id bytes */ + if (cmf == 0x1f && flg == 0x8b) { + /* check method is deflate */ + if (uzlib_get_byte(d) != 8) return UZLIB_DATA_ERROR; + + /* get flag byte */ + flg = uzlib_get_byte(d); + + /* check that reserved bits are zero */ + if (flg & 0xe0) return UZLIB_DATA_ERROR; + + /* -- find start of compressed data -- */ + + /* skip rest of base header of 10 bytes */ + tinf_skip_bytes(d, 6); + + /* skip extra data if present */ + if (flg & FEXTRA) + { + unsigned int xlen = tinf_get_uint16(d); + tinf_skip_bytes(d, xlen); + } + + /* skip file name if present */ + if (flg & FNAME) { while (uzlib_get_byte(d)); } + + /* skip file comment if present */ + if (flg & FCOMMENT) { while (uzlib_get_byte(d)); } + + /* check header crc if present */ + if (flg & FHCRC) + { + /*unsigned int hcrc =*/ tinf_get_uint16(d); + + // TODO: Check! + // if (hcrc != (tinf_crc32(src, start - src) & 0x0000ffff)) + // return UZLIB_DATA_ERROR; + } + + /* initialize for crc32 checksum */ + d->checksum_type = UZLIB_CHKSUM_CRC; + d->checksum = ~0; + + return UZLIB_HEADER_GZIP; + } else { + /* check checksum */ + if ((256*cmf + flg) % 31) return UZLIB_DATA_ERROR; + + /* check method is deflate */ + if ((cmf & 0x0f) != 8) return UZLIB_DATA_ERROR; + + /* check window size is valid */ + if ((cmf >> 4) > 7) return UZLIB_DATA_ERROR; + + /* check there is no preset dictionary */ + if (flg & 0x20) return UZLIB_DATA_ERROR; + + /* initialize for adler32 checksum */ + d->checksum_type = UZLIB_CHKSUM_ADLER; + d->checksum = 1; + + *wbits = (cmf >> 4) + 8; + + return UZLIB_HEADER_ZLIB; + } +} diff --git a/lib/uzlib/tinfgzip.c b/lib/uzlib/tinfgzip.c deleted file mode 100644 index 3b0feb532122..000000000000 --- a/lib/uzlib/tinfgzip.c +++ /dev/null @@ -1,110 +0,0 @@ -/* - * uzlib - tiny deflate/inflate library (deflate, gzip, zlib) - * - * Copyright (c) 2003 by Joergen Ibsen / Jibz - * All Rights Reserved - * - * http://www.ibsensoftware.com/ - * - * Copyright (c) 2014-2018 by Paul Sokolovsky - * - * This software is provided 'as-is', without any express - * or implied warranty. In no event will the authors be - * held liable for any damages arising from the use of - * this software. - * - * Permission is granted to anyone to use this software - * for any purpose, including commercial applications, - * and to alter it and redistribute it freely, subject to - * the following restrictions: - * - * 1. The origin of this software must not be - * misrepresented; you must not claim that you - * wrote the original software. If you use this - * software in a product, an acknowledgment in - * the product documentation would be appreciated - * but is not required. - * - * 2. Altered source versions must be plainly marked - * as such, and must not be misrepresented as - * being the original software. - * - * 3. This notice may not be removed or altered from - * any source distribution. - */ - -#include "uzlib.h" - -#define FTEXT 1 -#define FHCRC 2 -#define FEXTRA 4 -#define FNAME 8 -#define FCOMMENT 16 - -void tinf_skip_bytes(uzlib_uncomp_t *d, int num); -uint16_t tinf_get_uint16(uzlib_uncomp_t *d); - -void tinf_skip_bytes(uzlib_uncomp_t *d, int num) -{ - while (num--) uzlib_get_byte(d); -} - -uint16_t tinf_get_uint16(uzlib_uncomp_t *d) -{ - unsigned int v = uzlib_get_byte(d); - v = (uzlib_get_byte(d) << 8) | v; - return v; -} - -int uzlib_gzip_parse_header(uzlib_uncomp_t *d) -{ - unsigned char flg; - - /* -- check format -- */ - - /* check id bytes */ - if (uzlib_get_byte(d) != 0x1f || uzlib_get_byte(d) != 0x8b) return UZLIB_DATA_ERROR; - - /* check method is deflate */ - if (uzlib_get_byte(d) != 8) return UZLIB_DATA_ERROR; - - /* get flag byte */ - flg = uzlib_get_byte(d); - - /* check that reserved bits are zero */ - if (flg & 0xe0) return UZLIB_DATA_ERROR; - - /* -- find start of compressed data -- */ - - /* skip rest of base header of 10 bytes */ - tinf_skip_bytes(d, 6); - - /* skip extra data if present */ - if (flg & FEXTRA) - { - unsigned int xlen = tinf_get_uint16(d); - tinf_skip_bytes(d, xlen); - } - - /* skip file name if present */ - if (flg & FNAME) { while (uzlib_get_byte(d)); } - - /* skip file comment if present */ - if (flg & FCOMMENT) { while (uzlib_get_byte(d)); } - - /* check header crc if present */ - if (flg & FHCRC) - { - /*unsigned int hcrc =*/ tinf_get_uint16(d); - - // TODO: Check! -// if (hcrc != (tinf_crc32(src, start - src) & 0x0000ffff)) -// return UZLIB_DATA_ERROR; - } - - /* initialize for crc32 checksum */ - d->checksum_type = UZLIB_CHKSUM_CRC; - d->checksum = ~0; - - return UZLIB_OK; -} diff --git a/lib/uzlib/tinfzlib.c b/lib/uzlib/tinfzlib.c deleted file mode 100644 index 03954cefaeba..000000000000 --- a/lib/uzlib/tinfzlib.c +++ /dev/null @@ -1,66 +0,0 @@ -/* - * uzlib - tiny deflate/inflate library (deflate, gzip, zlib) - * - * Copyright (c) 2003 by Joergen Ibsen / Jibz - * All Rights Reserved - * - * http://www.ibsensoftware.com/ - * - * Copyright (c) 2014-2018 by Paul Sokolovsky - * - * This software is provided 'as-is', without any express - * or implied warranty. In no event will the authors be - * held liable for any damages arising from the use of - * this software. - * - * Permission is granted to anyone to use this software - * for any purpose, including commercial applications, - * and to alter it and redistribute it freely, subject to - * the following restrictions: - * - * 1. The origin of this software must not be - * misrepresented; you must not claim that you - * wrote the original software. If you use this - * software in a product, an acknowledgment in - * the product documentation would be appreciated - * but is not required. - * - * 2. Altered source versions must be plainly marked - * as such, and must not be misrepresented as - * being the original software. - * - * 3. This notice may not be removed or altered from - * any source distribution. - */ - -#include "uzlib.h" - -int uzlib_zlib_parse_header(uzlib_uncomp_t *d) -{ - unsigned char cmf, flg; - - /* -- get header bytes -- */ - - cmf = uzlib_get_byte(d); - flg = uzlib_get_byte(d); - - /* -- check format -- */ - - /* check checksum */ - if ((256*cmf + flg) % 31) return UZLIB_DATA_ERROR; - - /* check method is deflate */ - if ((cmf & 0x0f) != 8) return UZLIB_DATA_ERROR; - - /* check window size is valid */ - if ((cmf >> 4) > 7) return UZLIB_DATA_ERROR; - - /* check there is no preset dictionary */ - if (flg & 0x20) return UZLIB_DATA_ERROR; - - /* initialize for adler32 checksum */ - d->checksum_type = UZLIB_CHKSUM_ADLER; - d->checksum = 1; - - return cmf >> 4; -} diff --git a/lib/uzlib/uzlib.h b/lib/uzlib/uzlib.h index ca58299fc3ec..3756ea75393f 100644 --- a/lib/uzlib/uzlib.h +++ b/lib/uzlib/uzlib.h @@ -130,8 +130,9 @@ void uzlib_uncompress_init(uzlib_uncomp_t *d, void *dict, unsigned int dictLen); int uzlib_uncompress(uzlib_uncomp_t *d); int uzlib_uncompress_chksum(uzlib_uncomp_t *d); -int uzlib_zlib_parse_header(uzlib_uncomp_t *d); -int uzlib_gzip_parse_header(uzlib_uncomp_t *d); +#define UZLIB_HEADER_ZLIB 0 +#define UZLIB_HEADER_GZIP 1 +int uzlib_parse_zlib_gzip_header(uzlib_uncomp_t *d, int *wbits); /* Compression API */ diff --git a/ports/stm32/mboot/Makefile b/ports/stm32/mboot/Makefile index 1f6ba063b22e..630327d7002b 100755 --- a/ports/stm32/mboot/Makefile +++ b/ports/stm32/mboot/Makefile @@ -105,7 +105,7 @@ LIB_SRC_C += \ lib/oofatfs/ffunicode.c \ lib/uzlib/adler32.c \ lib/uzlib/crc32.c \ - lib/uzlib/tinfgzip.c \ + lib/uzlib/header.c \ lib/uzlib/tinflate.c SRC_C += \ diff --git a/ports/stm32/mboot/gzstream.c b/ports/stm32/mboot/gzstream.c index f008eca5cd79..764ef0a4779d 100644 --- a/ports/stm32/mboot/gzstream.c +++ b/ports/stm32/mboot/gzstream.c @@ -78,8 +78,9 @@ int gz_stream_init_from_stream(void *stream_data, stream_read_t stream_read) { memset(&gz_stream.decomp, 0, sizeof(gz_stream.decomp)); gz_stream.decomp.source_read_cb = gz_stream_read_src; - int st = uzlib_gzip_parse_header(&gz_stream.decomp); - if (st != UZLIB_OK) { + int header_wbits; + int st = uzlib_parse_zlib_gzip_header(&gz_stream.decomp, &header_wbits); + if (st != UZLIB_HEADER_GZIP) { return -MBOOT_ERRNO_GUNZIP_FAILED; } From ef5061fefdcfd025eb2fc9645b65515ae3ae6d78 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Tue, 27 Jun 2023 02:47:05 +1000 Subject: [PATCH 08/31] lib/uzlib/tinflate: Implement more compact lookup tables. Saves 68 bytes on PYBV11. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- lib/uzlib/tinflate.c | 116 ++++++++++++++++--------------------------- lib/uzlib/uzlib.h | 1 - 2 files changed, 43 insertions(+), 74 deletions(-) diff --git a/lib/uzlib/tinflate.c b/lib/uzlib/tinflate.c index e844d7faa491..312485fd0436 100644 --- a/lib/uzlib/tinflate.c +++ b/lib/uzlib/tinflate.c @@ -7,6 +7,9 @@ * * Copyright (c) 2014-2018 by Paul Sokolovsky * + * Optimised for MicroPython: + * Copyright (c) 2023 by Jim Mussared + * * This software is provided 'as-is', without any express * or implied warranty. In no event will the authors be * held liable for any damages arising from the use of @@ -51,45 +54,45 @@ uint32_t tinf_get_be_uint32(uzlib_uncomp_t *d); * -- uninitialized global data (static structures) -- * * --------------------------------------------------- */ -#ifdef RUNTIME_BITS_TABLES - -/* extra bits and base tables for length codes */ -unsigned char length_bits[30]; -unsigned short length_base[30]; - -/* extra bits and base tables for distance codes */ -unsigned char dist_bits[30]; -unsigned short dist_base[30]; - -#else - -const unsigned char length_bits[30] = { - 0, 0, 0, 0, 0, 0, 0, 0, - 1, 1, 1, 1, 2, 2, 2, 2, - 3, 3, 3, 3, 4, 4, 4, 4, - 5, 5, 5, 5 +typedef struct { + unsigned char length_bits : 3; + unsigned short length_base : 9; + unsigned char dist_bits : 4; + unsigned short dist_base : 15; +} lookup_table_entry_t; + +const lookup_table_entry_t lookup_table[30] = { + {0, 3, 0, 1}, + {0, 4, 0, 2}, + {0, 5, 0, 3}, + {0, 6, 0, 4}, + {0, 7, 1, 5}, + {0, 8, 1, 7}, + {0, 9, 2, 9}, + {0, 10, 2, 13}, + {1, 11, 3, 17}, + {1, 13, 3, 25}, + {1, 15, 4, 33}, + {1, 17, 4, 49}, + {2, 19, 5, 65}, + {2, 23, 5, 97}, + {2, 27, 6, 129}, + {2, 31, 6, 193}, + {3, 35, 7, 257}, + {3, 43, 7, 385}, + {3, 51, 8, 513}, + {3, 59, 8, 769}, + {4, 67, 9, 1025}, + {4, 83, 9, 1537}, + {4, 99, 10, 2049}, + {4, 115, 10, 3073}, + {5, 131, 11, 4097}, + {5, 163, 11, 6145}, + {5, 195, 12, 8193}, + {5, 227, 12, 12289}, + {0, 258, 13, 16385}, + {0, 0, 13, 24577}, }; -const unsigned short length_base[30] = { - 3, 4, 5, 6, 7, 8, 9, 10, - 11, 13, 15, 17, 19, 23, 27, 31, - 35, 43, 51, 59, 67, 83, 99, 115, - 131, 163, 195, 227, 258 -}; - -const unsigned char dist_bits[30] = { - 0, 0, 0, 0, 1, 1, 2, 2, - 3, 3, 4, 4, 5, 5, 6, 6, - 7, 7, 8, 8, 9, 9, 10, 10, - 11, 11, 12, 12, 13, 13 -}; -const unsigned short dist_base[30] = { - 1, 2, 3, 4, 5, 7, 9, 13, - 17, 25, 33, 49, 65, 97, 129, 193, - 257, 385, 513, 769, 1025, 1537, 2049, 3073, - 4097, 6145, 8193, 12289, 16385, 24577 -}; - -#endif /* special ordering of code length codes */ const unsigned char clcidx[] = { @@ -102,25 +105,6 @@ const unsigned char clcidx[] = { * -- utility functions -- * * ----------------------- */ -#ifdef RUNTIME_BITS_TABLES -/* build extra bits and base tables */ -static void tinf_build_bits_base(unsigned char *bits, unsigned short *base, int delta, int first) -{ - int i, sum; - - /* build bits table */ - for (i = 0; i < delta; ++i) bits[i] = 0; - for (i = 0; i < 30 - delta; ++i) bits[i + delta] = i / delta; - - /* build base table */ - for (sum = first, i = 0; i < 30; ++i) - { - base[i] = sum; - sum += 1 << bits[i]; - } -} -#endif - /* build the fixed huffman trees */ static void tinf_build_fixed_trees(TINF_TREE *lt, TINF_TREE *dt) { @@ -433,7 +417,7 @@ static int tinf_inflate_block_data(uzlib_uncomp_t *d, TINF_TREE *lt, TINF_TREE * } /* possibly get more bits from length code */ - d->curlen = tinf_read_bits(d, length_bits[sym], length_base[sym]); + d->curlen = tinf_read_bits(d, lookup_table[sym].length_bits, lookup_table[sym].length_base); dist = tinf_decode_symbol(d, dt); if (dist >= 30) { @@ -441,7 +425,7 @@ static int tinf_inflate_block_data(uzlib_uncomp_t *d, TINF_TREE *lt, TINF_TREE * } /* possibly get more bits from distance code */ - offs = tinf_read_bits(d, dist_bits[dist], dist_base[dist]); + offs = tinf_read_bits(d, lookup_table[dist].dist_bits, lookup_table[dist].dist_base); /* calculate and validate actual LZ offset to use */ if (d->dict_ring) { @@ -521,20 +505,6 @@ static int tinf_inflate_uncompressed_block(uzlib_uncomp_t *d) * -- public functions -- * * ---------------------- */ -/* initialize global (static) data */ -void uzlib_init(void) -{ -#ifdef RUNTIME_BITS_TABLES - /* build extra bits and base tables */ - tinf_build_bits_base(length_bits, length_base, 4, 3); - tinf_build_bits_base(dist_bits, dist_base, 2, 1); - - /* fix a special case */ - length_bits[28] = 0; - length_base[28] = 258; -#endif -} - /* initialize decompression structure */ void uzlib_uncompress_init(uzlib_uncomp_t *d, void *dict, unsigned int dictLen) { diff --git a/lib/uzlib/uzlib.h b/lib/uzlib/uzlib.h index 3756ea75393f..80f1b2a7238f 100644 --- a/lib/uzlib/uzlib.h +++ b/lib/uzlib/uzlib.h @@ -125,7 +125,6 @@ unsigned char uzlib_get_byte(uzlib_uncomp_t *d); /* Decompression API */ -void uzlib_init(void); void uzlib_uncompress_init(uzlib_uncomp_t *d, void *dict, unsigned int dictLen); int uzlib_uncompress(uzlib_uncomp_t *d); int uzlib_uncompress_chksum(uzlib_uncomp_t *d); From 7f16bfca9f6d5b558b33b6f89566b50918449ed6 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Tue, 27 Jun 2023 02:56:58 +1000 Subject: [PATCH 09/31] lib/uzlib/defl_static: Optimize zlib_start/finish_block. Collapsing the two adjacent calls to outbits saves 32 bytes. Bringing defl_static.c into lz77.c allows better inlining, saves 24 bytes. Merge the Outbuf/uzlib_lz77_state_t structs, a minor simplification that doesn't change code size. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- lib/uzlib/defl_static.c | 49 +++++++++++++++++++++-------------------- lib/uzlib/defl_static.h | 43 ------------------------------------ lib/uzlib/lz77.c | 14 +++++++----- lib/uzlib/uzlib.h | 18 +++++++++------ 4 files changed, 44 insertions(+), 80 deletions(-) delete mode 100644 lib/uzlib/defl_static.h diff --git a/lib/uzlib/defl_static.c b/lib/uzlib/defl_static.c index 529055ae847d..208e9ea5d2a5 100644 --- a/lib/uzlib/defl_static.c +++ b/lib/uzlib/defl_static.c @@ -52,15 +52,15 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * having to transmit the trees. */ -static void outbits(struct Outbuf *out, unsigned long bits, int nbits) +static void outbits(uzlib_lz77_state_t *state, unsigned long bits, int nbits) { - assert(out->noutbits + nbits <= 32); - out->outbits |= bits << out->noutbits; - out->noutbits += nbits; - while (out->noutbits >= 8) { - out->dest_write_cb(out->dest_write_data, out->outbits & 0xFF); - out->outbits >>= 8; - out->noutbits -= 8; + assert(state->noutbits + nbits <= 32); + state->outbits |= bits << state->noutbits; + state->noutbits += nbits; + while (state->noutbits >= 8) { + state->dest_write_cb(state->dest_write_data, state->outbits & 0xFF); + state->outbits >>= 8; + state->noutbits -= 8; } } @@ -81,18 +81,18 @@ static int int_log2(int x) { return r; } -void zlib_literal(struct Outbuf *out, unsigned char c) +static void uzlib_literal(uzlib_lz77_state_t *state, unsigned char c) { if (c <= 143) { /* 0 through 143 are 8 bits long starting at 00110000. */ - outbits(out, mirrorbyte(0x30 + c), 8); + outbits(state, mirrorbyte(0x30 + c), 8); } else { /* 144 through 255 are 9 bits long starting at 110010000. */ - outbits(out, 1 + 2 * mirrorbyte(0x90 - 144 + c), 9); + outbits(state, 1 + 2 * mirrorbyte(0x90 - 144 + c), 9); } } -void zlib_match(struct Outbuf *out, int distance, int len) +static void uzlib_match(uzlib_lz77_state_t *state, int distance, int len) { assert(distance >= 1 && distance <= 32768); distance -= 1; @@ -133,9 +133,9 @@ void zlib_match(struct Outbuf *out, int distance, int len) * 11000000. */ if (lcode <= 22) { - outbits(out, mirrorbyte((lcode + 1) * 2), 7); + outbits(state, mirrorbyte((lcode + 1) * 2), 7); } else { - outbits(out, mirrorbyte(lcode + 169), 8); + outbits(state, mirrorbyte(lcode + 169), 8); } /* @@ -144,7 +144,7 @@ void zlib_match(struct Outbuf *out, int distance, int len) if (thislen < 255 && x > 1) { int extrabits = x - 1; int lmin = (thislen >> extrabits) << extrabits; - outbits(out, thislen - lmin, extrabits); + outbits(state, thislen - lmin, extrabits); } x = int_log2(distance); @@ -153,7 +153,7 @@ void zlib_match(struct Outbuf *out, int distance, int len) /* * Transmit the distance code. Five bits starting at 00000. */ - outbits(out, mirrorbyte((x * 2 + y) * 8), 5); + outbits(state, mirrorbyte((x * 2 + y) * 8), 5); /* * Transmit the extra bits. @@ -161,20 +161,21 @@ void zlib_match(struct Outbuf *out, int distance, int len) if (x > 1) { int dextrabits = x - 1; int dmin = (distance >> dextrabits) << dextrabits; - outbits(out, distance - dmin, dextrabits); + outbits(state, distance - dmin, dextrabits); } } } -void zlib_start_block(struct Outbuf *out) +void uzlib_start_block(uzlib_lz77_state_t *state) { -// outbits(out, 0x9C78, 16); - outbits(out, 1, 1); /* Final block */ - outbits(out, 1, 2); /* Static huffman block */ + // Final block (0b1) + // Static huffman block (0b01) + outbits(state, 3, 3); } -void zlib_finish_block(struct Outbuf *out) +void uzlib_finish_block(uzlib_lz77_state_t *state) { - outbits(out, 0, 7); /* close block */ - outbits(out, 0, 7); /* Make sure all bits are flushed */ + // Close block (0b0000000) + // Make sure all bits are flushed (0b0000000) + outbits(state, 0, 14); } diff --git a/lib/uzlib/defl_static.h b/lib/uzlib/defl_static.h deleted file mode 100644 index 7329de36efa0..000000000000 --- a/lib/uzlib/defl_static.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) uzlib authors - * - * This software is provided 'as-is', without any express - * or implied warranty. In no event will the authors be - * held liable for any damages arising from the use of - * this software. - * - * Permission is granted to anyone to use this software - * for any purpose, including commercial applications, - * and to alter it and redistribute it freely, subject to - * the following restrictions: - * - * 1. The origin of this software must not be - * misrepresented; you must not claim that you - * wrote the original software. If you use this - * software in a product, an acknowledgment in - * the product documentation would be appreciated - * but is not required. - * - * 2. Altered source versions must be plainly marked - * as such, and must not be misrepresented as - * being the original software. - * - * 3. This notice may not be removed or altered from - * any source distribution. - */ - -/* This files contains type declaration and prototypes for defl_static.c. - They may be altered/distinct from the originals used in PuTTY source - code. */ - -struct Outbuf { - void *dest_write_data; - void (*dest_write_cb)(void *data, uint8_t byte); - unsigned long outbits; - int noutbits; -}; - -void zlib_start_block(struct Outbuf *ctx); -void zlib_finish_block(struct Outbuf *ctx); -void zlib_literal(struct Outbuf *ectx, unsigned char c); -void zlib_match(struct Outbuf *ectx, int distance, int len); diff --git a/lib/uzlib/lz77.c b/lib/uzlib/lz77.c index 1c7a8442ca85..117ce50f70f2 100644 --- a/lib/uzlib/lz77.c +++ b/lib/uzlib/lz77.c @@ -11,6 +11,8 @@ #include "uzlib.h" +#include "defl_static.c" + #define MATCH_LEN_MIN (3) #define MATCH_LEN_MAX (258) @@ -18,8 +20,8 @@ // hist_max should be greater than 0 a power of 2 (ie 1, 2, 4, 8, ...). // It's possible to pass in hist=NULL, and then the history window will be taken from the // src passed in to uzlib_lz77_compress (this is useful when not doing streaming compression). -void uzlib_lz77_init(struct uzlib_lz77_state *state, uint8_t *hist, size_t hist_max) { - memset(&state->outbuf, 0, sizeof(state->outbuf)); +void uzlib_lz77_init(uzlib_lz77_state_t *state, uint8_t *hist, size_t hist_max) { + memset(state, 0, sizeof(uzlib_lz77_state_t)); state->hist_buf = hist; state->hist_max = hist_max; state->hist_start = 0; @@ -30,7 +32,7 @@ void uzlib_lz77_init(struct uzlib_lz77_state *state, uint8_t *hist, size_t hist_ // Search back in the history for the maximum match of the given src data, // with support for searching beyond the end of the history and into the src buffer // (effectively the history and src buffer are concatenated). -static size_t uzlib_lz77_search_max_match(struct uzlib_lz77_state *state, const uint8_t *src, size_t len, size_t *longest_offset) { +static size_t uzlib_lz77_search_max_match(uzlib_lz77_state_t *state, const uint8_t *src, size_t len, size_t *longest_offset) { size_t longest_len = 0; for (size_t hist_search = 0; hist_search < state->hist_len; ++hist_search) { size_t match_len; @@ -55,7 +57,7 @@ static size_t uzlib_lz77_search_max_match(struct uzlib_lz77_state *state, const } // Compress the given chunk of data. -void uzlib_lz77_compress(struct uzlib_lz77_state *state, const uint8_t *src, unsigned len) { +void uzlib_lz77_compress(uzlib_lz77_state_t *state, const uint8_t *src, unsigned len) { const uint8_t *top = src + len; while (src < top) { // Look for a match in the history window. @@ -64,10 +66,10 @@ void uzlib_lz77_compress(struct uzlib_lz77_state *state, const uint8_t *src, uns // Encode the literal byte or the match. if (match_len == 0) { - zlib_literal(&state->outbuf, *src); + uzlib_literal(state, *src); match_len = 1; } else { - zlib_match(&state->outbuf, match_offset, match_len); + uzlib_match(state, match_offset, match_len); } // Push the bytes into the history buffer. diff --git a/lib/uzlib/uzlib.h b/lib/uzlib/uzlib.h index 80f1b2a7238f..030f7698c176 100644 --- a/lib/uzlib/uzlib.h +++ b/lib/uzlib/uzlib.h @@ -42,8 +42,6 @@ #include #include -#include "defl_static.h" - #include "uzlib_conf.h" #if UZLIB_CONF_DEBUG_LOG #include @@ -135,16 +133,22 @@ int uzlib_parse_zlib_gzip_header(uzlib_uncomp_t *d, int *wbits); /* Compression API */ -struct uzlib_lz77_state { - struct Outbuf outbuf; +typedef struct { + void *dest_write_data; + void (*dest_write_cb)(void *data, uint8_t byte); + unsigned long outbits; + int noutbits; uint8_t *hist_buf; size_t hist_max; size_t hist_start; size_t hist_len; -}; +} uzlib_lz77_state_t; + +void uzlib_lz77_init(uzlib_lz77_state_t *state, uint8_t *hist, size_t hist_max); +void uzlib_lz77_compress(uzlib_lz77_state_t *state, const uint8_t *src, unsigned len); -void uzlib_lz77_init(struct uzlib_lz77_state *state, uint8_t *hist, size_t hist_max); -void uzlib_lz77_compress(struct uzlib_lz77_state *state, const uint8_t *src, unsigned len); +void uzlib_start_block(uzlib_lz77_state_t *state); +void uzlib_finish_block(uzlib_lz77_state_t *state); /* Checksum API */ From e6c290c3d145de81e23f406c84359860c4d6e632 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Thu, 29 Jun 2023 11:56:08 +1000 Subject: [PATCH 10/31] lib/uzlib: Add a source_read_data var to pass to source_read_cb. For better abstraction for users of this API. Signed-off-by: Jim Mussared --- lib/uzlib/tinflate.c | 2 +- lib/uzlib/uzlib.h | 3 ++- ports/stm32/mboot/gzstream.c | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/uzlib/tinflate.c b/lib/uzlib/tinflate.c index 312485fd0436..53536272f329 100644 --- a/lib/uzlib/tinflate.c +++ b/lib/uzlib/tinflate.c @@ -185,7 +185,7 @@ unsigned char uzlib_get_byte(uzlib_uncomp_t *d) read next byte using it. (Note: the callback can also update ->source and ->source_limit). */ if (d->source_read_cb && !d->eof) { - int val = d->source_read_cb(d); + int val = d->source_read_cb(d->source_read_data); if (val >= 0) { return (unsigned char)val; } diff --git a/lib/uzlib/uzlib.h b/lib/uzlib/uzlib.h index 030f7698c176..16984a77d3eb 100644 --- a/lib/uzlib/uzlib.h +++ b/lib/uzlib/uzlib.h @@ -84,7 +84,8 @@ typedef struct _uzlib_uncomp_t { also return -1 in case of EOF (or irrecoverable error). Note that besides returning the next byte, it may also update source and source_limit fields, thus allowing for buffered operation. */ - int (*source_read_cb)(struct _uzlib_uncomp_t *uncomp); + void *source_read_data; + int (*source_read_cb)(void *); unsigned int tag; unsigned int bitcount; diff --git a/ports/stm32/mboot/gzstream.c b/ports/stm32/mboot/gzstream.c index 764ef0a4779d..f8b0987edf71 100644 --- a/ports/stm32/mboot/gzstream.c +++ b/ports/stm32/mboot/gzstream.c @@ -45,7 +45,8 @@ typedef struct _gz_stream_t { static gz_stream_t gz_stream SECTION_NOZERO_BSS; -static int gz_stream_read_src(uzlib_uncomp_t *decomp) { +static int gz_stream_read_src(void *data) { + uzlib_uncomp_t *decomp = data; int n = gz_stream.stream_read(gz_stream.stream_data, gz_stream.buf, sizeof(gz_stream.buf)); if (n < 0) { // Stream error @@ -76,6 +77,7 @@ int gz_stream_init_from_stream(void *stream_data, stream_read_t stream_read) { gz_stream.stream_read = stream_read; memset(&gz_stream.decomp, 0, sizeof(gz_stream.decomp)); + gz_stream.decomp.source_read_data = &gz_stream.decomp; gz_stream.decomp.source_read_cb = gz_stream_read_src; int header_wbits; From 3533924c36ae85ce6e8bf8598dd71cf16bbdb10b Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Mon, 26 Jun 2023 13:52:10 +1000 Subject: [PATCH 11/31] extmod/moddeflate: Add deflate module providing the DeflateIO class. This provides similar functionality to the former zlib.DecompIO and especially CPython's gzip.GzipFile for both compression and decompression. This class can be used directly, and also can be used from Python to implement (via io.BytesIO) zlib.decompress and zlib.compress, as well as gzip.GzipFile. Enable/disable this on all ports/boards that zlib was previously configured for. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- extmod/extmod.cmake | 1 + extmod/extmod.mk | 1 + extmod/modbinascii.c | 6 +- extmod/moddeflate.c | 404 ++++++++++++++++++ ports/cc3200/mpconfigport.h | 1 + ports/qemu-arm/mpconfigport.h | 1 + ports/samd/mpconfigport.h | 1 + .../boards/NUCLEO_G474RE/mpconfigboard.h | 1 + ports/windows/mpconfigport.h | 2 + ports/windows/msvc/sources.props | 1 + py/mpconfig.h | 12 +- tests/unix/extra_coverage.py.exp | 13 +- 12 files changed, 434 insertions(+), 10 deletions(-) create mode 100644 extmod/moddeflate.c diff --git a/extmod/extmod.cmake b/extmod/extmod.cmake index 29b4627614b4..305cdee94276 100644 --- a/extmod/extmod.cmake +++ b/extmod/extmod.cmake @@ -23,6 +23,7 @@ set(MICROPY_SOURCE_EXTMOD ${MICROPY_EXTMOD_DIR}/modbinascii.c ${MICROPY_EXTMOD_DIR}/modcryptolib.c ${MICROPY_EXTMOD_DIR}/moductypes.c + ${MICROPY_EXTMOD_DIR}/moddeflate.c ${MICROPY_EXTMOD_DIR}/modhashlib.c ${MICROPY_EXTMOD_DIR}/modheapq.c ${MICROPY_EXTMOD_DIR}/modjson.c diff --git a/extmod/extmod.mk b/extmod/extmod.mk index 0745fb49ebc0..a15b7e4a5fcd 100644 --- a/extmod/extmod.mk +++ b/extmod/extmod.mk @@ -16,6 +16,7 @@ SRC_EXTMOD_C += \ extmod/modbluetooth.c \ extmod/modbtree.c \ extmod/modcryptolib.c \ + extmod/moddeflate.c \ extmod/modframebuf.c \ extmod/modhashlib.c \ extmod/modheapq.c \ diff --git a/extmod/modbinascii.c b/extmod/modbinascii.c index 0c562de7c0f0..ed39960180ae 100644 --- a/extmod/modbinascii.c +++ b/extmod/modbinascii.c @@ -170,8 +170,8 @@ STATIC mp_obj_t mod_binascii_b2a_base64(size_t n_args, const mp_obj_t *pos_args, } STATIC MP_DEFINE_CONST_FUN_OBJ_KW(mod_binascii_b2a_base64_obj, 1, mod_binascii_b2a_base64); -#if 0 // MICROPY_PY_BINASCII_CRC32 -#include "lib/uzlib/crc32.c" +#if MICROPY_PY_BINASCII_CRC32 && MICROPY_PY_DEFLATE +#include "lib/uzlib/uzlib.h" STATIC mp_obj_t mod_binascii_crc32(size_t n_args, const mp_obj_t *args) { mp_buffer_info_t bufinfo; @@ -191,7 +191,7 @@ STATIC const mp_rom_map_elem_t mp_module_binascii_globals_table[] = { #endif { MP_ROM_QSTR(MP_QSTR_a2b_base64), MP_ROM_PTR(&mod_binascii_a2b_base64_obj) }, { MP_ROM_QSTR(MP_QSTR_b2a_base64), MP_ROM_PTR(&mod_binascii_b2a_base64_obj) }, - #if 0 // MICROPY_PY_BINASCII_CRC32 + #if MICROPY_PY_BINASCII_CRC32 && MICROPY_PY_DEFLATE { MP_ROM_QSTR(MP_QSTR_crc32), MP_ROM_PTR(&mod_binascii_crc32_obj) }, #endif }; diff --git a/extmod/moddeflate.c b/extmod/moddeflate.c new file mode 100644 index 000000000000..1d8a8acf73b0 --- /dev/null +++ b/extmod/moddeflate.c @@ -0,0 +1,404 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2023 Jim Mussared + * + * Based on extmod/modzlib.c + * Copyright (c) 2014-2016 Paul Sokolovsky + * Copyright (c) 2021-2023 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include + +#include "py/runtime.h" +#include "py/stream.h" +#include "py/mperrno.h" + +#if MICROPY_PY_DEFLATE + +#include "lib/uzlib/uzlib.h" + +#if 0 // print debugging info +#define DEBUG_printf DEBUG_printf +#else // don't print debugging info +#define DEBUG_printf(...) (void)0 +#endif + +typedef enum { + DEFLATEIO_FORMAT_MIN = 0, + DEFLATEIO_FORMAT_AUTO = DEFLATEIO_FORMAT_MIN, // Read mode this means auto-detect zlib/gzip, write mode this means RAW. + DEFLATEIO_FORMAT_RAW = 1, + DEFLATEIO_FORMAT_ZLIB = 2, + DEFLATEIO_FORMAT_GZIP = 3, + DEFLATEIO_FORMAT_MAX = DEFLATEIO_FORMAT_GZIP, +} deflateio_format_t; + +const int DEFLATEIO_DEFAULT_WBITS = 8; + +typedef struct { + void *window; + uzlib_uncomp_t decomp; + bool eof; +} mp_obj_deflateio_read_t; + +#if MICROPY_PY_DEFLATE_COMPRESS +typedef struct { + void *window; + size_t input_len; + uint32_t input_checksum; + uzlib_lz77_state_t lz77; +} mp_obj_deflateio_write_t; +#endif + +typedef struct { + mp_obj_base_t base; + mp_obj_t stream; + uint8_t format : 2; + uint8_t window_bits : 4; + bool close : 1; + mp_obj_deflateio_read_t *read; + #if MICROPY_PY_DEFLATE_COMPRESS + mp_obj_deflateio_write_t *write; + #endif +} mp_obj_deflateio_t; + +STATIC int deflateio_read_stream(void *data) { + mp_obj_deflateio_t *self = data; + const mp_stream_p_t *stream = mp_get_stream(self->stream); + int err; + byte c; + mp_uint_t out_sz = stream->read(self->stream, &c, 1, &err); + if (out_sz == MP_STREAM_ERROR) { + mp_raise_OSError(err); + } + if (out_sz == 0) { + mp_raise_type(&mp_type_EOFError); + } + return c; +} + +STATIC bool deflateio_init_read(mp_obj_deflateio_t *self) { + if (self->read) { + return true; + } + + mp_get_stream_raise(self->stream, MP_STREAM_OP_READ); + + self->read = m_new_obj(mp_obj_deflateio_read_t); + memset(&self->read->decomp, 0, sizeof(self->read->decomp)); + self->read->decomp.source_read_data = self; + self->read->decomp.source_read_cb = deflateio_read_stream; + self->read->eof = false; + + // Don't modify self->window_bits as it may also be used for write. + int wbits = self->window_bits; + + // Parse the header if we're in NONE/ZLIB/GZIP modes. + if (self->format != DEFLATEIO_FORMAT_RAW) { + int header_wbits = wbits; + int header_type = uzlib_parse_zlib_gzip_header(&self->read->decomp, &header_wbits); + if ((self->format == DEFLATEIO_FORMAT_ZLIB && header_type != UZLIB_HEADER_ZLIB) || (self->format == DEFLATEIO_FORMAT_GZIP && header_type != UZLIB_HEADER_GZIP)) { + return false; + } + if (wbits == 0 && header_wbits < 15) { + // If the header specified something lower than the default, then + // use that instead. + wbits = header_wbits; + } + } + + if (wbits == 0) { + wbits = DEFLATEIO_DEFAULT_WBITS; + } + + size_t window_len = 1 << wbits; + self->read->window = m_new(uint8_t, window_len); + + uzlib_uncompress_init(&self->read->decomp, self->read->window, window_len); + + return true; +} + +#if MICROPY_PY_DEFLATE_COMPRESS +STATIC void deflateio_out_byte(void *data, uint8_t b) { + mp_obj_deflateio_t *self = data; + const mp_stream_p_t *stream = mp_get_stream(self->stream); + int err; + mp_uint_t ret = stream->write(self->stream, &b, 1, &err); + if (ret == MP_STREAM_ERROR) { + mp_raise_OSError(err); + } +} + +STATIC bool deflateio_init_write(mp_obj_deflateio_t *self) { + if (self->write) { + return true; + } + + const mp_stream_p_t *stream = mp_get_stream_raise(self->stream, MP_STREAM_OP_WRITE); + + self->write = m_new_obj(mp_obj_deflateio_write_t); + self->write->input_len = 0; + + int wbits = self->window_bits; + if (wbits == 0) { + wbits = DEFLATEIO_DEFAULT_WBITS; + } + size_t window_len = 1 << wbits; + self->write->window = m_new(uint8_t, window_len); + + uzlib_lz77_init(&self->write->lz77, self->write->window, window_len); + self->write->lz77.dest_write_data = self; + self->write->lz77.dest_write_cb = deflateio_out_byte; + + // Write header if needed. + mp_uint_t ret = 0; + int err; + if (self->format == DEFLATEIO_FORMAT_ZLIB) { + // -----CMF------ ----------FLG--------------- + // CINFO(5) CM(3) FLEVEL(2) FDICT(1) FCHECK(5) + uint8_t buf[] = { 0x08, 0x80 }; // CM=2 (deflate), FLEVEL=2 (default), FDICT=0 (no dictionary) + buf[0] |= MAX(wbits - 8, 1) << 4; // base-2 logarithm of the LZ77 window size, minus eight. + buf[1] |= 31 - ((buf[0] * 256 + buf[1]) % 31); // (CMF*256 + FLG) % 31 == 0. + ret = stream->write(self->stream, buf, sizeof(buf), &err); + + self->write->input_checksum = 1; // ADLER32 + } else if (self->format == DEFLATEIO_FORMAT_GZIP) { + // ID1(8) ID2(8) CM(8) ---FLG--- MTIME(32) XFL(8) OS(8) + // FLG: x x x FCOMMENT FNAME FEXTRA FHCRC FTEXT + uint8_t buf[] = { 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03 }; // MTIME=0, XFL=4 (fastest), OS=3 (unix) + ret = stream->write(self->stream, buf, sizeof(buf), &err); + + self->write->input_checksum = ~0; // CRC32 + } + if (ret == MP_STREAM_ERROR) { + return false; + } + + // Write starting block. + uzlib_start_block(&self->write->lz77); + + return true; +} +#endif + +STATIC mp_obj_t deflateio_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args_in) { + // args: stream, format=NONE, wbits=0, close=False + mp_arg_check_num(n_args, n_kw, 1, 4, false); + + mp_int_t format = n_args > 1 ? mp_obj_get_int(args_in[1]) : DEFLATEIO_FORMAT_AUTO; + mp_int_t wbits = n_args > 2 ? mp_obj_get_int(args_in[2]) : 0; + + if (format < DEFLATEIO_FORMAT_MIN || format > DEFLATEIO_FORMAT_MAX) { + mp_raise_ValueError(MP_ERROR_TEXT("format")); + } + if (wbits != 0 && (wbits < 5 || wbits > 15)) { + mp_raise_ValueError(MP_ERROR_TEXT("wbits")); + } + + mp_obj_deflateio_t *self = mp_obj_malloc(mp_obj_deflateio_t, type); + self->stream = args_in[0]; + self->format = format; + self->window_bits = wbits; + self->read = NULL; + #if MICROPY_PY_DEFLATE_COMPRESS + self->write = NULL; + #endif + self->close = n_args > 3 ? mp_obj_is_true(args_in[3]) : false; + + return MP_OBJ_FROM_PTR(self); +} + +STATIC mp_uint_t deflateio_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) { + mp_obj_deflateio_t *self = MP_OBJ_TO_PTR(o_in); + + if (self->stream == MP_OBJ_NULL || !deflateio_init_read(self)) { + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; + } + + if (self->read->eof) { + return 0; + } + + self->read->decomp.dest = buf; + self->read->decomp.dest_limit = (uint8_t *)buf + size; + int st = uzlib_uncompress_chksum(&self->read->decomp); + if (st == UZLIB_DONE) { + self->read->eof = true; + } + if (st < 0) { + DEBUG_printf("uncompress error=" INT_FMT "\n", st); + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; + } + return self->read->decomp.dest - (uint8_t *)buf; +} + +#if MICROPY_PY_DEFLATE_COMPRESS +STATIC mp_uint_t deflateio_write(mp_obj_t self_in, const void *buf, mp_uint_t size, int *errcode) { + mp_obj_deflateio_t *self = MP_OBJ_TO_PTR(self_in); + + if (self->stream == MP_OBJ_NULL || !deflateio_init_write(self)) { + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; + } + + self->write->input_len += size; + if (self->format == DEFLATEIO_FORMAT_ZLIB) { + self->write->input_checksum = uzlib_adler32(buf, size, self->write->input_checksum); + } else if (self->format == DEFLATEIO_FORMAT_GZIP) { + self->write->input_checksum = uzlib_crc32(buf, size, self->write->input_checksum); + } + + uzlib_lz77_compress(&self->write->lz77, buf, size); + return size; +} + +static inline void put_le32(char *buf, uint32_t value) { + buf[0] = value & 0xff; + buf[1] = value >> 8 & 0xff; + buf[2] = value >> 16 & 0xff; + buf[3] = value >> 24 & 0xff; +} + +static inline void put_be32(char *buf, uint32_t value) { + buf[3] = value & 0xff; + buf[2] = value >> 8 & 0xff; + buf[1] = value >> 16 & 0xff; + buf[0] = value >> 24 & 0xff; +} +#endif + +STATIC mp_uint_t deflateio_ioctl(mp_obj_t self_in, mp_uint_t request, uintptr_t arg, int *errcode) { + if (request == MP_STREAM_CLOSE) { + mp_obj_deflateio_t *self = MP_OBJ_TO_PTR(self_in); + + mp_uint_t ret = 0; + + if (self->stream != MP_OBJ_NULL) { + #if MICROPY_PY_DEFLATE_COMPRESS + if (self->write) { + uzlib_finish_block(&self->write->lz77); + + const mp_stream_p_t *stream = mp_get_stream(self->stream); + + // Write footer if needed. + if (self->format == DEFLATEIO_FORMAT_ZLIB || self->format == DEFLATEIO_FORMAT_GZIP) { + char footer[8]; + size_t footer_len; + if (self->format == DEFLATEIO_FORMAT_ZLIB) { + put_be32(&footer[0], self->write->input_checksum); + footer_len = 4; + } else { // DEFLATEIO_FORMAT_GZIP + put_le32(&footer[0], ~self->write->input_checksum); + put_le32(&footer[4], self->write->input_len); + footer_len = 8; + } + if (stream->write(self->stream, footer, footer_len, errcode) == MP_STREAM_ERROR) { + ret = MP_STREAM_ERROR; + } + } + } + #endif + + // Only close the stream if required. e.g. when using io.BytesIO + // it needs to stay open so that getvalue() can be called. + if (self->close) { + mp_stream_close(self->stream); + } + + // Either way, free the reference to the stream. + self->stream = MP_OBJ_NULL; + } + + return ret; + } else { + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; + } +} + +STATIC const mp_stream_p_t deflateio_stream_p = { + .read = deflateio_read, + #if MICROPY_PY_DEFLATE_COMPRESS + .write = deflateio_write, + #endif + .ioctl = deflateio_ioctl, +}; + +#if !MICROPY_ENABLE_DYNRUNTIME +STATIC const mp_rom_map_elem_t deflateio_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) }, + { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) }, + { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) }, + #if MICROPY_PY_DEFLATE_COMPRESS + { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) }, + #endif + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, + { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&mp_stream___exit___obj) }, +}; +STATIC MP_DEFINE_CONST_DICT(deflateio_locals_dict, deflateio_locals_dict_table); + +STATIC MP_DEFINE_CONST_OBJ_TYPE( + deflateio_type, + MP_QSTR_DeflateIO, + MP_TYPE_FLAG_NONE, + make_new, deflateio_make_new, + protocol, &deflateio_stream_p, + locals_dict, &deflateio_locals_dict + ); + +STATIC const mp_rom_map_elem_t mp_module_deflate_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_deflate) }, + { MP_ROM_QSTR(MP_QSTR_DeflateIO), MP_ROM_PTR(&deflateio_type) }, + { MP_ROM_QSTR(MP_QSTR_AUTO), MP_ROM_INT(DEFLATEIO_FORMAT_AUTO) }, + { MP_ROM_QSTR(MP_QSTR_RAW), MP_ROM_INT(DEFLATEIO_FORMAT_RAW) }, + { MP_ROM_QSTR(MP_QSTR_ZLIB), MP_ROM_INT(DEFLATEIO_FORMAT_ZLIB) }, + { MP_ROM_QSTR(MP_QSTR_GZIP), MP_ROM_INT(DEFLATEIO_FORMAT_GZIP) }, +}; +STATIC MP_DEFINE_CONST_DICT(mp_module_deflate_globals, mp_module_deflate_globals_table); + +const mp_obj_module_t mp_module_deflate = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t *)&mp_module_deflate_globals, +}; + +MP_REGISTER_MODULE(MP_QSTR_deflate, mp_module_deflate); +#endif // !MICROPY_ENABLE_DYNRUNTIME + +// Source files #include'd here to make sure they're compiled in +// only if the module is enabled. + +#include "lib/uzlib/tinflate.c" +#include "lib/uzlib/header.c" +#include "lib/uzlib/adler32.c" +#include "lib/uzlib/crc32.c" + +#if MICROPY_PY_DEFLATE_COMPRESS +#include "lib/uzlib/lz77.c" +#endif + +#endif // MICROPY_PY_DEFLATE diff --git a/ports/cc3200/mpconfigport.h b/ports/cc3200/mpconfigport.h index dbceabb9d86a..b6b412f178f3 100644 --- a/ports/cc3200/mpconfigport.h +++ b/ports/cc3200/mpconfigport.h @@ -110,6 +110,7 @@ #define MICROPY_PY_THREAD_GIL (1) #define MICROPY_PY_BINASCII (1) #define MICROPY_PY_UCTYPES (0) +#define MICROPY_PY_DEFLATE (0) #define MICROPY_PY_JSON (1) #define MICROPY_PY_RE (1) #define MICROPY_PY_HEAPQ (0) diff --git a/ports/qemu-arm/mpconfigport.h b/ports/qemu-arm/mpconfigport.h index d186f70b7eb3..4341fba58f36 100644 --- a/ports/qemu-arm/mpconfigport.h +++ b/ports/qemu-arm/mpconfigport.h @@ -40,6 +40,7 @@ #define MICROPY_PY_BINASCII (1) #define MICROPY_PY_RANDOM (1) #define MICROPY_PY_UCTYPES (1) +#define MICROPY_PY_DEFLATE (1) #define MICROPY_PY_JSON (1) #define MICROPY_PY_OS (1) #define MICROPY_PY_RE (1) diff --git a/ports/samd/mpconfigport.h b/ports/samd/mpconfigport.h index 827e952b00de..c983d7e6d669 100644 --- a/ports/samd/mpconfigport.h +++ b/ports/samd/mpconfigport.h @@ -91,6 +91,7 @@ #define MICROPY_PY_UCTYPES (1) #define MICROPY_PY_HEAPQ (1) #define MICROPY_PY_RANDOM (1) +#define MICROPY_PY_DEFLATE (1) #define MICROPY_PY_ASYNCIO (1) #define MICROPY_PY_MACHINE_RTC (1) #ifndef MICROPY_PY_MACHINE_ADC diff --git a/ports/stm32/boards/NUCLEO_G474RE/mpconfigboard.h b/ports/stm32/boards/NUCLEO_G474RE/mpconfigboard.h index 0093b2df39bb..62d104a5462b 100644 --- a/ports/stm32/boards/NUCLEO_G474RE/mpconfigboard.h +++ b/ports/stm32/boards/NUCLEO_G474RE/mpconfigboard.h @@ -10,6 +10,7 @@ #define MICROPY_HW_HAS_FLASH (0) // QSPI extflash not mounted #define MICROPY_PY_ASYNCIO (0) +#define MICROPY_PY_DEFLATE (0) #define MICROPY_PY_BINASCII (0) #define MICROPY_PY_HASHLIB (0) #define MICROPY_PY_JSON (0) diff --git a/ports/windows/mpconfigport.h b/ports/windows/mpconfigport.h index 0658838075f6..de209423dd09 100644 --- a/ports/windows/mpconfigport.h +++ b/ports/windows/mpconfigport.h @@ -144,6 +144,8 @@ #define MICROPY_PY_TIME_INCLUDEFILE "ports/unix/modtime.c" #define MICROPY_PY_ERRNO (1) #define MICROPY_PY_UCTYPES (1) +#define MICROPY_PY_DEFLATE (1) +#define MICROPY_PY_DEFLATE_COMPRESS (1) #define MICROPY_PY_JSON (1) #define MICROPY_PY_RE (1) #define MICROPY_PY_HEAPQ (1) diff --git a/ports/windows/msvc/sources.props b/ports/windows/msvc/sources.props index 6d438f3f0db0..53c4fdddfd65 100644 --- a/ports/windows/msvc/sources.props +++ b/ports/windows/msvc/sources.props @@ -10,6 +10,7 @@ + diff --git a/py/mpconfig.h b/py/mpconfig.h index 122d0a70f2a7..e6f1531ce1bf 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -1555,6 +1555,16 @@ typedef double mp_float_t; #define MICROPY_PY_UCTYPES_NATIVE_C_TYPES (1) #endif +// Whether to provide "deflate" module (decompression-only by default) +#ifndef MICROPY_PY_DEFLATE +#define MICROPY_PY_DEFLATE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#endif + +// Whether to provide compression support in "deflate" module +#ifndef MICROPY_PY_DEFLATE_COMPRESS +#define MICROPY_PY_DEFLATE_COMPRESS (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_FULL_FEATURES) +#endif + #ifndef MICROPY_PY_JSON #define MICROPY_PY_JSON (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif @@ -1629,7 +1639,7 @@ typedef double mp_float_t; #define MICROPY_PY_BINASCII (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif -// Depends on MICROPY_PY_ZLIB +// Depends on MICROPY_PY_DEFLATE #ifndef MICROPY_PY_BINASCII_CRC32 #define MICROPY_PY_BINASCII_CRC32 (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif diff --git a/tests/unix/extra_coverage.py.exp b/tests/unix/extra_coverage.py.exp index 9c7660de6dc6..352bf39630ac 100644 --- a/tests/unix/extra_coverage.py.exp +++ b/tests/unix/extra_coverage.py.exp @@ -53,12 +53,13 @@ port builtins micropython _asyncio _thread array binascii btree cexample cmath collections cppexample cryptolib -errno example_package ffi -framebuf gc hashlib heapq -io json machine math -os random re select -socket ssl struct sys -termios time uctypes websocket +deflate errno example_package +ffi framebuf gc hashlib +heapq io json machine +math os random re +select socket ssl struct +sys termios time uctypes +websocket me micropython machine math From 8b315ef0d82a31991def597906f7f24bd39c6e23 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Thu, 29 Jun 2023 15:52:03 +1000 Subject: [PATCH 12/31] tests/extmod: Add deflate.DeflateIO tests. Signed-off-by: Jim Mussared --- tests/extmod/deflate_compress.py | 148 +++++++++++++++++++ tests/extmod/deflate_compress.py.exp | 41 ++++++ tests/extmod/deflate_decompress.py | 174 +++++++++++++++++++++++ tests/extmod/deflate_decompress.py.exp | 80 +++++++++++ tests/extmod/deflate_stream_error.py | 89 ++++++++++++ tests/extmod/deflate_stream_error.py.exp | 25 ++++ 6 files changed, 557 insertions(+) create mode 100644 tests/extmod/deflate_compress.py create mode 100644 tests/extmod/deflate_compress.py.exp create mode 100644 tests/extmod/deflate_decompress.py create mode 100644 tests/extmod/deflate_decompress.py.exp create mode 100644 tests/extmod/deflate_stream_error.py create mode 100644 tests/extmod/deflate_stream_error.py.exp diff --git a/tests/extmod/deflate_compress.py b/tests/extmod/deflate_compress.py new file mode 100644 index 000000000000..612af663e4fd --- /dev/null +++ b/tests/extmod/deflate_compress.py @@ -0,0 +1,148 @@ +try: + # Check if deflate is available. + import deflate + import io +except ImportError: + print("SKIP") + raise SystemExit + +# Check if compression is enabled. +if not hasattr(deflate.DeflateIO, "write"): + print("SKIP") + raise SystemExit + +# Simple compression & decompression. +b = io.BytesIO() +g = deflate.DeflateIO(b, deflate.RAW) +data = b"micropython" +N = 10 +for i in range(N): + g.write(data) +g.close() +result_raw = b.getvalue() +print(len(result_raw) < len(data) * N) +b = io.BytesIO(result_raw) +g = deflate.DeflateIO(b, deflate.RAW) +print(g.read()) + +# Same, but using a context manager. +b = io.BytesIO() +with deflate.DeflateIO(b, deflate.RAW) as g: + for i in range(N): + g.write(data) +result_raw = b.getvalue() +print(len(result_raw) < len(data) * N) +b = io.BytesIO(result_raw) +with deflate.DeflateIO(b, deflate.RAW) as g: + print(g.read()) + +# Writing to a closed underlying stream. +b = io.BytesIO() +g = deflate.DeflateIO(b, deflate.RAW) +g.write(b"micropython") +b.close() +try: + g.write(b"micropython") +except ValueError: + print("ValueError") + +# Writing to a closed DeflateIO. +b = io.BytesIO() +g = deflate.DeflateIO(b, deflate.RAW) +g.write(b"micropython") +g.close() +try: + g.write(b"micropython") +except OSError: + print("OSError") + + +def decompress(data, *args): + buf = io.BytesIO(data) + with deflate.DeflateIO(buf, *args) as g: + return g.read() + + +def compress(data, *args): + b = io.BytesIO() + with deflate.DeflateIO(b, *args) as g: + g.write(data) + return b.getvalue() + + +def compress_error(data, *args): + try: + compress(data, *args) + except OSError: + print("OSError") + except ValueError: + print("ValueError") + + +# More test patterns. +PATTERNS_RAW = ( + (b"0", b"3\x00\x00"), + (b"a", b"K\x04\x00"), + (b"0" * 100, b"3\xa0\x03\x00\x00"), + ( + bytes(range(64)), + b"c`dbfaec\xe7\xe0\xe4\xe2\xe6\xe1\xe5\xe3\x17\x10\x14\x12\x16\x11\x15\x13\x97\x90\x94\x92\x96\x91\x95\x93WPTRVQUS\xd7\xd0\xd4\xd2\xd6\xd1\xd5\xd370426153\xb7\xb0\xb4\xb2\xb6\xb1\xb5\xb3\x07\x00", + ), +) +for unpacked, packed in PATTERNS_RAW: + print(compress(unpacked) == packed) + print(compress(unpacked, deflate.RAW) == packed) + +# Verify header and checksum format. +unpacked = b"hello" +packed = b"\xcbH\xcd\xc9\xc9\x07\x00" + + +def check_header(n, a, b): + if a == b: + print(n) + else: + print(n, a, b) + + +check_header("RAW", compress(unpacked, deflate.RAW), packed) +check_header( + "ZLIB(9)", compress(unpacked, deflate.ZLIB, 9), b"\x18\x95" + packed + b"\x06,\x02\x15" +) +check_header( + "ZLIB(15)", compress(unpacked, deflate.ZLIB, 15), b"\x78\x9c" + packed + b"\x06,\x02\x15" +) +check_header( + "GZIP", + compress(unpacked, deflate.GZIP, 9), + b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x04\x03" + packed + b"\x86\xa6\x106\x05\x00\x00\x00", +) + +# Valid wbits values. +compress_error(unpacked, deflate.RAW, -1) +print(len(compress(unpacked, deflate.RAW, 0))) +compress_error(unpacked, deflate.RAW, 1) +compress_error(unpacked, deflate.RAW, 4) +for i in range(5, 16): + print(len(compress(unpacked, deflate.RAW, i))) +compress_error(unpacked, deflate.RAW, 16) + +# Invalid values for format. +compress_error(unpacked, -1) +compress_error(unpacked, 5) + +# Fill buf with a predictable pseudorandom sequence. +buf = bytearray(1024) +lfsr = 1 << 15 | 1 +for i in range(len(buf)): + bit = (lfsr ^ (lfsr >> 1) ^ (lfsr >> 3) ^ (lfsr >> 12)) & 1 + lfsr = (lfsr >> 1) | (bit << 15) + buf[i] = lfsr & 0xFF + +# Verify that compression improves as the window size increases. +prev_len = len(buf) +for wbits in range(5, 10): + result = compress(buf, deflate.RAW, wbits) + next_len = len(result) + print(next_len < prev_len and decompress(result, deflate.RAW, wbits) == buf) + prev_len = next_len diff --git a/tests/extmod/deflate_compress.py.exp b/tests/extmod/deflate_compress.py.exp new file mode 100644 index 000000000000..5da70f491bde --- /dev/null +++ b/tests/extmod/deflate_compress.py.exp @@ -0,0 +1,41 @@ +True +b'micropythonmicropythonmicropythonmicropythonmicropythonmicropythonmicropythonmicropythonmicropythonmicropython' +True +b'micropythonmicropythonmicropythonmicropythonmicropythonmicropythonmicropythonmicropythonmicropythonmicropython' +ValueError +OSError +True +True +True +True +True +True +True +True +RAW +ZLIB(9) +ZLIB(15) +GZIP +ValueError +7 +ValueError +ValueError +7 +7 +7 +7 +7 +7 +7 +7 +7 +7 +7 +ValueError +ValueError +ValueError +False +True +True +True +True diff --git a/tests/extmod/deflate_decompress.py b/tests/extmod/deflate_decompress.py new file mode 100644 index 000000000000..29d3ec2d7110 --- /dev/null +++ b/tests/extmod/deflate_decompress.py @@ -0,0 +1,174 @@ +try: + # Check if deflate is available. + import deflate + import io +except ImportError: + print("SKIP") + raise SystemExit + +# zlib.compress(b'micropython hello world hello world micropython', wbits=-9) +data_raw = b'\xcb\xcdL.\xca/\xa8,\xc9\xc8\xcfS\xc8H\xcd\xc9\xc9W(\xcf/\xcaIAa\xe7"\xd4\x00\x00' +# zlib.compress(b'micropython hello world hello world micropython', wbits=9) +data_zlib = b'\x18\x95\xcb\xcdL.\xca/\xa8,\xc9\xc8\xcfS\xc8H\xcd\xc9\xc9W(\xcf/\xcaIAa\xe7"\xd4\x00\x00\xbc\xfa\x12\x91' +# zlib.compress(b'micropython hello world hello world micropython', wbits=25) +data_gzip = b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\xcb\xcdL.\xca/\xa8,\xc9\xc8\xcfS\xc8H\xcd\xc9\xc9W(\xcf/\xcaIAa\xe7"\xd4\x00\x00"\xeb\xc4\x98/\x00\x00\x00' + +# compress(b'hello' + bytearray(300) + b'hello', format=deflate.RAW, 5) +data_wbits_5 = b"\xcbH\xcd\xc9\xc9g\x18\xe9\x00\x08\x88\x95\xcfH\xcd\xc9\xc9\x07\x00" +# compress(b'hello' + bytearray(300) + b'hello', format=deflate.RAW, 6) +data_wbits_6 = b"\xcbH\xcd\xc9\xc9g\x18\xe9\x00\x08\x88\xd5\x9f\x91\x9a\x93\x93\x0f\x00" +# compress(b'hello' + bytearray(300) + b'hello', format=deflate.RAW, 8) +data_wbits_8 = b"\xcbH\xcd\xc9\xc9g\x18\xe9\x00\x08\x88\xf5\x7fFjNN>\x00" +# compress(b'hello' + bytearray(2000) + b'hello', format=deflate.RAW, 10) +data_wbits_10 = b"\xcbH\xcd\xc9\xc9g\x18\xe9\x00\x08Fz\x18\x00\xc3`\xa4'\x03`2\x18\xe99\x01\x98\x13Fz\xfe\x07\xe6\xff\x91\x9e\xff\x81\xf9\x7f\xa4\xe7\x7f`\xfe\x1f\xba\xf9?#5''\x1f\x00" + + +def decompress(data, *args): + buf = io.BytesIO(data) + with deflate.DeflateIO(buf, *args) as g: + return g.read() + + +def decompress_error(data, *args): + try: + decompress(data, *args) + except OSError: + print("OSError") + except EOFError: + print("EOFError") + except ValueError: + print("ValueError") + + +# Basic handling of format and detection. +print(decompress(data_raw, deflate.RAW)) +print(decompress(data_zlib, deflate.ZLIB)) +print(decompress(data_gzip, deflate.GZIP)) +print(decompress(data_zlib)) # detect zlib/gzip. +print(decompress(data_gzip)) # detect zlib/gzip. + +decompress_error(data_raw) # cannot detect zlib/gzip from raw stream +decompress_error(data_raw, deflate.ZLIB) +decompress_error(data_raw, deflate.GZIP) +decompress_error(data_zlib, deflate.RAW) +decompress_error(data_zlib, deflate.GZIP) +decompress_error(data_gzip, deflate.RAW) +decompress_error(data_gzip, deflate.ZLIB) + +# Invalid data stream. +decompress_error(b"abcef", deflate.RAW) + +# Invalid block type. final-block, block-type=3. +decompress_error(b"\x07", deflate.RAW) + +# Truncated stream. +decompress_error(data_raw[:10], deflate.RAW) + +# Partial reads. +buf = io.BytesIO(data_zlib) +with deflate.DeflateIO(buf) as g: + print(buf.seek(0, 1)) # verify stream is not read until first read of the DeflateIO stream. + print(g.read(1)) + print(buf.seek(0, 1)) # verify that only the minimal amount is read from the source + print(g.read(1)) + print(buf.seek(0, 1)) + print(g.read(2)) + print(buf.seek(0, 1)) + print(g.read()) + print(buf.seek(0, 1)) + print(g.read(1)) + print(buf.seek(0, 1)) + print(g.read()) + +# Invalid zlib checksum (+ length for gzip). Note: only checksum errors are +# currently detected, see the end of uzlib_uncompress_chksum(). +decompress_error(data_zlib[:-4] + b"\x00\x00\x00\x00") +decompress_error(data_gzip[:-8] + b"\x00\x00\x00\x00\x00\x00\x00\x00") +decompress_error(data_zlib[:-4] + b"\x00\x00\x00\x00", deflate.ZLIB) +decompress_error(data_gzip[:-8] + b"\x00\x00\x00\x00\x00\x00\x00\x00", deflate.GZIP) + +# Reading from a closed underlying stream. +b = io.BytesIO(data_raw) +g = deflate.DeflateIO(b, deflate.RAW) +g.read(4) +b.close() +try: + g.read(4) +except ValueError: + print("ValueError") + +# Reading from a closed DeflateIO. +b = io.BytesIO(data_raw) +g = deflate.DeflateIO(b, deflate.RAW) +g.read(4) +g.close() +try: + g.read(4) +except OSError: + print("OSError") + +# Gzip header with extra flags (FCOMMENT FNAME FEXTRA FHCRC) enabled. +data_gzip_header_extra = b"\x1f\x8b\x08\x1e}\x9a\x9bd\x02\x00\x00\x00\x00\x00\x00\xff\xcb\xcdL.\xca/\xa8,\xc9\xc8\xcf\x03\x00\xf2KF>\x0b\x00\x00\x00" +print(decompress(data_gzip_header_extra)) + +# Test patterns. +PATTERNS_ZLIB = [ + # Packed results produced by CPy's zlib.compress() + (b"0", b"x\x9c3\x00\x00\x001\x001"), + (b"a", b"x\x9cK\x04\x00\x00b\x00b"), + (b"0" * 100, b"x\x9c30\xa0=\x00\x00\xb3q\x12\xc1"), + ( + bytes(range(64)), + b"x\x9cc`dbfaec\xe7\xe0\xe4\xe2\xe6\xe1\xe5\xe3\x17\x10\x14\x12\x16\x11\x15\x13\x97\x90\x94\x92\x96\x91\x95\x93WPTRVQUS\xd7\xd0\xd4\xd2\xd6\xd1\xd5\xd370426153\xb7\xb0\xb4\xb2\xb6\xb1\xb5\xb3\x07\x00\xaa\xe0\x07\xe1", + ), + (b"hello", b"x\x01\x01\x05\x00\xfa\xffhello\x06,\x02\x15"), # compression level 0 + # adaptive/dynamic huffman tree + ( + b"13371813150|13764518736|12345678901", + b"x\x9c\x05\xc1\x81\x01\x000\x04\x04\xb1\x95\\\x1f\xcfn\x86o\x82d\x06Qq\xc8\x9d\xc5X}I}\x00\x951D>I}\x00\x951D>I}\x00\x951D>I}\x00\x951D", + b"x\x9c\x05\xc11\x01\x00\x00\x00\x010\x95\x14py\x84\x12C_\x9bR\x8cV\x8a\xd1J1Z)F\x1fw`\x089", + ), +] +for unpacked, packed in PATTERNS_ZLIB: + print(decompress(packed) == unpacked) + print(decompress(packed, deflate.ZLIB) == unpacked) + +# Older version's of CPython's zlib module still included the checksum and length (as if it were a zlib/gzip stream). +# Make sure there're no problem decompressing this. +data_raw_with_footer = data_raw + b"\x00\x00\x00\x00\x00\x00\x00\x00" +print(decompress(data_raw_with_footer, deflate.RAW)) + +# Valid wbits values. +decompress_error(data_wbits_5, deflate.RAW, -1) +print(len(decompress(data_wbits_5, deflate.RAW, 0))) +decompress_error(data_wbits_5, deflate.RAW, 1) +decompress_error(data_wbits_5, deflate.RAW, 4) +for i in range(5, 16): + print(len(decompress(data_wbits_5, deflate.RAW, i))) +decompress_error(data_wbits_5, deflate.RAW, 16) + +# Invalid values for format. +decompress_error(data_raw, -1) +decompress_error(data_raw, 5) + +# Data that requires a higher wbits value. +decompress_error(data_wbits_6, deflate.RAW, 5) +print(len(decompress(data_wbits_6, deflate.RAW, 6))) +print(len(decompress(data_wbits_6, deflate.RAW, 7))) +decompress_error(data_wbits_8, deflate.RAW, 7) +print(len(decompress(data_wbits_8, deflate.RAW, 8))) +print(len(decompress(data_wbits_8, deflate.RAW, 9))) +decompress_error(data_wbits_10, deflate.RAW) +decompress_error(data_wbits_10, deflate.RAW, 9) +print(len(decompress(data_wbits_10, deflate.RAW, 10))) + +# zlib header sets the size, so works with wbits unset or wbits >= 10. +data_wbits_10_zlib = b"(\x91\xcbH\xcd\xc9\xc9g\x18\xe9\x00\x08Fz\x18\x00\xc3`\xa4'\x03`2\x18\xe99\x01\x98\x13Fz\xfe\x07\xe6\xff\x91\x9e\xff\x81\xf9\x7f\xa4\xe7\x7f`\xfe\x1f\xba\xf9?#5''\x1f\x00[\xbc\x04)" +print(len(decompress(data_wbits_10_zlib, deflate.ZLIB))) +decompress_error(data_wbits_10_zlib, deflate.ZLIB, 9) +print(len(decompress(data_wbits_10_zlib, deflate.ZLIB, 10))) +print(len(decompress(data_wbits_10_zlib))) diff --git a/tests/extmod/deflate_decompress.py.exp b/tests/extmod/deflate_decompress.py.exp new file mode 100644 index 000000000000..381f2b068590 --- /dev/null +++ b/tests/extmod/deflate_decompress.py.exp @@ -0,0 +1,80 @@ +b'micropython hello world hello world micropython' +b'micropython hello world hello world micropython' +b'micropython hello world hello world micropython' +b'micropython hello world hello world micropython' +b'micropython hello world hello world micropython' +OSError +OSError +OSError +OSError +OSError +OSError +OSError +OSError +OSError +EOFError +0 +b'm' +4 +b'i' +5 +b'cr' +7 +b'opython hello world hello world micropython' +36 +b'' +36 +b'' +OSError +OSError +OSError +OSError +ValueError +OSError +b'micropython' +True +True +True +True +True +True +True +True +True +True +True +True +True +True +b'micropython hello world hello world micropython' +ValueError +310 +ValueError +ValueError +310 +310 +310 +310 +310 +310 +310 +310 +310 +310 +310 +ValueError +ValueError +ValueError +OSError +310 +310 +OSError +310 +310 +OSError +OSError +2010 +2010 +OSError +2010 +2010 diff --git a/tests/extmod/deflate_stream_error.py b/tests/extmod/deflate_stream_error.py new file mode 100644 index 000000000000..aee6b2803337 --- /dev/null +++ b/tests/extmod/deflate_stream_error.py @@ -0,0 +1,89 @@ +# Test deflate module with stream errors. + +try: + # Check if deflate & IOBase are available. + import deflate, io + + io.IOBase +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit + +# Check if compression is enabled. +if not hasattr(deflate.DeflateIO, "write"): + print("SKIP") + raise SystemExit + +formats = (deflate.RAW, deflate.ZLIB, deflate.GZIP) + +# Test error on read when decompressing. + + +class Stream(io.IOBase): + def readinto(self, buf): + print("Stream.readinto", len(buf)) + return -1 + + +try: + deflate.DeflateIO(Stream()).read() +except OSError as er: + print(repr(er)) + +# Test error on write when compressing. + + +class Stream(io.IOBase): + def write(self, buf): + print("Stream.write", buf) + return -1 + + +for format in formats: + try: + deflate.DeflateIO(Stream(), format).write("a") + except OSError as er: + print(repr(er)) + +# Test write after close. + + +class Stream(io.IOBase): + def write(self, buf): + print("Stream.write", buf) + return -1 + + def ioctl(self, cmd, arg): + print("Stream.ioctl", cmd, arg) + return 0 + + +try: + d = deflate.DeflateIO(Stream(), deflate.RAW, 0, True) + d.close() + d.write("a") +except OSError as er: + print(repr(er)) + +# Test error on write when closing. + + +class Stream(io.IOBase): + def __init__(self): + self.num_writes = 0 + + def write(self, buf): + print("Stream.write", buf) + if self.num_writes >= 4: + return -1 + self.num_writes += 1 + return len(buf) + + +for format in formats: + d = deflate.DeflateIO(Stream(), format) + d.write("a") + try: + d.close() + except OSError as er: + print(repr(er)) diff --git a/tests/extmod/deflate_stream_error.py.exp b/tests/extmod/deflate_stream_error.py.exp new file mode 100644 index 000000000000..4ec90d79977e --- /dev/null +++ b/tests/extmod/deflate_stream_error.py.exp @@ -0,0 +1,25 @@ +Stream.readinto 1 +OSError(1,) +Stream.write bytearray(b'K') +OSError(1,) +Stream.write bytearray(b'\x18\x95') +OSError(22,) +Stream.write bytearray(b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x04\x03') +OSError(22,) +Stream.ioctl 4 0 +OSError(22,) +Stream.write bytearray(b'K') +Stream.write bytearray(b'\x04') +Stream.write bytearray(b'\x00') +Stream.write bytearray(b'\x18\x95') +Stream.write bytearray(b'K') +Stream.write bytearray(b'\x04') +Stream.write bytearray(b'\x00') +Stream.write bytearray(b'\x00b\x00b') +OSError(1,) +Stream.write bytearray(b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x04\x03') +Stream.write bytearray(b'K') +Stream.write bytearray(b'\x04') +Stream.write bytearray(b'\x00') +Stream.write bytearray(b'C\xbe\xb7\xe8\x01\x00\x00\x00') +OSError(1,) From b804443cb3df2ca4b71070739aa1c8f3c86464fc Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Tue, 27 Jun 2023 02:17:41 +1000 Subject: [PATCH 13/31] docs/library/deflate: Add docs for deflate.DeflateIO. Also update zlib & gzip docs to describe the micropython-lib modules. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- docs/library/deflate.rst | 177 +++++++++++++++++++++++++++++++++++++++ docs/library/gzip.rst | 106 +++++++++++++++++++++++ docs/library/index.rst | 12 +-- docs/library/zlib.rst | 90 +++++++++++++++----- 4 files changed, 357 insertions(+), 28 deletions(-) create mode 100644 docs/library/deflate.rst create mode 100644 docs/library/gzip.rst diff --git a/docs/library/deflate.rst b/docs/library/deflate.rst new file mode 100644 index 000000000000..9752af5925fc --- /dev/null +++ b/docs/library/deflate.rst @@ -0,0 +1,177 @@ +:mod:`deflate` -- deflate compression & decompression +===================================================== + +.. module:: deflate + :synopsis: deflate compression & decompression + +This module allows compression and decompression of binary data with the +`DEFLATE algorithm `_ +(commonly used in the zlib library and gzip archiver). + +**Availability:** + +* Added in MicroPython v1.21. + +* Decompression: Enabled via the ``MICROPY_PY_DEFLATE`` build option, on by default + on ports with the "extra features" level or higher (which is most boards). + +* Compression: Enabled via the ``MICROPY_PY_DEFLATE_COMPRESS`` build option, on + by default on ports with the "full features" level or higher (generally this means + you need to build your own firmware to enable this). + +Classes +------- + +.. class:: DeflateIO(stream, format=AUTO, wbits=0, close=False, /) + + This class can be used to wrap a *stream* which is any + :term:`stream-like ` object such as a file, socket, or stream + (including :class:`io.BytesIO`). It is itself a stream and implements the + standard read/readinto/write/close methods. + + The *stream* must be a blocking stream. Non-blocking streams are currently + not supported. + + The *format* can be set to any of the constants defined below, and defaults + to ``AUTO`` which for decompressing will auto-detect gzip or zlib streams, + and for compressing it will generate a raw stream. + + The *wbits* parameter sets the base-2 logarithm of the DEFLATE dictionary + window size. So for example, setting *wbits* to ``10`` sets the window size + to 1024 bytes. Valid values are ``5`` to ``15`` inclusive (corresponding to + window sizes of 32 to 32k bytes). + + If *wbits* is set to ``0`` (the default), then a window size of 256 bytes + will be used (corresponding to *wbits* set to ``8``), except when + :ref:`decompressing a zlib stream `. + + See the :ref:`window size ` notes below for more information + about the window size, zlib, and gzip streams. + + If *close* is set to ``True`` then the underlying stream will be closed + automatically when the :class:`deflate.DeflateIO` stream is closed. This is + useful if you want to return a :class:`deflate.DeflateIO` stream that wraps + another stream and not have the caller need to know about managing the + underlying stream. + + If compression is enabled, a given :class:`deflate.DeflateIO` instance + supports both reading and writing. For example, a bidirectional stream like + a socket can be wrapped, which allows for compression/decompression in both + directions. + +Constants +--------- + +.. data:: deflate.AUTO + deflate.RAW + deflate.ZLIB + deflate.GZIP + + Supported values for the *format* parameter. + +Examples +-------- + +A typical use case for :class:`deflate.DeflateIO` is to read or write a compressed +file from storage: + +.. code:: python + + import deflate + + # Writing a zlib-compressed stream (uses the default window size of 256 bytes). + with open("data.gz", "wb") as f: + with deflate.DeflateIO(f, deflate.ZLIB) as d: + # Use d.write(...) etc + + # Reading a zlib-compressed stream (auto-detect window size). + with open("data.z", "rb") as f: + with deflate.DeflateIO(f, deflate.ZLIB) as d: + # Use d.read(), d.readinto(), etc. + +Because :class:`deflate.DeflateIO` is a stream, it can be used for example +with :meth:`json.dump` and :meth:`json.load` (and any other places streams can +be used): + +.. code:: python + + import deflate, json + + # Write a dictionary as JSON in gzip format, with a + # small (64 byte) window size. + config = { ... } + with open("config.gz", "wb") as f: + with deflate.DeflateIO(f, deflate.GZIP, 6) as f: + json.dump(config, f) + + # Read back that dictionary. + with open("config.gz", "rb") as f: + with deflate.DeflateIO(f, deflate.GZIP, 6) as f: + config = json.load(f) + +If your source data is not in a stream format, you can use :class:`io.BytesIO` +to turn it into a stream suitable for use with :class:`deflate.DeflateIO`: + +.. code:: python + + import deflate, io + + # Decompress a bytes/bytearray value. + compressed_data = get_data_z() + with deflate.DeflateIO(io.BytesIO(compressed_data), deflate.ZLIB) as d: + decompressed_data = d.read() + + # Compress a bytes/bytearray value. + uncompressed_data = get_data() + stream = io.BytesIO() + with deflate.DeflateIO(stream, deflate.ZLIB) as d: + d.write(uncompressed_data) + compressed_data = stream.getvalue() + +.. _deflate_wbits: + +Deflate window size +------------------- + +The window size limits how far back in the stream the (de)compressor can +reference. Increasing the window size will improve compression, but will +require more memory. + +However, just because a given window size is used for compression, this does not +mean that the stream will require the same size window for decompression, as +the stream may not reference data as far back as the window allows (for example, +if the length of the input is smaller than the window size). + +If the decompressor uses a smaller window size than necessary for the input data +stream, it will fail mid-way through decompression with :exc:`OSError`. + +.. _deflate_wbits_zlib: + +The zlib format includes a header which specifies the window size used to +compress the data (which due to the above, may be larger than the size required +for the decompressor). + +If this header value is lower than the specified *wbits* value, then the header +value will be used instead in order to reduce the memory allocation size. If +the *wbits* parameter is zero (the default), then the header value will only be +used if it is less than the maximum value of ``15`` (which is default value +used by most compressors [#f1]_). + +In other words, if the source zlib stream has been compressed with a custom window +size (i.e. less than ``15``), then using the default *wbits* parameter of zero +will decompress any such stream. + +The gzip file format does not include the window size in the header. +Additionally, most compressor libraries (including CPython's implementation +of :class:`gzip.GzipFile`) will default to the maximum possible window size. +This makes it difficult to decompress most gzip streams on MicroPython unless +your board has a lot of free RAM. + +If you control the source of the compressed data, then prefer to use the zlib +format, with a window size that is suitable for your target device. + +.. rubric:: Footnotes + +.. [#f1] The assumption here is that if the header value is the default used by + most compressors, then nothing is known about the likely required window + size and we should ignore it. diff --git a/docs/library/gzip.rst b/docs/library/gzip.rst new file mode 100644 index 000000000000..f36f896db3a1 --- /dev/null +++ b/docs/library/gzip.rst @@ -0,0 +1,106 @@ +:mod:`gzip` -- gzip compression & decompression +=============================================== + +.. module:: gzip + :synopsis: gzip compression & decompression + +|see_cpython_module| :mod:`python:gzip`. + +This module allows compression and decompression of binary data with the +`DEFLATE algorithm `_ used by the gzip +file format. + +.. note:: Prefer to use :class:`deflate.DeflateIO` instead of the functions in this + module as it provides a streaming interface to compression and decompression + which is convenient and more memory efficient when working with reading or + writing compressed data to a file, socket, or stream. + +**Availability:** + +* This module is **not present by default** in official MicroPython firmware + releases as it duplicates functionality available in the :mod:`deflate + ` module. + +* A copy of this module can be installed (or frozen) + from :term:`micropython-lib` (`source `_). + See :ref:`packages` for more information. This documentation describes that module. + +* Compression support will only be available if compression support is enabled + in the built-in :mod:`deflate ` module. + +Functions +--------- + +.. function:: open(filename, mode, /) + + Wrapper around built-in :func:`open` returning a GzipFile instance. + +.. function:: decompress(data, /) + + Decompresses *data* into a bytes object. + +.. function:: compress(data, /) + + Compresses *data* into a bytes object. + +Classes +------- + +.. class:: GzipFile(*, fileobj, mode) + + This class can be used to wrap a *fileobj* which is any + :term:`stream-like ` object such as a file, socket, or stream + (including :class:`io.BytesIO`). It is itself a stream and implements the + standard read/readinto/write/close methods. + + When the *mode* argument is ``"rb"``, reads from the GzipFile instance will + decompress the data in the underlying stream and return decompressed data. + + If compression support is enabled then the *mode* argument can be set to + ``"wb"``, and writes to the GzipFile instance will be compressed and written + to the underlying stream. + + By default the GzipFile class will read and write data using the gzip file + format, including a header and footer with checksum and a window size of 512 + bytes. + + The **file**, **compresslevel**, and **mtime** arguments are not + supported. **fileobj** and **mode** must always be specified as keyword + arguments. + +Examples +-------- + +A typical use case for :class:`gzip.GzipFile` is to read or write a compressed +file from storage: + +.. code:: python + + import gzip + + # Reading: + with open("data.gz", "rb") as f: + with gzip.GzipFile(fileobj=f, mode="rb") as g: + # Use g.read(), g.readinto(), etc. + + # Same, but using gzip.open: + with gzip.open("data.gz", "rb") as f: + # Use f.read(), f.readinto(), etc. + + # Writing: + with open("data.gz", "wb") as f: + with gzip.GzipFile(fileobj=f, mode="wb") as g: + # Use g.write(...) etc + + # Same, but using gzip.open: + with gzip.open("data.gz", "wb") as f: + # Use f.write(...) etc + + # Write a dictionary as JSON in gzip format, with a + # small (64 byte) window size. + config = { ... } + with gzip.open("config.gz", "wb") as f: + json.dump(config, f) + +For guidance on working with gzip sources and choosing the window size see the +note at the :ref:`end of the deflate documentation `. diff --git a/docs/library/index.rst b/docs/library/index.rst index 69bc81ade5e1..ae5d3e7d7117 100644 --- a/docs/library/index.rst +++ b/docs/library/index.rst @@ -64,6 +64,7 @@ library. collections.rst errno.rst gc.rst + gzip.rst hashlib.rst heapq.rst io.rst @@ -95,6 +96,7 @@ the following libraries. bluetooth.rst btree.rst cryptolib.rst + deflate.rst framebuf.rst machine.rst micropython.rst @@ -194,11 +196,11 @@ Extending built-in libraries from Python A subset of the built-in modules are able to be extended by Python code by providing a module of the same name in the filesystem. This extensibility applies to the following Python standard library modules which are built-in to -the firmware: ``array``, ``binascii``, ``collections``, ``errno``, ``hashlib``, -``heapq``, ``io``, ``json``, ``os``, ``platform``, ``random``, ``re``, -``select``, ``socket``, ``ssl``, ``struct``, ``time`` ``zlib``, as well as the -MicroPython-specific ``machine`` module. All other built-in modules cannot be -extended from the filesystem. +the firmware: ``array``, ``binascii``, ``collections``, ``errno``, ``gzip``, +``hashlib``, ``heapq``, ``io``, ``json``, ``os``, ``platform``, ``random``, +``re``, ``select``, ``socket``, ``ssl``, ``struct``, ``time`` ``zlib``, as well +as the MicroPython-specific ``machine`` module. All other built-in modules +cannot be extended from the filesystem. This allows the user to provide an extended implementation of a built-in library (perhaps to provide additional CPython compatibility or missing functionality). diff --git a/docs/library/zlib.rst b/docs/library/zlib.rst index 96d6c245232b..54310b72f2a3 100644 --- a/docs/library/zlib.rst +++ b/docs/library/zlib.rst @@ -1,38 +1,82 @@ -:mod:`zlib` -- zlib decompression -================================= +:mod:`zlib` -- zlib compression & decompression +=============================================== .. module:: zlib - :synopsis: zlib decompression + :synopsis: zlib compression & decompression |see_cpython_module| :mod:`python:zlib`. -This module allows to decompress binary data compressed with +This module allows compression and decompression of binary data with the `DEFLATE algorithm `_ -(commonly used in zlib library and gzip archiver). Compression -is not yet implemented. +(commonly used in the zlib library and gzip archiver). + +.. note:: Prefer to use :class:`deflate.DeflateIO` instead of the functions in this + module as it provides a streaming interface to compression and decompression + which is convenient and more memory efficient when working with reading or + writing compressed data to a file, socket, or stream. + +**Availability:** + +* From MicroPython v1.21 onwards, this module may not be present by default on + all MicroPython firmware as it duplicates functionality available in + the :mod:`deflate ` module. + +* A copy of this module can be installed (or frozen) + from :term:`micropython-lib` (`source `_). + See :ref:`packages` for more information. This documentation describes that module. + +* Requires the built-in :mod:`deflate ` module (available since MicroPython v1.21) + +* Compression support will only be available if compression support is enabled + in the built-in :mod:`deflate ` module. Functions --------- -.. function:: decompress(data, wbits=0, bufsize=0, /) +.. function:: decompress(data, wbits=15, /) + + Decompresses *data* into a bytes object. + + The *wbits* parameter works the same way as for :meth:`zlib.compress` + with the following additional valid values: + + * ``0``: Automatically determine the window size from the zlib header + (*data* must be in zlib format). + * ``35`` to ``47``: Auto-detect either the zlib or gzip format. + + As for :meth:`zlib.compress`, see the :mod:`CPython documentation for zlib ` + for more information about the *wbits* parameter. As for :meth:`zlib.compress`, + MicroPython also supports smaller window sizes than CPython. See more + :ref:`MicroPython-specific details ` in the + :mod:`deflate ` module documentation. + + If the data to be decompressed requires a larger window size, it will + fail during decompression. + +.. function:: compress(data, wbits=15, /) - Return decompressed *data* as bytes. *wbits* is DEFLATE dictionary window - size used during compression (8-15, the dictionary size is power of 2 of - that value). Additionally, if value is positive, *data* is assumed to be - zlib stream (with zlib header). Otherwise, if it's negative, it's assumed - to be raw DEFLATE stream. *bufsize* parameter is for compatibility with - CPython and is ignored. + Compresses *data* into a bytes object. -.. class:: DecompIO(stream, wbits=0, /) + *wbits* allows you to configure the DEFLATE dictionary window size and the + output format. The window size allows you to trade-off memory usage for + compression level. A larger window size will allow the compressor to + reference fragments further back in the input. The output formats are "raw" + DEFLATE (no header/footer), zlib, and gzip, where the latter two + include a header and checksum. - Create a `stream` wrapper which allows transparent decompression of - compressed data in another *stream*. This allows to process compressed - streams with data larger than available heap size. In addition to - values described in :func:`decompress`, *wbits* may take values - 24..31 (16 + 8..15), meaning that input stream has gzip header. + The low four bits of the absolute value of *wbits* set the base-2 logarithm of + the DEFLATE dictionary window size. So for example, ``wbits=10``, + ``wbits=-10``, and ``wbits=26`` all set the window size to 1024 bytes. Valid + window sizes are ``5`` to ``15`` inclusive (corresponding to 32 to 32k bytes). - .. admonition:: Difference to CPython - :class: attention + Negative values of *wbits* between ``-5`` and ``-15`` correspond to "raw" + output mode, positive values between ``5`` and ``15`` correspond to zlib + output mode, and positive values between ``21`` and ``31`` correspond to + gzip output mode. - This class is MicroPython extension. It's included on provisional - basis and may be changed considerably or removed in later versions. + See the :mod:`CPython documentation for zlib ` for more + information about the *wbits* parameter. Note that MicroPython allows + for smaller window sizes, which is useful when memory is constrained while + still achieving a reasonable level of compression. It also speeds up + the compressor. See more :ref:`MicroPython-specific details ` + in the :mod:`deflate ` module documentation. From ea1a5e43d08429bed96e251f68f77aa0334bb371 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Fri, 14 Jul 2023 17:24:20 +1000 Subject: [PATCH 14/31] examples/natmod/deflate: Add deflate as a dynamic native module. This replaces the previous zlib version. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- examples/natmod/deflate/Makefile | 13 ++++++ examples/natmod/deflate/deflate.c | 70 +++++++++++++++++++++++++++++++ tests/run-natmodtests.py | 1 + tools/ci.sh | 3 +- 4 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 examples/natmod/deflate/Makefile create mode 100644 examples/natmod/deflate/deflate.c diff --git a/examples/natmod/deflate/Makefile b/examples/natmod/deflate/Makefile new file mode 100644 index 000000000000..86ef29b6324b --- /dev/null +++ b/examples/natmod/deflate/Makefile @@ -0,0 +1,13 @@ +# Location of top-level MicroPython directory +MPY_DIR = ../../.. + +# Name of module (different to built-in uzlib so it can coexist) +MOD = deflate_$(ARCH) + +# Source files (.c or .py) +SRC = deflate.c + +# Architecture to build for (x86, x64, armv7m, xtensa, xtensawin) +ARCH = x64 + +include $(MPY_DIR)/py/dynruntime.mk diff --git a/examples/natmod/deflate/deflate.c b/examples/natmod/deflate/deflate.c new file mode 100644 index 000000000000..fa475bd06724 --- /dev/null +++ b/examples/natmod/deflate/deflate.c @@ -0,0 +1,70 @@ +#define MICROPY_PY_DEFLATE (1) +#define MICROPY_PY_DEFLATE_COMPRESS (1) + +#include "py/dynruntime.h" + +#if !defined(__linux__) +void *memcpy(void *dst, const void *src, size_t n) { + return mp_fun_table.memmove_(dst, src, n); +} +void *memset(void *s, int c, size_t n) { + return mp_fun_table.memset_(s, c, n); +} +#endif + +mp_obj_full_type_t deflateio_type; + +#include "extmod/moddeflate.c" + +// Re-implemented from py/stream.c, not yet available in dynruntime.h. +mp_obj_t mp_stream_close(mp_obj_t stream) { + const mp_stream_p_t *stream_p = mp_get_stream(stream); + int error; + mp_uint_t res = stream_p->ioctl(stream, MP_STREAM_CLOSE, 0, &error); + if (res == MP_STREAM_ERROR) { + mp_raise_OSError(error); + } + return mp_const_none; +} +MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_close_obj, mp_stream_close); + +// Re-implemented from py/stream.c, not yet available in dynruntime.h. +STATIC mp_obj_t mp_stream___exit__(size_t n_args, const mp_obj_t *args) { + (void)n_args; + return mp_stream_close(args[0]); +} +MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream___exit___obj, 4, 4, mp_stream___exit__); + +// Re-implemented from obj.c, not yet available in dynruntime.h. +mp_obj_t mp_identity(mp_obj_t self) { + return self; +} +MP_DEFINE_CONST_FUN_OBJ_1(mp_identity_obj, mp_identity); + +mp_map_elem_t deflateio_locals_dict_table[7]; +STATIC MP_DEFINE_CONST_DICT(deflateio_locals_dict, deflateio_locals_dict_table); + +mp_obj_t mpy_init(mp_obj_fun_bc_t *self, size_t n_args, size_t n_kw, mp_obj_t *args) { + MP_DYNRUNTIME_INIT_ENTRY + + deflateio_type.base.type = mp_fun_table.type_type; + deflateio_type.name = MP_QSTR_DeflateIO; + MP_OBJ_TYPE_SET_SLOT(&deflateio_type, make_new, &deflateio_make_new, 0); + MP_OBJ_TYPE_SET_SLOT(&deflateio_type, protocol, &deflateio_stream_p, 1); + deflateio_locals_dict_table[0] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_read), MP_OBJ_FROM_PTR(&mp_stream_read_obj) }; + deflateio_locals_dict_table[1] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readinto), MP_OBJ_FROM_PTR(&mp_stream_readinto_obj) }; + deflateio_locals_dict_table[2] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_readline), MP_OBJ_FROM_PTR(&mp_stream_unbuffered_readline_obj) }; + deflateio_locals_dict_table[3] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_write), MP_OBJ_FROM_PTR(&mp_stream_write_obj) }; + deflateio_locals_dict_table[4] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR_close), MP_OBJ_FROM_PTR(&mp_stream_close_obj) }; + deflateio_locals_dict_table[5] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR___enter__), MP_OBJ_FROM_PTR(&mp_identity_obj) }; + deflateio_locals_dict_table[6] = (mp_map_elem_t){ MP_OBJ_NEW_QSTR(MP_QSTR___exit__), MP_OBJ_FROM_PTR(&mp_stream___exit___obj) }; + MP_OBJ_TYPE_SET_SLOT(&deflateio_type, locals_dict, (void*)&deflateio_locals_dict, 2); + + mp_store_global(MP_QSTR___name__, MP_OBJ_NEW_QSTR(MP_QSTR_deflate)); + mp_store_global(MP_QSTR_DeflateIO, MP_OBJ_FROM_PTR(&deflateio_type)); + mp_store_global(MP_QSTR_RAW, MP_OBJ_NEW_SMALL_INT(DEFLATEIO_FORMAT_RAW)); + mp_store_global(MP_QSTR_ZLIB, MP_OBJ_NEW_SMALL_INT(DEFLATEIO_FORMAT_ZLIB)); + mp_store_global(MP_QSTR_GZIP, MP_OBJ_NEW_SMALL_INT(DEFLATEIO_FORMAT_GZIP)); + + MP_DYNRUNTIME_INIT_EXIT +} diff --git a/tests/run-natmodtests.py b/tests/run-natmodtests.py index ce4f94135063..576402147214 100755 --- a/tests/run-natmodtests.py +++ b/tests/run-natmodtests.py @@ -21,6 +21,7 @@ # Supported tests and their corresponding mpy module TEST_MAPPINGS = { "btree": "btree/btree_$(ARCH).mpy", + "deflate": "deflate/deflate_$(ARCH).mpy", "framebuf": "framebuf/framebuf_$(ARCH).mpy", "heapq": "heapq/heapq_$(ARCH).mpy", "random": "random/random_$(ARCH).mpy", diff --git a/tools/ci.sh b/tools/ci.sh index bbff5359a1e1..98e78dc78a6f 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -426,6 +426,7 @@ function ci_native_mpy_modules_build { make -C examples/natmod/features2 ARCH=$arch make -C examples/natmod/features3 ARCH=$arch make -C examples/natmod/btree ARCH=$arch + make -C examples/natmod/deflate ARCH=$arch make -C examples/natmod/framebuf ARCH=$arch make -C examples/natmod/heapq ARCH=$arch make -C examples/natmod/random ARCH=$arch @@ -495,7 +496,7 @@ function ci_unix_coverage_run_mpy_merge_tests { function ci_unix_coverage_run_native_mpy_tests { MICROPYPATH=examples/natmod/features2 ./ports/unix/build-coverage/micropython -m features2 - (cd tests && ./run-natmodtests.py "$@" extmod/{btree*,framebuf*,heapq*,random*,re*}.py) + (cd tests && ./run-natmodtests.py "$@" extmod/{btree*,deflate*,framebuf*,heapq*,random*,re*}.py) } function ci_unix_32bit_setup { From 8ef5622b9b8cb8caabb138eaf3b1ecccccd882fa Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 21 Jul 2023 10:55:06 +1000 Subject: [PATCH 15/31] py/runtime: Always initialise sched_state in mp_init. When MICROPY_SCHEDULER_STATIC_NODES is enabled, the logic is unchanged. When MICROPY_SCHEDULER_STATIC_NODES is disable, sched_state is now always initialised to MP_SCHED_IDLE when calling mp_init(). For example, the use of mp_sched_vm_abort(), if it aborts a running scheduled function, can lead to the scheduler starting off in a locked state when the runtime is restarted, and then it stays locked. This commit fixes that case by resetting sched_state. Signed-off-by: Damien George --- py/runtime.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/py/runtime.c b/py/runtime.c index f5d219728f70..c3ada72296d2 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -69,11 +69,10 @@ void mp_init(void) { // no pending exceptions to start with MP_STATE_THREAD(mp_pending_exception) = MP_OBJ_NULL; #if MICROPY_ENABLE_SCHEDULER + // no pending callbacks to start with + MP_STATE_VM(sched_state) = MP_SCHED_IDLE; #if MICROPY_SCHEDULER_STATIC_NODES - if (MP_STATE_VM(sched_head) == NULL) { - // no pending callbacks to start with - MP_STATE_VM(sched_state) = MP_SCHED_IDLE; - } else { + if (MP_STATE_VM(sched_head) != NULL) { // pending callbacks are on the list, eg from before a soft reset MP_STATE_VM(sched_state) = MP_SCHED_PENDING; } From c1acea0e7364c54f6bd6f918dac4e2d4dc227691 Mon Sep 17 00:00:00 2001 From: iabdalkader Date: Sun, 23 Jul 2023 15:43:37 +0300 Subject: [PATCH 16/31] esp32/boards/ARDUINO_NANO_ESP32: Fix deploy instructions. Signed-off-by: iabdalkader --- ports/esp32/boards/ARDUINO_NANO_ESP32/board.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ports/esp32/boards/ARDUINO_NANO_ESP32/board.json b/ports/esp32/boards/ARDUINO_NANO_ESP32/board.json index 031ee17415b6..afb24e10494b 100644 --- a/ports/esp32/boards/ARDUINO_NANO_ESP32/board.json +++ b/ports/esp32/boards/ARDUINO_NANO_ESP32/board.json @@ -1,6 +1,6 @@ { "deploy": [ - "../deploy_s3.md" + "deploy.md" ], "docs": "", "features": [ From 1bde5f3316f4d0e4733b5b4b395a4d341b2392f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20de=20Giessen?= Date: Tue, 31 May 2022 14:55:39 +0200 Subject: [PATCH 17/31] esp32/main: Remove unused mbedtls debug function. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit beeb74 we already check in modussl_mbedtls whether this function is provided by the ESP-IDF before calling it, thus we no longer need to define it here in order to compile. Removing it so that if CONFIG_MBEDTLS_DEBUG is defined we do not cause any 'multiple definition' compile errors. Signed-off-by: Daniël van de Giessen --- ports/esp32/main.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ports/esp32/main.c b/ports/esp32/main.c index b8ba03e513ff..3a172e6f8d4a 100644 --- a/ports/esp32/main.c +++ b/ports/esp32/main.c @@ -216,11 +216,6 @@ void nlr_jump_fail(void *val) { esp_restart(); } -// modussl_mbedtls uses this function but it's not enabled in ESP IDF -void mbedtls_debug_set_threshold(int threshold) { - (void)threshold; -} - void *esp_native_code_commit(void *buf, size_t len, void *reloc) { len = (len + 3) & ~3; uint32_t *p = heap_caps_malloc(len, MALLOC_CAP_EXEC); From 52dc48b2a0da912396bd3224cd9c906f4e6288ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20de=20Giessen?= Date: Tue, 4 Jul 2023 15:35:29 +0200 Subject: [PATCH 18/31] esp32/machine_wdt: Allow feeding WDT from threads. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This changes the ESP32 WDT implementation to use a custom handle so that it becomes possible to reset the WDT from a thread. By default esp_task_wdt_add subscribes the task_id of the current task. That means that if we're running in a different task we are unable to reset the WDT, which prevents feeding the WDT from a thread directly, or even from a timer (which may randomly run in a different task when there's multiple threads). As an added bonus, the name we set makes the error clearly specify that it was the user-specified WDT that reset the chip. Signed-off-by: Daniël van de Giessen --- ports/esp32/machine_wdt.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/ports/esp32/machine_wdt.c b/ports/esp32/machine_wdt.c index 2cb6c518175e..bf924c35e09b 100644 --- a/ports/esp32/machine_wdt.c +++ b/ports/esp32/machine_wdt.c @@ -37,9 +37,12 @@ const mp_obj_type_t machine_wdt_type; typedef struct _machine_wdt_obj_t { mp_obj_base_t base; + esp_task_wdt_user_handle_t twdt_user_handle; } machine_wdt_obj_t; -STATIC machine_wdt_obj_t wdt_default = {{&machine_wdt_type}}; +STATIC machine_wdt_obj_t wdt_default = { + {&machine_wdt_type}, 0 +}; STATIC mp_obj_t machine_wdt_make_new(const mp_obj_type_t *type_in, size_t n_args, size_t n_kw, const mp_obj_t *all_args) { enum { ARG_id, ARG_timeout }; @@ -68,14 +71,22 @@ STATIC mp_obj_t machine_wdt_make_new(const mp_obj_type_t *type_in, size_t n_args mp_raise_OSError(rs_code); } - esp_task_wdt_add(NULL); + if (wdt_default.twdt_user_handle == NULL) { + rs_code = esp_task_wdt_add_user("mpy_machine_wdt", &wdt_default.twdt_user_handle); + if (rs_code != ESP_OK) { + mp_raise_OSError(rs_code); + } + } return &wdt_default; } STATIC mp_obj_t machine_wdt_feed(mp_obj_t self_in) { (void)self_in; - esp_task_wdt_reset(); + mp_int_t rs_code = esp_task_wdt_reset_user(wdt_default.twdt_user_handle); + if (rs_code != ESP_OK) { + mp_raise_OSError(rs_code); + } return mp_const_none; } STATIC MP_DEFINE_CONST_FUN_OBJ_1(machine_wdt_feed_obj, machine_wdt_feed); From 625e03d2dc88df40e9d5a8a29011084881730a2f Mon Sep 17 00:00:00 2001 From: Armin Brauns Date: Thu, 13 Jul 2023 12:05:45 +0000 Subject: [PATCH 19/31] mpy-cross: Allow specifying source files starting with -. Signed-off-by: Armin Brauns --- mpy-cross/main.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mpy-cross/main.c b/mpy-cross/main.c index 8a4dd5bcbed5..cf82759c1726 100644 --- a/mpy-cross/main.c +++ b/mpy-cross/main.c @@ -100,7 +100,7 @@ STATIC int compile_and_save(const char *file, const char *output_file, const cha STATIC int usage(char **argv) { printf( - "usage: %s [] [-X ] \n" + "usage: %s [] [-X ] [--] \n" "Options:\n" "--version : show version information\n" "-o : output file for compiled bytecode (defaults to input with .mpy extension)\n" @@ -220,10 +220,11 @@ MP_NOINLINE int main_(int argc, char **argv) { const char *input_file = NULL; const char *output_file = NULL; const char *source_file = NULL; + bool option_parsing_active = true; // parse main options for (int a = 1; a < argc; a++) { - if (argv[a][0] == '-') { + if (option_parsing_active && argv[a][0] == '-') { if (strcmp(argv[a], "-X") == 0) { a += 1; } else if (strcmp(argv[a], "--version") == 0) { @@ -309,6 +310,8 @@ MP_NOINLINE int main_(int argc, char **argv) { } else { return usage(argv); } + } else if (strcmp(argv[a], "--") == 0) { + option_parsing_active = false; } else { return usage(argv); } From 6a61e4ecd12c8d780bd2e09e017c2ee0eab3ee3e Mon Sep 17 00:00:00 2001 From: Armin Brauns Date: Thu, 13 Jul 2023 12:06:00 +0000 Subject: [PATCH 20/31] mpy-cross: Allow reading from stdin and writing to stdout. Signed-off-by: Armin Brauns --- mpy-cross/main.c | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/mpy-cross/main.c b/mpy-cross/main.c index cf82759c1726..20f3f85dba11 100644 --- a/mpy-cross/main.c +++ b/mpy-cross/main.c @@ -48,6 +48,14 @@ mp_uint_t mp_verbose_flag = 0; // Make it larger on a 64 bit machine, because pointers are larger. long heap_size = 1024 * 1024 * (sizeof(mp_uint_t) / 4); +STATIC void stdout_print_strn(void *env, const char *str, size_t len) { + (void)env; + ssize_t dummy = write(STDOUT_FILENO, str, len); + (void)dummy; +} + +STATIC const mp_print_t mp_stdout_print = {NULL, stdout_print_strn}; + STATIC void stderr_print_strn(void *env, const char *str, size_t len) { (void)env; ssize_t dummy = write(STDERR_FILENO, str, len); @@ -59,7 +67,12 @@ STATIC const mp_print_t mp_stderr_print = {NULL, stderr_print_strn}; STATIC int compile_and_save(const char *file, const char *output_file, const char *source_file) { nlr_buf_t nlr; if (nlr_push(&nlr) == 0) { - mp_lexer_t *lex = mp_lexer_new_from_file(file); + mp_lexer_t *lex; + if (strcmp(file, "-") == 0) { + lex = mp_lexer_new_from_fd(MP_QSTR__lt_stdin_gt_, STDIN_FILENO, false); + } else { + lex = mp_lexer_new_from_file(file); + } qstr source_name; if (source_file == NULL) { @@ -77,17 +90,22 @@ STATIC int compile_and_save(const char *file, const char *output_file, const cha cm.context = m_new_obj(mp_module_context_t); mp_compile_to_raw_code(&parse_tree, source_name, false, &cm); - vstr_t vstr; - vstr_init(&vstr, 16); - if (output_file == NULL) { - vstr_add_str(&vstr, file); - vstr_cut_tail_bytes(&vstr, 2); - vstr_add_str(&vstr, "mpy"); + if (output_file != NULL && strcmp(output_file, "-") == 0) { + mp_raw_code_save(&cm, (mp_print_t *)&mp_stdout_print); } else { - vstr_add_str(&vstr, output_file); + vstr_t vstr; + vstr_init(&vstr, 16); + if (output_file == NULL) { + vstr_add_str(&vstr, file); + vstr_cut_tail_bytes(&vstr, 2); + vstr_add_str(&vstr, "mpy"); + } else { + vstr_add_str(&vstr, output_file); + } + + mp_raw_code_save_file(&cm, vstr_null_terminated_str(&vstr)); + vstr_clear(&vstr); } - mp_raw_code_save_file(&cm, vstr_null_terminated_str(&vstr)); - vstr_clear(&vstr); nlr_pop(); return 0; From 3164749b3d19a35bc3d0064da18ce0e55c4d7f20 Mon Sep 17 00:00:00 2001 From: Armin Brauns Date: Fri, 14 Jul 2023 10:31:33 +0000 Subject: [PATCH 21/31] mpy-cross: When reading from stdin, write output to stdout. Unless -o is given, output defaults to stdout unless a source file is given (in which case the source file name is used to derive an output file name). Signed-off-by: Armin Brauns --- mpy-cross/main.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mpy-cross/main.c b/mpy-cross/main.c index 20f3f85dba11..87e44e379d20 100644 --- a/mpy-cross/main.c +++ b/mpy-cross/main.c @@ -90,7 +90,8 @@ STATIC int compile_and_save(const char *file, const char *output_file, const cha cm.context = m_new_obj(mp_module_context_t); mp_compile_to_raw_code(&parse_tree, source_name, false, &cm); - if (output_file != NULL && strcmp(output_file, "-") == 0) { + if ((output_file != NULL && strcmp(output_file, "-") == 0) || + (output_file == NULL && strcmp(file, "-") == 0)) { mp_raw_code_save(&cm, (mp_print_t *)&mp_stdout_print); } else { vstr_t vstr; @@ -121,7 +122,7 @@ STATIC int usage(char **argv) { "usage: %s [] [-X ] [--] \n" "Options:\n" "--version : show version information\n" - "-o : output file for compiled bytecode (defaults to input with .mpy extension)\n" + "-o : output file for compiled bytecode (defaults to input filename with .mpy extension, or stdout if input is stdin)\n" "-s : source filename to embed in the compiled bytecode (defaults to input file)\n" "-v : verbose (trace various operations); can be multiple\n" "-O[N] : apply bytecode optimizations of level N\n" From 14374850ce54f0e1d446355738b31d4b0ba7aabe Mon Sep 17 00:00:00 2001 From: Armin Brauns Date: Mon, 24 Jul 2023 07:02:06 +0000 Subject: [PATCH 22/31] mpy-cross: Allow specifying stdin as input without --. This way, a bare `-` is never interpreted as an option, even before `--`. Filenames starting with `-` still need to be put after `--`. Signed-off-by: Armin Brauns --- mpy-cross/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mpy-cross/main.c b/mpy-cross/main.c index 87e44e379d20..3ef77d436f85 100644 --- a/mpy-cross/main.c +++ b/mpy-cross/main.c @@ -243,7 +243,7 @@ MP_NOINLINE int main_(int argc, char **argv) { // parse main options for (int a = 1; a < argc; a++) { - if (option_parsing_active && argv[a][0] == '-') { + if (option_parsing_active && argv[a][0] == '-' && argv[a][1] != '\0') { if (strcmp(argv[a], "-X") == 0) { a += 1; } else if (strcmp(argv[a], "--version") == 0) { From 975a687447f5fed27e08001d5760259ec523631c Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Fri, 21 Jul 2023 13:39:36 +1000 Subject: [PATCH 23/31] py/mpconfig: Add MICROPY_PY_PLATFORM, enabled at extra features level. Previously this was explicitly enabled on esp32/stm32/renesas/mimxrt/samd, but didn't get a default feature level because it wasn't in py/mpconfig.h. With this commit it's now enabled at the "extra features" level, which adds rp2, unix-standard, windows, esp8266, webassembly, and some nrf boards. This work was funded through GitHub Sponsors. Signed-off-by: Jim Mussared --- ports/esp32/mpconfigport.h | 1 - ports/mimxrt/mpconfigport.h | 1 - ports/renesas-ra/mpconfigport.h | 3 --- ports/stm32/mpconfigport.h | 3 --- py/mpconfig.h | 5 +++++ tests/unix/extra_coverage.py.exp | 8 ++++---- 6 files changed, 9 insertions(+), 12 deletions(-) diff --git a/ports/esp32/mpconfigport.h b/ports/esp32/mpconfigport.h index 714871d4fc92..aa13eaf2fecf 100644 --- a/ports/esp32/mpconfigport.h +++ b/ports/esp32/mpconfigport.h @@ -140,7 +140,6 @@ #define MICROPY_PY_WEBSOCKET (1) #define MICROPY_PY_WEBREPL (1) #define MICROPY_PY_ONEWIRE (1) -#define MICROPY_PY_PLATFORM (1) #define MICROPY_PY_SOCKET_EVENTS (MICROPY_PY_WEBREPL) #define MICROPY_PY_BLUETOOTH_RANDOM_ADDR (1) #define MICROPY_PY_BLUETOOTH_DEFAULT_GAP_NAME ("ESP32") diff --git a/ports/mimxrt/mpconfigport.h b/ports/mimxrt/mpconfigport.h index ce71cb279534..52054c5d4d6b 100644 --- a/ports/mimxrt/mpconfigport.h +++ b/ports/mimxrt/mpconfigport.h @@ -94,7 +94,6 @@ uint32_t trng_random_u32(void); #define MICROPY_PY_MACHINE_TIMER (1) #define MICROPY_SOFT_TIMER_TICKS_MS systick_ms #define MICROPY_PY_ONEWIRE (1) -#define MICROPY_PY_PLATFORM (1) // fatfs configuration used in ffconf.h #define MICROPY_FATFS_ENABLE_LFN (1) diff --git a/ports/renesas-ra/mpconfigport.h b/ports/renesas-ra/mpconfigport.h index 842de1eff757..34b299bd16c0 100644 --- a/ports/renesas-ra/mpconfigport.h +++ b/ports/renesas-ra/mpconfigport.h @@ -132,9 +132,6 @@ #ifndef MICROPY_PY_ONEWIRE #define MICROPY_PY_ONEWIRE (1) #endif -#ifndef MICROPY_PY_PLATFORM -#define MICROPY_PY_PLATFORM (1) -#endif // fatfs configuration used in ffconf.h #define MICROPY_FATFS_ENABLE_LFN (1) diff --git a/ports/stm32/mpconfigport.h b/ports/stm32/mpconfigport.h index e3e24f4a9d46..37165b158956 100644 --- a/ports/stm32/mpconfigport.h +++ b/ports/stm32/mpconfigport.h @@ -137,9 +137,6 @@ #ifndef MICROPY_PY_ONEWIRE #define MICROPY_PY_ONEWIRE (1) #endif -#ifndef MICROPY_PY_PLATFORM -#define MICROPY_PY_PLATFORM (1) -#endif // fatfs configuration used in ffconf.h #define MICROPY_FATFS_ENABLE_LFN (1) diff --git a/py/mpconfig.h b/py/mpconfig.h index e6f1531ce1bf..c617f573b6c6 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -1726,6 +1726,11 @@ typedef double mp_float_t; #define MICROPY_PY_ONEWIRE (0) #endif +// Whether to provide the "platform" module +#ifndef MICROPY_PY_PLATFORM +#define MICROPY_PY_PLATFORM (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) +#endif + /*****************************************************************************/ /* Hooks for a port to add builtins */ diff --git a/tests/unix/extra_coverage.py.exp b/tests/unix/extra_coverage.py.exp index 352bf39630ac..ec39500746d5 100644 --- a/tests/unix/extra_coverage.py.exp +++ b/tests/unix/extra_coverage.py.exp @@ -56,10 +56,10 @@ cmath collections cppexample cryptolib deflate errno example_package ffi framebuf gc hashlib heapq io json machine -math os random re -select socket ssl struct -sys termios time uctypes -websocket +math os platform random +re select socket ssl +struct sys termios time +uctypes websocket me micropython machine math From aab8061dceb2ea4c6e87a876f48969e4a5a85c94 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 25 Jul 2023 10:52:50 +1000 Subject: [PATCH 24/31] esp32/machine_hw_spi: Fix access of SPI(2). SPI3_HOST is not a macro but rather an enum, so use SOC_SPI_PERIPH_NUM to detect if it's defined. Fixes issue #11919. Signed-off-by: Damien George --- ports/esp32/machine_hw_spi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ports/esp32/machine_hw_spi.c b/ports/esp32/machine_hw_spi.c index 662d0e599419..2f4509f302a5 100644 --- a/ports/esp32/machine_hw_spi.c +++ b/ports/esp32/machine_hw_spi.c @@ -232,7 +232,7 @@ STATIC void machine_hw_spi_init_internal( #ifdef FSPI_HOST && self->host != FSPI_HOST #endif - #ifdef SPI3_HOST + #if SOC_SPI_PERIPH_NUM > 2 && self->host != SPI3_HOST #endif ) { From 862944a71f6a7bce7e6c9d749d63cf539a3bc3d7 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 25 Jul 2023 10:54:41 +1000 Subject: [PATCH 25/31] esp32/machine_hw_spi: Remove unnecessary duplicate SPI pin defaults. Signed-off-by: Damien George --- ports/esp32/machine_hw_spi.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/ports/esp32/machine_hw_spi.c b/ports/esp32/machine_hw_spi.c index 2f4509f302a5..dedf7fc1ee29 100644 --- a/ports/esp32/machine_hw_spi.c +++ b/ports/esp32/machine_hw_spi.c @@ -46,24 +46,13 @@ // Default pins for SPI(id=1) aka IDF SPI2, can be overridden by a board #ifndef MICROPY_HW_SPI1_SCK -#ifdef SPI2_IOMUX_PIN_NUM_CLK // Use IO_MUX pins by default. // If SPI lines are routed to other pins through GPIO matrix // routing adds some delay and lower limit applies to SPI clk freq -#define MICROPY_HW_SPI1_SCK SPI2_IOMUX_PIN_NUM_CLK // pin 14 on ESP32 -#define MICROPY_HW_SPI1_MOSI SPI2_IOMUX_PIN_NUM_MOSI // pin 13 on ESP32 -#define MICROPY_HW_SPI1_MISO SPI2_IOMUX_PIN_NUM_MISO // pin 12 on ESP32 -// Only for compatibility with IDF 4.2 and older -#elif CONFIG_IDF_TARGET_ESP32S2 -#define MICROPY_HW_SPI1_SCK FSPI_IOMUX_PIN_NUM_CLK -#define MICROPY_HW_SPI1_MOSI FSPI_IOMUX_PIN_NUM_MOSI -#define MICROPY_HW_SPI1_MISO FSPI_IOMUX_PIN_NUM_MISO -#else #define MICROPY_HW_SPI1_SCK SPI2_IOMUX_PIN_NUM_CLK #define MICROPY_HW_SPI1_MOSI SPI2_IOMUX_PIN_NUM_MOSI #define MICROPY_HW_SPI1_MISO SPI2_IOMUX_PIN_NUM_MISO #endif -#endif // Default pins for SPI(id=2) aka IDF SPI3, can be overridden by a board #ifndef MICROPY_HW_SPI2_SCK From 162dd022b188b3e26738bb692417480a0c5546c4 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 25 Jul 2023 11:10:58 +1000 Subject: [PATCH 26/31] esp32/machine_hw_spi: Remove SPI host renaming for C3 and S3 variants. On ESP32C3 it's not doing anything. On ESP32S3 the original code prevented prevented machine.SPI(1) from working. Signed-off-by: Damien George --- ports/esp32/machine_hw_spi.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/ports/esp32/machine_hw_spi.c b/ports/esp32/machine_hw_spi.c index dedf7fc1ee29..b00b5881a302 100644 --- a/ports/esp32/machine_hw_spi.c +++ b/ports/esp32/machine_hw_spi.c @@ -73,13 +73,6 @@ #define MP_HW_SPI_MAX_XFER_BYTES (4092) #define MP_HW_SPI_MAX_XFER_BITS (MP_HW_SPI_MAX_XFER_BYTES * 8) // Has to be an even multiple of 8 -#if CONFIG_IDF_TARGET_ESP32C3 -#define SPI2_HOST SPI2_HOST -#elif CONFIG_IDF_TARGET_ESP32S3 -#define SPI2_HOST SPI3_HOST -#define FSPI_HOST SPI2_HOST -#endif - typedef struct _machine_hw_spi_default_pins_t { int8_t sck; int8_t mosi; @@ -218,9 +211,6 @@ STATIC void machine_hw_spi_init_internal( } if (self->host != SPI2_HOST - #ifdef FSPI_HOST - && self->host != FSPI_HOST - #endif #if SOC_SPI_PERIPH_NUM > 2 && self->host != SPI3_HOST #endif From b2adfc8077435419c5a5eecb2818cba44f1d3f8f Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 25 Jul 2023 11:33:51 +1000 Subject: [PATCH 27/31] esp32/machine_hw_spi: Check for valid SPI id in constructor, not init. Otherwise constructing an invalid SPI instance (eg machine.SPI(3)) will mess up machine.SPI(2)'s state before it's detected that it's an invalid SPI id. Signed-off-by: Damien George --- ports/esp32/machine_hw_spi.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/ports/esp32/machine_hw_spi.c b/ports/esp32/machine_hw_spi.c index b00b5881a302..e09c493f5b94 100644 --- a/ports/esp32/machine_hw_spi.c +++ b/ports/esp32/machine_hw_spi.c @@ -44,6 +44,13 @@ // SPI(id=1) | HSPI/SPI2 | FSPI/SPI2 | SPI2 | SPI2 // SPI(id=2) | VSPI/SPI3 | HSPI/SPI3 | SPI3 | err +// Number of available hardware SPI peripherals. +#if SOC_SPI_PERIPH_NUM > 2 +#define MICROPY_HW_SPI_MAX (2) +#else +#define MICROPY_HW_SPI_MAX (1) +#endif + // Default pins for SPI(id=1) aka IDF SPI2, can be overridden by a board #ifndef MICROPY_HW_SPI1_SCK // Use IO_MUX pins by default. @@ -99,15 +106,15 @@ typedef struct _machine_hw_spi_obj_t { } machine_hw_spi_obj_t; // Default pin mappings for the hardware SPI instances -STATIC const machine_hw_spi_default_pins_t machine_hw_spi_default_pins[2] = { +STATIC const machine_hw_spi_default_pins_t machine_hw_spi_default_pins[MICROPY_HW_SPI_MAX] = { { .sck = MICROPY_HW_SPI1_SCK, .mosi = MICROPY_HW_SPI1_MOSI, .miso = MICROPY_HW_SPI1_MISO }, #ifdef MICROPY_HW_SPI2_SCK { .sck = MICROPY_HW_SPI2_SCK, .mosi = MICROPY_HW_SPI2_MOSI, .miso = MICROPY_HW_SPI2_MISO }, #endif }; -// Static objects mapping to SPI2 and SPI3 hardware peripherals -STATIC machine_hw_spi_obj_t machine_hw_spi_obj[2]; +// Static objects mapping to SPI2 (and SPI3 if available) hardware peripherals. +STATIC machine_hw_spi_obj_t machine_hw_spi_obj[MICROPY_HW_SPI_MAX]; STATIC void machine_hw_spi_deinit_internal(machine_hw_spi_obj_t *self) { switch (spi_bus_remove_device(self->spi)) { @@ -210,14 +217,6 @@ STATIC void machine_hw_spi_init_internal( changed = true; } - if (self->host != SPI2_HOST - #if SOC_SPI_PERIPH_NUM > 2 - && self->host != SPI3_HOST - #endif - ) { - mp_raise_msg_varg(&mp_type_ValueError, MP_ERROR_TEXT("SPI(%d) doesn't exist"), self->host); - } - if (changed) { if (self->state == MACHINE_HW_SPI_STATE_INIT) { self->state = MACHINE_HW_SPI_STATE_DEINIT; @@ -464,13 +463,14 @@ mp_obj_t machine_hw_spi_make_new(const mp_obj_type_t *type, size_t n_args, size_ machine_hw_spi_obj_t *self; const machine_hw_spi_default_pins_t *default_pins; - if (args[ARG_id].u_int == 1) { // SPI2_HOST which is FSPI_HOST on ESP32Sx, SPI2_HOST on others - self = &machine_hw_spi_obj[0]; - default_pins = &machine_hw_spi_default_pins[0]; + mp_int_t spi_id = args[ARG_id].u_int; + if (1 <= spi_id && spi_id <= MICROPY_HW_SPI_MAX) { + self = &machine_hw_spi_obj[spi_id - 1]; + default_pins = &machine_hw_spi_default_pins[spi_id - 1]; } else { - self = &machine_hw_spi_obj[1]; - default_pins = &machine_hw_spi_default_pins[1]; + mp_raise_msg_varg(&mp_type_ValueError, MP_ERROR_TEXT("SPI(%d) doesn't exist"), spi_id); } + self->base.type = &machine_spi_type; int8_t sck, mosi, miso; From c9d2c5537b0f911b494fc877e2006022a65c2973 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 25 Jul 2023 12:35:21 +1000 Subject: [PATCH 28/31] esp32/boards: Remove references to the IDF version in board.md files. Listing the IDF version number in the board description is not as important as it once was, when the IDF was still undergoing a lot of changes. Now, all builds use IDF 5.x and it's possible to query the exact version with platform.platform(). Signed-off-by: Damien George --- ports/esp32/boards/GENERIC/board.md | 4 +--- ports/esp32/boards/GENERIC_SPIRAM/board.md | 4 +--- ports/esp32/boards/OLIMEX_ESP32_POE/board.md | 4 +--- ports/esp32/boards/UM_FEATHERS2/board.md | 3 +-- ports/esp32/boards/UM_FEATHERS2NEO/board.md | 3 +-- ports/esp32/boards/UM_FEATHERS3/board.md | 3 +-- ports/esp32/boards/UM_PROS3/board.md | 3 +-- ports/esp32/boards/UM_TINYPICO/board.md | 4 +--- ports/esp32/boards/UM_TINYS2/board.md | 3 +-- ports/esp32/boards/UM_TINYS3/board.md | 3 +-- 10 files changed, 10 insertions(+), 24 deletions(-) diff --git a/ports/esp32/boards/GENERIC/board.md b/ports/esp32/boards/GENERIC/board.md index 576ea80450f6..efb2b2cab7c2 100644 --- a/ports/esp32/boards/GENERIC/board.md +++ b/ports/esp32/boards/GENERIC/board.md @@ -1,3 +1 @@ -The following files are daily firmware for ESP32-based boards without external SPIRAM. - -This firmware is compiled using ESP-IDF v4.x. Some older releases are also provided that are compiled with ESP-IDF v3.x. +The following files are firmware for ESP32-based boards without external SPIRAM. diff --git a/ports/esp32/boards/GENERIC_SPIRAM/board.md b/ports/esp32/boards/GENERIC_SPIRAM/board.md index 3b02b902cd75..c638f63dd3e1 100644 --- a/ports/esp32/boards/GENERIC_SPIRAM/board.md +++ b/ports/esp32/boards/GENERIC_SPIRAM/board.md @@ -1,3 +1 @@ -The following files are daily firmware for ESP32-based boards with external SPIRAM (also known as PSRAM). - -This firmware is compiled using ESP-IDF v4.x. Some older releases are also provided that are compiled with ESP-IDF v3.x. +The following files are firmware for ESP32-based boards with external SPIRAM (also known as PSRAM). diff --git a/ports/esp32/boards/OLIMEX_ESP32_POE/board.md b/ports/esp32/boards/OLIMEX_ESP32_POE/board.md index 1e2dbb744a0e..f077deda95cb 100644 --- a/ports/esp32/boards/OLIMEX_ESP32_POE/board.md +++ b/ports/esp32/boards/OLIMEX_ESP32_POE/board.md @@ -1,4 +1,2 @@ -The following files are daily firmware for Olimex ESP32 boards with Ethernet. +The following files are firmware for Olimex ESP32 boards with Ethernet. They match the boards ESP32 ETH-PoE, ESP32 ETH-PoE-ISO and ESP32 Gateway. - -This firmware is compiled using ESP-IDF v4.x. diff --git a/ports/esp32/boards/UM_FEATHERS2/board.md b/ports/esp32/boards/UM_FEATHERS2/board.md index e04a8936c846..86ca33382d36 100644 --- a/ports/esp32/boards/UM_FEATHERS2/board.md +++ b/ports/esp32/boards/UM_FEATHERS2/board.md @@ -1,2 +1 @@ -The following files are daily firmware for the FeatherS2. This firmware is -compiled using ESP-IDF v4.3 or later. +The following files are firmware for the FeatherS2. diff --git a/ports/esp32/boards/UM_FEATHERS2NEO/board.md b/ports/esp32/boards/UM_FEATHERS2NEO/board.md index 60ee024a1261..26012d0bae5d 100644 --- a/ports/esp32/boards/UM_FEATHERS2NEO/board.md +++ b/ports/esp32/boards/UM_FEATHERS2NEO/board.md @@ -1,2 +1 @@ -The following files are daily firmware for the FeatherS2 Neo. This firmware is -compiled using ESP-IDF v4.3 or later. +The following files are firmware for the FeatherS2 Neo. diff --git a/ports/esp32/boards/UM_FEATHERS3/board.md b/ports/esp32/boards/UM_FEATHERS3/board.md index ca9c36ad331f..bed179041c28 100644 --- a/ports/esp32/boards/UM_FEATHERS3/board.md +++ b/ports/esp32/boards/UM_FEATHERS3/board.md @@ -1,2 +1 @@ -The following files are daily firmware for the FeatherS3. This firmware is -compiled using ESP-IDF v4.4 or later. +The following files are firmware for the FeatherS3. diff --git a/ports/esp32/boards/UM_PROS3/board.md b/ports/esp32/boards/UM_PROS3/board.md index 4d1c435a07d1..406abf74e3a1 100644 --- a/ports/esp32/boards/UM_PROS3/board.md +++ b/ports/esp32/boards/UM_PROS3/board.md @@ -1,2 +1 @@ -The following files are daily firmware for the ProS3. This firmware is -compiled using ESP-IDF v4.4 or later. +The following files are firmware for the ProS3. diff --git a/ports/esp32/boards/UM_TINYPICO/board.md b/ports/esp32/boards/UM_TINYPICO/board.md index d0b1266d2eec..0fb41c76e551 100644 --- a/ports/esp32/boards/UM_TINYPICO/board.md +++ b/ports/esp32/boards/UM_TINYPICO/board.md @@ -1,3 +1 @@ -The following files are daily firmware for the TinyPICO. This firmware is compiled -using ESP-IDF v4.2 or later. Some older releases are also provided that are -compiled with ESP-IDF v3.x. +The following files are firmware for the TinyPICO. diff --git a/ports/esp32/boards/UM_TINYS2/board.md b/ports/esp32/boards/UM_TINYS2/board.md index 9a61374edf32..739453da8200 100644 --- a/ports/esp32/boards/UM_TINYS2/board.md +++ b/ports/esp32/boards/UM_TINYS2/board.md @@ -1,2 +1 @@ -The following files are daily firmware for the TinyS2. This firmware is compiled -using ESP-IDF v4.3 or later. +The following files are firmware for the TinyS2. diff --git a/ports/esp32/boards/UM_TINYS3/board.md b/ports/esp32/boards/UM_TINYS3/board.md index da06e191f28c..5b4b5d8f1269 100644 --- a/ports/esp32/boards/UM_TINYS3/board.md +++ b/ports/esp32/boards/UM_TINYS3/board.md @@ -1,2 +1 @@ -The following files are daily firmware for the TinyS3. This firmware is -compiled using ESP-IDF v4.4 or later. +The following files are firmware for the TinyS3. From cfcce4b5311c7b112d3628392c8012b4c29bdd63 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 25 Jul 2023 12:59:02 +1000 Subject: [PATCH 29/31] esp32/README: Specify that only IDF v5.0.2 is supported. Signed-off-by: Damien George --- ports/esp32/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ports/esp32/README.md b/ports/esp32/README.md index dc9e9f8b723a..2d5b05b9266a 100644 --- a/ports/esp32/README.md +++ b/ports/esp32/README.md @@ -28,8 +28,7 @@ manage the ESP32 microcontroller, as well as a way to manage the required build environment and toolchains needed to build the firmware. The ESP-IDF changes quickly and MicroPython only supports certain versions. -Currently MicroPython supports v5.0.2, -although other IDF v4 versions may also work. +Currently MicroPython supports only v5.0.2. To install the ESP-IDF the full instructions can be found at the [Espressif Getting Started guide](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/get-started/index.html#installation-step-by-step). From d14ddcbdb5f373bf73df8e7ff9e3b1a60d7d5e25 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 25 Jul 2023 00:15:15 +1000 Subject: [PATCH 30/31] tools/autobuild: Add support for application .bin files for esp32. On esp32, the build output consists of: - micropython.elf - micropython.map - micropython.bin -- application only - micropython.uf2 -- application only - firmware.bin -- bootloader, partition table and application Currently everything is available at the download page except micropython.bin. This commit adds that file but with the extension changed to .app-bin, to distinguish it from .bin (the full thing). Signed-off-by: Damien George --- tools/autobuild/build-boards.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/autobuild/build-boards.sh b/tools/autobuild/build-boards.sh index aae9981f94a3..689444a98875 100755 --- a/tools/autobuild/build-boards.sh +++ b/tools/autobuild/build-boards.sh @@ -33,6 +33,9 @@ function build_board { elif [ -r $build_dir/micropython.$ext ]; then # esp32 has micropython.elf, etc mv $build_dir/micropython.$ext $dest + elif [ $ext = app-bin -a -r $build_dir/micropython.bin ]; then + # esp32 has micropython.bin which is just the application + mv $build_dir/micropython.bin $dest fi done ) @@ -62,7 +65,7 @@ function build_boards { } function build_esp32_boards { - build_boards modesp32.c $1 $2 bin elf map uf2 + build_boards modesp32.c $1 $2 bin elf map uf2 app-bin } function build_mimxrt_boards { From 01c758e26a846eca42e9c4444639fa822adf005f Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 26 Jul 2023 17:00:03 -0700 Subject: [PATCH 31/31] unix/README: Fix Markdown link markup. Signed-off-by: Brett Cannon --- ports/unix/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ports/unix/README.md b/ports/unix/README.md index a3159a3262e8..a1af28996858 100644 --- a/ports/unix/README.md +++ b/ports/unix/README.md @@ -40,8 +40,8 @@ or >>> import mip >>> mip.install("hmac") -Browse available modules at [micropython-lib] -(https://github.com/micropython/micropython-lib). See +Browse available modules at +[micropython-lib](https://github.com/micropython/micropython-lib). See [Package management](https://docs.micropython.org/en/latest/reference/packages.html) for more information about `mip`.