extmod/vfs_rom: Add VfsRom filesystem object.

This commit defines a new ROMFS filesystem for storing read-only files that
can be memory mapped, and a new VfsRom driver.  Files opened from this
filesystem support the buffer protocol.  This allows naturally getting the
memory-mapped address of the file using:
- memoryview(file)
- uctypes.addressof(file)

Furthermore, if these files are .mpy files then their content can be
referenced in-place when importing.  Such imports take up a lot less RAM
than importing from a normal filesystem.  This is essentially dynamically
frozen .mpy files, building on the revamped v6 .mpy file format.

Signed-off-by: Damien George <damien@micropython.org>

extmod/extmod.cmake

@@ -58,6 +58,8 @@ set(MICROPY_SOURCE_EXTMOD
     ${MICROPY_EXTMOD_DIR}/vfs_fat_diskio.c
     ${MICROPY_EXTMOD_DIR}/vfs_fat_file.c
     ${MICROPY_EXTMOD_DIR}/vfs_lfs.c
+    ${MICROPY_EXTMOD_DIR}/vfs_rom.c
+    ${MICROPY_EXTMOD_DIR}/vfs_rom_file.c
     ${MICROPY_EXTMOD_DIR}/vfs_posix.c
     ${MICROPY_EXTMOD_DIR}/vfs_posix_file.c
     ${MICROPY_EXTMOD_DIR}/vfs_reader.c

extmod/extmod.mk

@@ -61,6 +61,8 @@ SRC_EXTMOD_C += \
 	extmod/vfs_fat_diskio.c \
 	extmod/vfs_fat_file.c \
 	extmod/vfs_lfs.c \
+	extmod/vfs_rom.c \
+	extmod/vfs_rom_file.c \
 	extmod/vfs_posix.c \
 	extmod/vfs_posix_file.c \
 	extmod/vfs_reader.c \

extmod/modvfs.c

@@ -32,6 +32,7 @@
 #include "extmod/vfs_fat.h"
 #include "extmod/vfs_lfs.h"
 #include "extmod/vfs_posix.h"
+#include "extmod/vfs_rom.h"
 
 #if !MICROPY_VFS
 #error "MICROPY_PY_VFS requires MICROPY_VFS"
@@ -51,6 +52,9 @@ static const mp_rom_map_elem_t vfs_module_globals_table[] = {
     #if MICROPY_VFS_LFS2
     { MP_ROM_QSTR(MP_QSTR_VfsLfs2), MP_ROM_PTR(&mp_type_vfs_lfs2) },
     #endif
+    #if MICROPY_VFS_ROM
+    { MP_ROM_QSTR(MP_QSTR_VfsRom), MP_ROM_PTR(&mp_type_vfs_rom) },
+    #endif
     #if MICROPY_VFS_POSIX
     { MP_ROM_QSTR(MP_QSTR_VfsPosix), MP_ROM_PTR(&mp_type_vfs_posix) },
     #endif

extmod/vfs_rom.c

@@ -0,0 +1,399 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2022 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+// ROMFS filesystem format
+// =======================
+//
+// ROMFS is a flexible and extensible filesystem format designed to represent a
+// directory hierarchy with files, where those files are read-only and their data
+// can be memory mapped.
+//
+// Concepts:
+// - varuint: An unsigned integer that is encoded in a variable number of bytes. It is
+//   stored big-endian with the high bit of the byte set if there are following bytes.
+// - record: A variable sized element with a type.  It is stored as two varuint's and then
+//   a payload.  The first varuint is the record kind and the second varuint is the
+//   payload length (which may be zero bytes long).
+//
+// A ROMFS filesystem is a record with record kind 0x14a6b1, chosen so the encoded value
+// is 0xd2-0xcd-0x31 which is "RM1" with the first two bytes having their high bit set.
+// If the ROMFS record's payload is non-empty then it contains records.
+//
+// Record types:
+// - 0 = unused, can be used to detect corruption of the filesystem.
+// - 1 = padding/comments, can contain any data in their payload.
+// - 2 = verbatim data, used to store file data.
+// - 3 = indirect data, pointer to offset within the ROMFS payload.
+// - 4 = a directory: payload contains a varuint which is the length of the directory
+//       name in bytes, then the name, then optional nested records for the contents
+//       of the directory (including optional metadata).
+// - 5 = a file: payload contains a varuint which is the length of the filename in bytes
+//       then the name, then optional nested records.
+//
+// Remarks:
+// - A varuint can be padded if needed by prepending with one or more 0x80 bytes.  This
+//   padding does not change any semantics.
+// - The size of the ROMFS record (including kind and length and payload) must be a
+//   multiple of 2 (because it's not possible to add a padding record of one byte).
+// - File data can be optionally aligned using padding records and/or indirect data
+//   records.
+// - There is no limit to the size of directory/file names or file data.
+//
+// Unknown record types must be skipped over.  They may in the future add optional
+// features, while still retaining backwards compatibility.  Such features may be:
+// - Alignment requirements of the ROMFS record.
+// - Timestamps on directories/files.
+// - A precomputed hash of a file, or other metadata.
+// - An optimised lookup table indexing the directory hierarchy.
+
+#include <string.h>
+
+#include "py/bc.h"
+#include "py/runtime.h"
+#include "py/mperrno.h"
+#include "extmod/vfs.h"
+#include "extmod/vfs_rom.h"
+
+#if MICROPY_VFS_ROM
+
+#define ROMFS_SIZE_MIN (4)
+#define ROMFS_HEADER_BYTE0 (0x80 | 'R')
+#define ROMFS_HEADER_BYTE1 (0x80 | 'M')
+#define ROMFS_HEADER_BYTE2 (0x00 | '1')
+
+// Values for `record_kind_t`.
+#define ROMFS_RECORD_KIND_UNUSED (0)
+#define ROMFS_RECORD_KIND_PADDING (1)
+#define ROMFS_RECORD_KIND_DATA_VERBATIM (2)
+#define ROMFS_RECORD_KIND_DATA_POINTER (3)
+#define ROMFS_RECORD_KIND_DIRECTORY (4)
+#define ROMFS_RECORD_KIND_FILE (5)
+
+typedef mp_uint_t record_kind_t;
+
+struct _mp_obj_vfs_rom_t {
+    mp_obj_base_t base;
+    mp_obj_t memory;
+    const uint8_t *filesystem;
+    const uint8_t *filesystem_end;
+};
+
+static record_kind_t extract_record(const uint8_t **fs, const uint8_t **fs_next) {
+    record_kind_t record_kind = mp_decode_uint(fs);
+    mp_uint_t record_len = mp_decode_uint(fs);
+    *fs_next = *fs + record_len;
+    return record_kind;
+}
+
+static void extract_data(mp_obj_vfs_rom_t *self, const uint8_t *fs, const uint8_t *fs_top, size_t *size_out, const uint8_t **data_out) {
+    *size_out = 0;
+    *data_out = NULL;
+    while (fs < fs_top) {
+        const uint8_t *fs_next;
+        record_kind_t record_kind = extract_record(&fs, &fs_next);
+        if (record_kind == ROMFS_RECORD_KIND_DATA_VERBATIM) {
+            // Verbatim data.
+            *size_out = fs_next - fs;
+            *data_out = fs;
+            break;
+        } else if (record_kind == ROMFS_RECORD_KIND_DATA_POINTER) {
+            // Pointer to data.
+            *size_out = mp_decode_uint(&fs);
+            *data_out = self->filesystem + mp_decode_uint(&fs);
+            break;
+        } else {
+            // Skip this record.
+            fs = fs_next;
+        }
+    }
+}
+
+// Searches for `path` in the filesystem.
+// `path` must be null-terminated.
+mp_import_stat_t mp_vfs_rom_search_filesystem(mp_obj_vfs_rom_t *self, const char *path, size_t *size_out, const uint8_t **data_out) {
+    const uint8_t *fs = self->filesystem;
+    const uint8_t *fs_top = self->filesystem_end;
+    size_t path_len = strlen(path);
+    if (*path == '/') {
+        // An optional slash at the start of the path enters the top-level filesystem.
+        ++path;
+        --path_len;
+    }
+    while (path_len > 0 && fs < fs_top) {
+        const uint8_t *fs_next;
+        record_kind_t record_kind = extract_record(&fs, &fs_next);
+        if (record_kind == ROMFS_RECORD_KIND_DIRECTORY || record_kind == ROMFS_RECORD_KIND_FILE) {
+            // A directory or file record.
+            mp_uint_t name_len = mp_decode_uint(&fs);
+            if ((name_len == path_len
+                 || (name_len < path_len && path[name_len] == '/'))
+                && memcmp(path, fs, name_len) == 0) {
+                // Name matches, so enter this record.
+                fs += name_len;
+                fs_top = fs_next;
+                path += name_len;
+                path_len -= name_len;
+                if (record_kind == ROMFS_RECORD_KIND_DIRECTORY) {
+                    // Continue searching in this directory.
+                    if (*path == '/') {
+                        ++path;
+                        --path_len;
+                    }
+                } else {
+                    // Return this file.
+                    if (path_len != 0) {
+                        return MP_IMPORT_STAT_NO_EXIST;
+                    }
+                    if (size_out != NULL) {
+                        extract_data(self, fs, fs_top, size_out, data_out);
+                    }
+                    return MP_IMPORT_STAT_FILE;
+                }
+            } else {
+                // Skip this directory/file record.
+                fs = fs_next;
+            }
+        } else {
+            // Skip this record.
+            fs = fs_next;
+        }
+    }
+    if (path_len == 0) {
+        if (size_out != NULL) {
+            *size_out = fs_top - fs;
+            *data_out = fs;
+        }
+        return MP_IMPORT_STAT_DIR;
+    }
+    return MP_IMPORT_STAT_NO_EXIST;
+}
+
+static mp_obj_t vfs_rom_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) {
+    mp_arg_check_num(n_args, n_kw, 1, 1, false);
+
+    mp_obj_vfs_rom_t *self = m_new_obj(mp_obj_vfs_rom_t);
+    self->base.type = type;
+    self->memory = args[0];
+
+    mp_buffer_info_t bufinfo;
+    if (mp_get_buffer(self->memory, &bufinfo, MP_BUFFER_READ)) {
+        if (bufinfo.len < ROMFS_SIZE_MIN) {
+            mp_raise_OSError(MP_ENODEV);
+        }
+        self->filesystem = bufinfo.buf;
+    } else {
+        self->filesystem = (const uint8_t *)(uintptr_t)mp_obj_get_int_truncated(self->memory);
+    }
+
+    // Verify it is a ROMFS.
+    if (!(self->filesystem[0] == ROMFS_HEADER_BYTE0
+          && self->filesystem[1] == ROMFS_HEADER_BYTE1
+          && self->filesystem[2] == ROMFS_HEADER_BYTE2)) {
+        mp_raise_OSError(MP_ENODEV);
+    }
+
+    // The ROMFS is a record itself, so enter into it and compute its limit.
+    extract_record(&self->filesystem, &self->filesystem_end);
+
+    return MP_OBJ_FROM_PTR(self);
+}
+
+static mp_obj_t vfs_rom_mount(mp_obj_t self_in, mp_obj_t readonly, mp_obj_t mkfs) {
+    (void)self_in;
+    (void)readonly;
+    if (mp_obj_is_true(mkfs)) {
+        mp_raise_OSError(MP_EPERM);
+    }
+    return mp_const_none;
+}
+static MP_DEFINE_CONST_FUN_OBJ_3(vfs_rom_mount_obj, vfs_rom_mount);
+
+// mp_vfs_rom_file_open is implemented in vfs_rom_file.c.
+static MP_DEFINE_CONST_FUN_OBJ_3(vfs_rom_open_obj, mp_vfs_rom_file_open);
+
+static mp_obj_t vfs_rom_chdir(mp_obj_t self_in, mp_obj_t path_in) {
+    mp_obj_vfs_rom_t *self = MP_OBJ_TO_PTR(self_in);
+    const char *path = mp_vfs_rom_get_path_str(self, path_in);
+    if (path[0] == '/' && path[1] == '\0') {
+        // Allow chdir to the root of the filesystem.
+    } else {
+        // Don't allow chdir to any subdirectory (not currently implemented).
+        mp_raise_OSError(MP_EOPNOTSUPP);
+    }
+    return mp_const_none;
+}
+static MP_DEFINE_CONST_FUN_OBJ_2(vfs_rom_chdir_obj, vfs_rom_chdir);
+
+typedef struct _vfs_rom_ilistdir_it_t {
+    mp_obj_base_t base;
+    mp_fun_1_t iternext;
+    mp_obj_vfs_rom_t *vfs_rom;
+    bool is_str;
+    const uint8_t *index;
+    const uint8_t *index_top;
+} vfs_rom_ilistdir_it_t;
+
+static mp_obj_t vfs_rom_ilistdir_it_iternext(mp_obj_t self_in) {
+    vfs_rom_ilistdir_it_t *self = MP_OBJ_TO_PTR(self_in);
+
+    while (self->index < self->index_top) {
+        const uint8_t *index_next;
+        record_kind_t record_kind = extract_record(&self->index, &index_next);
+        uint32_t type;
+        mp_uint_t name_len;
+        size_t data_len;
+        if (record_kind == ROMFS_RECORD_KIND_DIRECTORY || record_kind == ROMFS_RECORD_KIND_FILE) {
+            // A directory or file record.
+            name_len = mp_decode_uint(&self->index);
+            if (record_kind == ROMFS_RECORD_KIND_DIRECTORY) {
+                // A directory.
+                type = MP_S_IFDIR;
+                data_len = index_next - self->index - name_len;
+            } else {
+                // A file.
+                type = MP_S_IFREG;
+                const uint8_t *data_value;
+                extract_data(self->vfs_rom, self->index + name_len, index_next, &data_len, &data_value);
+            }
+        } else {
+            // Skip this record.
+            self->index = index_next;
+            continue;
+        }
+
+        const uint8_t *name_str = self->index;
+        self->index = index_next;
+
+        // Make 4-tuple with info about this entry: (name, attr, inode, size)
+        mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(4, NULL));
+
+        if (self->is_str) {
+            t->items[0] = mp_obj_new_str((const char *)name_str, name_len);
+        } else {
+            t->items[0] = mp_obj_new_bytes(name_str, name_len);
+        }
+
+        t->items[1] = MP_OBJ_NEW_SMALL_INT(type);
+        t->items[2] = MP_OBJ_NEW_SMALL_INT(0);
+        t->items[3] = mp_obj_new_int(data_len);
+
+        return MP_OBJ_FROM_PTR(t);
+    }
+
+    return MP_OBJ_STOP_ITERATION;
+}
+
+static mp_obj_t vfs_rom_ilistdir(mp_obj_t self_in, mp_obj_t path_in) {
+    mp_obj_vfs_rom_t *self = MP_OBJ_TO_PTR(self_in);
+    vfs_rom_ilistdir_it_t *iter = m_new_obj(vfs_rom_ilistdir_it_t);
+    iter->base.type = &mp_type_polymorph_iter;
+    iter->iternext = vfs_rom_ilistdir_it_iternext;
+    iter->vfs_rom = self;
+    iter->is_str = mp_obj_get_type(path_in) == &mp_type_str;
+    const char *path = mp_vfs_rom_get_path_str(self, path_in);
+    size_t size;
+    if (mp_vfs_rom_search_filesystem(self, path, &size, &iter->index) != MP_IMPORT_STAT_DIR) {
+        mp_raise_OSError(MP_ENOENT);
+    }
+    iter->index_top = iter->index + size;
+    return MP_OBJ_FROM_PTR(iter);
+}
+static MP_DEFINE_CONST_FUN_OBJ_2(vfs_rom_ilistdir_obj, vfs_rom_ilistdir);
+
+static mp_obj_t vfs_rom_stat(mp_obj_t self_in, mp_obj_t path_in) {
+    mp_obj_vfs_rom_t *self = MP_OBJ_TO_PTR(self_in);
+    const char *path = mp_vfs_rom_get_path_str(self, path_in);
+    size_t file_size;
+    const uint8_t *file_data;
+    mp_import_stat_t stat = mp_vfs_rom_search_filesystem(self, path, &file_size, &file_data);
+    if (stat == MP_IMPORT_STAT_NO_EXIST) {
+        mp_raise_OSError(MP_ENOENT);
+    }
+    mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL));
+    t->items[0] = MP_OBJ_NEW_SMALL_INT(stat == MP_IMPORT_STAT_FILE ? MP_S_IFREG : MP_S_IFDIR); // st_mode
+    t->items[1] = MP_OBJ_NEW_SMALL_INT(0); // st_ino
+    t->items[2] = MP_OBJ_NEW_SMALL_INT(0); // st_dev
+    t->items[3] = MP_OBJ_NEW_SMALL_INT(0); // st_nlink
+    t->items[4] = MP_OBJ_NEW_SMALL_INT(0); // st_uid
+    t->items[5] = MP_OBJ_NEW_SMALL_INT(0); // st_gid
+    t->items[6] = MP_OBJ_NEW_SMALL_INT(file_size); // st_size
+    t->items[7] = MP_OBJ_NEW_SMALL_INT(0); // st_atime
+    t->items[8] = MP_OBJ_NEW_SMALL_INT(0); // st_mtime
+    t->items[9] = MP_OBJ_NEW_SMALL_INT(0); // st_ctime
+    return MP_OBJ_FROM_PTR(t);
+}
+static MP_DEFINE_CONST_FUN_OBJ_2(vfs_rom_stat_obj, vfs_rom_stat);
+
+static mp_obj_t vfs_rom_statvfs(mp_obj_t self_in, mp_obj_t path_in) {
+    mp_obj_vfs_rom_t *self = MP_OBJ_TO_PTR(self_in);
+    (void)path_in;
+    size_t filesystem_len = self->filesystem_end - self->filesystem;
+    mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL));
+    t->items[0] = MP_OBJ_NEW_SMALL_INT(1); // f_bsize
+    t->items[1] = MP_OBJ_NEW_SMALL_INT(0); // f_frsize
+    t->items[2] = mp_obj_new_int_from_uint(filesystem_len); // f_blocks
+    t->items[3] = MP_OBJ_NEW_SMALL_INT(0); // f_bfree
+    t->items[4] = MP_OBJ_NEW_SMALL_INT(0); // f_bavail
+    t->items[5] = MP_OBJ_NEW_SMALL_INT(0); // f_files
+    t->items[6] = MP_OBJ_NEW_SMALL_INT(0); // f_ffree
+    t->items[7] = MP_OBJ_NEW_SMALL_INT(0); // f_favail
+    t->items[8] = MP_OBJ_NEW_SMALL_INT(0); // f_flags
+    t->items[9] = MP_OBJ_NEW_SMALL_INT(32767); // f_namemax
+    return MP_OBJ_FROM_PTR(t);
+}
+static MP_DEFINE_CONST_FUN_OBJ_2(vfs_rom_statvfs_obj, vfs_rom_statvfs);
+
+static const mp_rom_map_elem_t vfs_rom_locals_dict_table[] = {
+    { MP_ROM_QSTR(MP_QSTR_mount), MP_ROM_PTR(&vfs_rom_mount_obj) },
+    { MP_ROM_QSTR(MP_QSTR_umount), MP_ROM_PTR(&mp_identity_obj) },
+    { MP_ROM_QSTR(MP_QSTR_open), MP_ROM_PTR(&vfs_rom_open_obj) },
+
+    { MP_ROM_QSTR(MP_QSTR_chdir), MP_ROM_PTR(&vfs_rom_chdir_obj) },
+    { MP_ROM_QSTR(MP_QSTR_ilistdir), MP_ROM_PTR(&vfs_rom_ilistdir_obj) },
+    { MP_ROM_QSTR(MP_QSTR_stat), MP_ROM_PTR(&vfs_rom_stat_obj) },
+    { MP_ROM_QSTR(MP_QSTR_statvfs), MP_ROM_PTR(&vfs_rom_statvfs_obj) },
+};
+static MP_DEFINE_CONST_DICT(vfs_rom_locals_dict, vfs_rom_locals_dict_table);
+
+static mp_import_stat_t mp_vfs_rom_import_stat(void *self_in, const char *path) {
+    mp_obj_vfs_rom_t *self = MP_OBJ_TO_PTR(self_in);
+    return mp_vfs_rom_search_filesystem(self, path, NULL, NULL);
+}
+
+static const mp_vfs_proto_t vfs_rom_proto = {
+    .import_stat = mp_vfs_rom_import_stat,
+};
+
+MP_DEFINE_CONST_OBJ_TYPE(
+    mp_type_vfs_rom,
+    MP_QSTR_VfsRom,
+    MP_TYPE_FLAG_NONE,
+    make_new, vfs_rom_make_new,
+    protocol, &vfs_rom_proto,
+    locals_dict, &vfs_rom_locals_dict
+    );
+
+#endif // MICROPY_VFS_ROM

extmod/vfs_rom.h

@@ -0,0 +1,47 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2022 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef MICROPY_INCLUDED_EXTMOD_VFS_ROM_H
+#define MICROPY_INCLUDED_EXTMOD_VFS_ROM_H
+
+#include "py/builtin.h"
+#include "py/obj.h"
+
+#if MICROPY_VFS_ROM
+
+typedef struct _mp_obj_vfs_rom_t mp_obj_vfs_rom_t;
+
+extern const mp_obj_type_t mp_type_vfs_rom;
+
+static inline const char *mp_vfs_rom_get_path_str(mp_obj_vfs_rom_t *self, mp_obj_t path) {
+    return mp_obj_str_get_str(path);
+}
+
+mp_import_stat_t mp_vfs_rom_search_filesystem(mp_obj_vfs_rom_t *self, const char *path, size_t *size_out, const uint8_t **data_out);
+mp_obj_t mp_vfs_rom_file_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_in);
+
+#endif // MICROPY_VFS_ROM
+
+#endif // MICROPY_INCLUDED_EXTMOD_VFS_ROM_H

extmod/vfs_rom_file.c

@@ -0,0 +1,180 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2022 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <string.h>
+
+#include "py/reader.h"
+#include "py/runtime.h"
+#include "py/stream.h"
+#include "extmod/vfs_rom.h"
+
+#if MICROPY_VFS_ROM
+
+typedef struct _mp_obj_vfs_rom_file_t {
+    mp_obj_base_t base;
+    size_t file_size;
+    size_t file_offset;
+    const uint8_t *file_data;
+} mp_obj_vfs_rom_file_t;
+
+static const mp_obj_type_t mp_type_vfs_rom_fileio;
+static const mp_obj_type_t mp_type_vfs_rom_textio;
+
+mp_obj_t mp_vfs_rom_file_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_in) {
+    mp_obj_vfs_rom_t *self = MP_OBJ_TO_PTR(self_in);
+
+    const char *mode_s = mp_obj_str_get_str(mode_in);
+    const mp_obj_type_t *type = &mp_type_vfs_rom_textio;
+    while (*mode_s) {
+        switch (*mode_s++) {
+            case 'r':
+                break;
+            case 'w':
+            case 'a':
+            case '+':
+                mp_raise_OSError(MP_EROFS);
+            case 'b':
+                type = &mp_type_vfs_rom_fileio;
+                break;
+            case 't':
+                type = &mp_type_vfs_rom_textio;
+                break;
+        }
+    }
+
+    mp_obj_vfs_rom_file_t *o = m_new_obj(mp_obj_vfs_rom_file_t);
+    o->base.type = type;
+    o->file_offset = 0;
+
+    const char *path = mp_vfs_rom_get_path_str(self, path_in);
+    mp_import_stat_t stat = mp_vfs_rom_search_filesystem(self, path, &o->file_size, &o->file_data);
+    if (stat == MP_IMPORT_STAT_NO_EXIST) {
+        mp_raise_OSError(MP_ENOENT);
+    } else if (stat == MP_IMPORT_STAT_DIR) {
+        mp_raise_OSError(MP_EISDIR);
+    }
+
+    return MP_OBJ_FROM_PTR(o);
+}
+
+static mp_int_t vfs_rom_file_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, mp_uint_t flags) {
+    mp_obj_vfs_rom_file_t *self = MP_OBJ_TO_PTR(self_in);
+    if (flags == MP_BUFFER_READ) {
+        bufinfo->buf = (void *)self->file_data;
+        bufinfo->len = self->file_size;
+        bufinfo->typecode = 'B';
+        return 0;
+    } else {
+        // Can't write to a ROM file.
+        return 1;
+    }
+}
+
+static mp_uint_t vfs_rom_file_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) {
+    mp_obj_vfs_rom_file_t *self = MP_OBJ_TO_PTR(o_in);
+    size_t remain = self->file_size - self->file_offset;
+    if (size > remain) {
+        size = remain;
+    }
+    memcpy(buf, self->file_data + self->file_offset, size);
+    self->file_offset += size;
+    return size;
+}
+
+static mp_uint_t vfs_rom_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, int *errcode) {
+    mp_obj_vfs_rom_file_t *self = MP_OBJ_TO_PTR(o_in);
+
+    switch (request) {
+        case MP_STREAM_SEEK: {
+            struct mp_stream_seek_t *s = (struct mp_stream_seek_t *)arg;
+            if (s->whence == 0) { // SEEK_SET
+                self->file_offset = (size_t)s->offset;
+            } else if (s->whence == 1) { // SEEK_CUR
+                self->file_offset += s->offset;
+            } else { // SEEK_END
+                self->file_offset = self->file_size + s->offset;
+            }
+            if (self->file_offset > self->file_size) {
+                if (s->offset < 0) {
+                    // Seek to before the start of the file.
+                    *errcode = MP_EINVAL;
+                    return MP_STREAM_ERROR;
+                }
+                self->file_offset = self->file_size;
+            }
+            s->offset = self->file_offset;
+            return 0;
+        }
+        case MP_STREAM_CLOSE:
+            return 0;
+        default:
+            *errcode = MP_EINVAL;
+            return MP_STREAM_ERROR;
+    }
+}
+
+static const mp_rom_map_elem_t vfs_rom_rawfile_locals_dict_table[] = {
+    { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) },
+    { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) },
+    { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) },
+    { MP_ROM_QSTR(MP_QSTR_readlines), MP_ROM_PTR(&mp_stream_unbuffered_readlines_obj) },
+    { MP_ROM_QSTR(MP_QSTR_seek), MP_ROM_PTR(&mp_stream_seek_obj) },
+    { MP_ROM_QSTR(MP_QSTR_tell), MP_ROM_PTR(&mp_stream_tell_obj) },
+    { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) },
+    { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) },
+    { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&mp_stream___exit___obj) },
+};
+static MP_DEFINE_CONST_DICT(vfs_rom_rawfile_locals_dict, vfs_rom_rawfile_locals_dict_table);
+
+static const mp_stream_p_t vfs_rom_fileio_stream_p = {
+    .read = vfs_rom_file_read,
+    .ioctl = vfs_rom_file_ioctl,
+};
+
+static MP_DEFINE_CONST_OBJ_TYPE(
+    mp_type_vfs_rom_fileio,
+    MP_QSTR_FileIO,
+    MP_TYPE_FLAG_ITER_IS_STREAM,
+    buffer, vfs_rom_file_get_buffer,
+    protocol, &vfs_rom_fileio_stream_p,
+    locals_dict, &vfs_rom_rawfile_locals_dict
+    );
+
+static const mp_stream_p_t vfs_rom_textio_stream_p = {
+    .read = vfs_rom_file_read,
+    .ioctl = vfs_rom_file_ioctl,
+    .is_text = true,
+};
+
+static MP_DEFINE_CONST_OBJ_TYPE(
+    mp_type_vfs_rom_textio,
+    MP_QSTR_TextIOWrapper,
+    MP_TYPE_FLAG_ITER_IS_STREAM,
+    protocol, &vfs_rom_textio_stream_p,
+    locals_dict, &vfs_rom_rawfile_locals_dict
+    );
+
+#endif // MICROPY_VFS_ROM

py/mpconfig.h

@@ -1016,6 +1016,11 @@ typedef double mp_float_t;
 #define MICROPY_VFS_LFS2 (0)
 #endif
 
+// Support for ROMFS.
+#ifndef MICROPY_VFS_ROM
+#define MICROPY_VFS_ROM (0)
+#endif
+
 /*****************************************************************************/
 /* Fine control over Python builtins, classes, modules, etc                  */