py/persistentcode: Decouple native code loading from emitters' presence.

This commit lets the interpreter load MPY files containing native code
even if the target platform does not have a native emitter, or if native
code generation is disabled.

Native code loading has been tied to native code generation being
enabled as a discriminant to allow said operation.  This blocks native
code loading on platforms that could benefit from such a thing but they
don't (and probably won't) have a native code generation target written
for them (ie. AArch64 and RISC-V 64).  This also forces a firmware image
to have a full native code compiler present even if it doesn't need to
generate anything, as native modules already have all the code they will
ever need to load.

There is a new configuration setting,
MICROPY_PERSISTENT_CODE_LOAD_NATIVE, that if enabled it will allow
loading native code modules even if code generation
(MICROPY_EMIT_<platform> and MICROPY_EMIT_INLINE_<platform>) is
explicitly turned off.

Signed-off-by: Alessandro Gatti <a.gatti@frob.it>

Author: agatti

py/bc.c

@@ -332,7 +332,7 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw
     mp_setup_code_state_helper(code_state, n_args, n_kw, args);
 }
 
-#if MICROPY_EMIT_NATIVE
+#if MICROPY_ENABLE_NATIVE_CODE
 // On entry code_state should be allocated somewhere (stack/heap) and
 // contain the following valid entries:
 //    - code_state->fun_bc should contain a pointer to the function object

py/emitglue.c

@@ -92,7 +92,7 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code,
     #endif
 }
 
-#if MICROPY_EMIT_MACHINE_CODE
+#if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE
 void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, const void *fun_data, mp_uint_t fun_len,
     mp_raw_code_t **children,
     #if MICROPY_PERSISTENT_CODE_SAVE
@@ -201,7 +201,7 @@ mp_obj_t mp_make_function_from_proto_fun(mp_proto_fun_t proto_fun, const mp_modu
     // make the function, depending on the raw code kind
     mp_obj_t fun;
     switch (rc->kind) {
-        #if MICROPY_EMIT_NATIVE
+        #if MICROPY_ENABLE_NATIVE_CODE
         case MP_CODE_NATIVE_PY:
             fun = mp_obj_new_fun_native(def_args, rc->fun_data, context, rc->children);
             // Check for a generator function, and if so change the type of the object

py/emitglue.h

@@ -80,7 +80,7 @@ typedef struct _mp_raw_code_t {
     #if MICROPY_PERSISTENT_CODE_SAVE
     uint32_t fun_data_len; // for mp_raw_code_save
     uint16_t n_children;
-    #if MICROPY_EMIT_MACHINE_CODE
+    #if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE
     uint16_t prelude_offset;
     #endif
     #if MICROPY_PY_SYS_SETTRACE
@@ -110,7 +110,7 @@ typedef struct _mp_raw_code_truncated_t {
     #if MICROPY_PERSISTENT_CODE_SAVE
     uint32_t fun_data_len;
     uint16_t n_children;
-    #if MICROPY_EMIT_MACHINE_CODE
+    #if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE
     uint16_t prelude_offset;
     #endif
     #if MICROPY_PY_SYS_SETTRACE

py/mpconfig.h

@@ -411,6 +411,11 @@ typedef uint64_t mp_uint_t;
 #define MICROPY_PERSISTENT_CODE_LOAD (0)
 #endif
 
+// Whether to support loading of persistent native code
+#ifndef MICROPY_PERSISTENT_CODE_LOAD_NATIVE
+#define MICROPY_PERSISTENT_CODE_LOAD_NATIVE (MICROPY_EMIT_MACHINE_CODE)
+#endif
+
 // Whether to support saving of persistent code, i.e. for mpy-cross to
 // generate .mpy files. Enabling this enables additional metadata on raw code
 // objects which is also required for sys.settrace.
@@ -431,7 +436,7 @@ typedef uint64_t mp_uint_t;
 // Whether generated code can persist independently of the VM/runtime instance
 // This is enabled automatically when needed by other features
 #ifndef MICROPY_PERSISTENT_CODE
-#define MICROPY_PERSISTENT_CODE (MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE || MICROPY_MODULE_FROZEN_MPY)
+#define MICROPY_PERSISTENT_CODE (MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_LOAD_NATIVE || MICROPY_PERSISTENT_CODE_SAVE || MICROPY_MODULE_FROZEN_MPY)
 #endif
 
 // Whether bytecode uses a qstr_table to map internal qstr indices in the bytecode
@@ -536,6 +541,10 @@ typedef uint64_t mp_uint_t;
 // Convenience definition for whether any native or inline assembler emitter is enabled
 #define MICROPY_EMIT_MACHINE_CODE (MICROPY_EMIT_NATIVE || MICROPY_EMIT_INLINE_ASM)
 
+// Convenience definition for whether native code has to be dealt with (either
+// generated or loaded from a file).  This does not cover inline asm code.
+#define MICROPY_ENABLE_NATIVE_CODE (MICROPY_EMIT_NATIVE || MICROPY_PERSISTENT_CODE_LOAD_NATIVE)
+
 /*****************************************************************************/
 /* Compiler configuration                                                    */
 
@@ -2276,7 +2285,7 @@ typedef time_t mp_timestamp_t;
 // can be overridden if needed by defining both MICROPY_PERSISTENT_CODE_TRACK_FUN_DATA
 // and MICROPY_PERSISTENT_CODE_TRACK_BSS_RODATA.
 #ifndef MICROPY_PERSISTENT_CODE_TRACK_FUN_DATA
-#if MICROPY_EMIT_MACHINE_CODE && MICROPY_PERSISTENT_CODE_LOAD
+#if (MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE) && MICROPY_PERSISTENT_CODE_LOAD
 // Pointer tracking is required when loading native code is enabled.
 #if defined(MP_PLAT_ALLOC_EXEC) || defined(MP_PLAT_COMMIT_EXEC)
 // If a port defined a custom allocator or commit function for native text, then the
@@ -2297,7 +2306,7 @@ typedef time_t mp_timestamp_t;
 #define MICROPY_PERSISTENT_CODE_TRACK_FUN_DATA (1)
 #define MICROPY_PERSISTENT_CODE_TRACK_BSS_RODATA (0)
 #endif
-#else // MICROPY_EMIT_MACHINE_CODE && MICROPY_PERSISTENT_CODE_LOAD
+#else // (MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE) && MICROPY_PERSISTENT_CODE_LOAD
 // Pointer tracking not needed when loading native code is disabled.
 #define MICROPY_PERSISTENT_CODE_TRACK_FUN_DATA (0)
 #define MICROPY_PERSISTENT_CODE_TRACK_BSS_RODATA (0)

py/nativeglue.c

@@ -41,7 +41,7 @@
 #define DEBUG_printf(...) (void)0
 #endif
 
-#if MICROPY_EMIT_NATIVE
+#if MICROPY_ENABLE_NATIVE_CODE
 
 int mp_native_type_from_qstr(qstr qst) {
     switch (qst) {
@@ -91,7 +91,7 @@ mp_uint_t mp_native_from_obj(mp_obj_t obj, mp_uint_t type) {
 
 #endif
 
-#if MICROPY_EMIT_MACHINE_CODE
+#if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE
 
 // convert a native value to a MicroPython object based on type
 mp_obj_t mp_native_to_obj(mp_uint_t val, mp_uint_t type) {
@@ -115,7 +115,7 @@ mp_obj_t mp_native_to_obj(mp_uint_t val, mp_uint_t type) {
 
 #endif
 
-#if MICROPY_EMIT_NATIVE && !MICROPY_DYNAMIC_COMPILER
+#if MICROPY_ENABLE_NATIVE_CODE && !MICROPY_DYNAMIC_COMPILER
 
 #if !MICROPY_PY_BUILTINS_SET
 mp_obj_t mp_obj_new_set(size_t n_args, mp_obj_t *items) {
@@ -354,8 +354,8 @@ const mp_fun_table_t mp_fun_table = {
     &mp_stream_write_obj,
 };
 
-#elif MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER
+#elif MICROPY_ENABLE_NATIVE_CODE && MICROPY_DYNAMIC_COMPILER
 
 const int mp_fun_table;
 
-#endif // MICROPY_EMIT_NATIVE
+#endif // MICROPY_ENABLE_NATIVE_CODE

py/nativeglue.h

@@ -181,9 +181,9 @@ typedef struct _mp_fun_table_t {
     const mp_obj_fun_builtin_var_t *stream_write_obj;
 } mp_fun_table_t;
 
-#if (MICROPY_EMIT_NATIVE && !MICROPY_DYNAMIC_COMPILER) || MICROPY_ENABLE_DYNRUNTIME
+#if (MICROPY_ENABLE_NATIVE_CODE && !MICROPY_DYNAMIC_COMPILER) || MICROPY_ENABLE_DYNRUNTIME
 extern const mp_fun_table_t mp_fun_table;
-#elif MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER
+#elif MICROPY_ENABLE_NATIVE_CODE && MICROPY_DYNAMIC_COMPILER
 // In dynamic-compiler mode eliminate dependency on entries in mp_fun_table.
 // This only needs to be an independent pointer, content doesn't matter.
 extern const int mp_fun_table;

py/objfun.c

@@ -136,7 +136,7 @@ qstr mp_obj_fun_get_name(mp_const_obj_t fun_in) {
     const mp_obj_fun_bc_t *fun = MP_OBJ_TO_PTR(fun_in);
     const byte *bc = fun->bytecode;
 
-    #if MICROPY_EMIT_NATIVE
+    #if MICROPY_ENABLE_NATIVE_CODE
     if (fun->base.type == &mp_type_fun_native || fun->base.type == &mp_type_native_gen_wrap) {
         bc = mp_obj_fun_native_get_prelude_ptr(fun);
     }
@@ -443,7 +443,7 @@ mp_obj_t mp_obj_new_fun_bc(const mp_obj_t *def_args, const byte *code, const mp_
 /******************************************************************************/
 /* native functions                                                           */
 
-#if MICROPY_EMIT_NATIVE
+#if MICROPY_ENABLE_NATIVE_CODE
 
 static mp_obj_t fun_native_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) {
     mp_cstack_check();
@@ -472,13 +472,9 @@ MP_DEFINE_CONST_OBJ_TYPE(
     call, fun_native_call
     );
 
-#endif // MICROPY_EMIT_NATIVE
-
 /******************************************************************************/
 /* viper functions                                                           */
 
-#if MICROPY_EMIT_NATIVE
-
 static mp_obj_t fun_viper_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) {
     mp_cstack_check();
     mp_obj_fun_bc_t *self = MP_OBJ_TO_PTR(self_in);
@@ -493,7 +489,7 @@ MP_DEFINE_CONST_OBJ_TYPE(
     call, fun_viper_call
     );
 
-#endif // MICROPY_EMIT_NATIVE
+#endif // MICROPY_ENABLE_NATIVE_CODE
 
 /******************************************************************************/
 /* inline assembler functions                                                 */

py/objfun.h

@@ -53,7 +53,7 @@ typedef struct _mp_obj_fun_asm_t {
 mp_obj_t mp_obj_new_fun_bc(const mp_obj_t *def_args, const byte *code, const mp_module_context_t *cm, struct _mp_raw_code_t *const *raw_code_table);
 void mp_obj_fun_bc_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest);
 
-#if MICROPY_EMIT_NATIVE
+#if MICROPY_ENABLE_NATIVE_CODE
 
 static inline mp_obj_t mp_obj_new_fun_native(const mp_obj_t *def_args, const void *fun_data, const mp_module_context_t *mc, struct _mp_raw_code_t *const *child_table) {
     mp_obj_fun_bc_t *o = (mp_obj_fun_bc_t *)MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args, (const byte *)fun_data, mc, child_table));

py/objgenerator.c

@@ -87,7 +87,7 @@ MP_DEFINE_CONST_OBJ_TYPE(
 /******************************************************************************/
 // native generator wrapper
 
-#if MICROPY_EMIT_NATIVE
+#if MICROPY_ENABLE_NATIVE_CODE
 
 // Based on mp_obj_gen_instance_t.
 typedef struct _mp_obj_gen_instance_native_t {
@@ -139,7 +139,7 @@ MP_DEFINE_CONST_OBJ_TYPE(
     NATIVE_GEN_WRAP_TYPE_ATTR
     );
 
-#endif // MICROPY_EMIT_NATIVE
+#endif // MICROPY_ENABLE_NATIVE_CODE
 
 /******************************************************************************/
 /* generator instance                                                         */
@@ -175,7 +175,7 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_
 
     // If the generator is started, allow sending a value.
     void *state_start = self->code_state.state - 1;
-    #if MICROPY_EMIT_NATIVE
+    #if MICROPY_ENABLE_NATIVE_CODE
     if (self->code_state.exc_sp_idx == MP_CODE_STATE_EXC_SP_IDX_SENTINEL) {
         state_start = ((mp_obj_gen_instance_native_t *)self)->code_state.state - 1;
     }
@@ -197,7 +197,7 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_
 
     mp_vm_return_kind_t ret_kind;
 
-    #if MICROPY_EMIT_NATIVE
+    #if MICROPY_ENABLE_NATIVE_CODE
     if (self->code_state.exc_sp_idx == MP_CODE_STATE_EXC_SP_IDX_SENTINEL) {
         // A native generator.
         typedef uintptr_t (*mp_fun_native_gen_t)(void *, mp_obj_t);
@@ -235,7 +235,7 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_
 
         case MP_VM_RETURN_EXCEPTION: {
             self->code_state.ip = 0;
-            #if MICROPY_EMIT_NATIVE
+            #if MICROPY_ENABLE_NATIVE_CODE
             if (self->code_state.exc_sp_idx == MP_CODE_STATE_EXC_SP_IDX_SENTINEL) {
                 *ret_val = ((mp_obj_gen_instance_native_t *)self)->code_state.state[0];
             } else

py/persistentcode.c

@@ -76,7 +76,7 @@ typedef struct _bytecode_prelude_t {
 static int read_byte(mp_reader_t *reader);
 static size_t read_uint(mp_reader_t *reader);
 
-#if MICROPY_EMIT_MACHINE_CODE
+#if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE
 
 #if MICROPY_PERSISTENT_CODE_TRACK_FUN_DATA || MICROPY_PERSISTENT_CODE_TRACK_BSS_RODATA
 
@@ -222,7 +222,7 @@ static mp_obj_t mp_obj_new_str_static(const mp_obj_type_t *type, const byte *dat
 
 static mp_obj_t load_obj(mp_reader_t *reader) {
     byte obj_type = read_byte(reader);
-    #if MICROPY_EMIT_MACHINE_CODE
+    #if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE
     if (obj_type == MP_PERSISTENT_OBJ_FUN_TABLE) {
         return MP_OBJ_FROM_PTR(&mp_fun_table);
     } else
@@ -295,14 +295,14 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co
     bool has_children = !!(kind_len & 4);
     size_t fun_data_len = kind_len >> 3;
 
-    #if !MICROPY_EMIT_MACHINE_CODE
+    #if !(MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE)
     if (kind != MP_CODE_BYTECODE) {
         mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy file"));
     }
     #endif
 
     uint8_t *fun_data = NULL;
-    #if MICROPY_EMIT_MACHINE_CODE
+    #if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE
     size_t prelude_offset = 0;
     mp_uint_t native_scope_flags = 0;
     mp_uint_t native_n_pos_args = 0;
@@ -322,7 +322,7 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co
             read_bytes(reader, fun_data, fun_data_len);
         }
 
-    #if MICROPY_EMIT_MACHINE_CODE
+    #if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE
     } else {
         // Allocate memory for native data and load it
         size_t fun_alloc;
@@ -349,7 +349,7 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co
     size_t n_children = 0;
     mp_raw_code_t **children = NULL;
 
-    #if MICROPY_EMIT_MACHINE_CODE
+    #if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE
     // Load optional BSS/rodata for viper.
     uint8_t *rodata = NULL;
     uint8_t *bss = NULL;
@@ -404,7 +404,7 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co
             #endif
             scope_flags);
 
-    #if MICROPY_EMIT_MACHINE_CODE
+    #if MICROPY_EMIT_INLINE_ASM || MICROPY_ENABLE_NATIVE_CODE
     } else {
         const uint8_t *prelude_ptr = NULL;
         #if MICROPY_EMIT_NATIVE_PRELUDE_SEPARATE_FROM_MACHINE_CODE