From ec264ec033db5290f9cca0c77a2ad2f56b09315e Mon Sep 17 00:00:00 2001 From: Dmitrii Kuvaiskii Date: Tue, 15 Oct 2024 04:45:01 -0700 Subject: [PATCH] [PAL/Linux-SGX] Add AEX-Notify enabling code This is a preparatory commit for AEX-Notify support. This commit: - Introduces the `sgx.experimental_enable_aex_notify` manifest option. - Adds architectural flags/bits for SECS, TCS, SSA data structures. - Adds a Gramine startup check whether AEX-Notify hardware feature is supported by the platform. - Adds dynamic enablement/disablement of AEX-Notify feature per enclave thread (enable on thread creation, disable on thread termination). Currently per-thread enablement is commented out, as otherwise Gramine would segfault (as in-enclave code doesn't yet implement AEX-Notify). Signed-off-by: Dmitrii Kuvaiskii --- Documentation/manifest-syntax.rst | 24 +++++++++++++++ pal/src/host/linux-sgx/generated_offsets.c | 4 +++ pal/src/host/linux-sgx/host_framework.c | 18 +++++++++++ pal/src/host/linux-sgx/host_internal.h | 1 + pal/src/host/linux-sgx/host_main.c | 18 +++++++++++ pal/src/host/linux-sgx/pal_exception.c | 35 ++++++++++++++++++++++ pal/src/host/linux-sgx/pal_linux.h | 4 +++ pal/src/host/linux-sgx/pal_main.c | 9 ++++++ pal/src/host/linux-sgx/pal_tcb.h | 1 + pal/src/host/linux-sgx/pal_threading.c | 5 ++++ pal/src/host/linux-sgx/sgx_arch.h | 9 ++++-- python/graminelibos/manifest.py | 1 + python/graminelibos/manifest_check.py | 1 + python/graminelibos/sgx_sign.py | 6 ++++ 14 files changed, 134 insertions(+), 2 deletions(-) diff --git a/Documentation/manifest-syntax.rst b/Documentation/manifest-syntax.rst index c531139ff0..fd40ffbcf4 100644 --- a/Documentation/manifest-syntax.rst +++ b/Documentation/manifest-syntax.rst @@ -1392,6 +1392,30 @@ In addition, the application manifest must also contain ``sgx.debug = true``. See :ref:`vtune-sgx-profiling` for more information. +Enabling AEX-Notify +^^^^^^^^^^^^^^^^^^^ + +:: + + sgx.experimental_enable_aex_notify = [true|false] + (Default: false) + +When enabled, this option instructs Gramine to use the AEX-Notify hardware feature. +AEX-Notify is a flexible hardware extension that makes SGX enclaves interrupt +aware: enclaves can register a trusted handler to be run after an interrupt or +exception. AEX-Notify can be used as a building block for implementing +countermeasures against different types of interrupt-based attacks in software. + +For more information on AEX-Notify, refer to the `academic paper +`__ +and to the `official whitepaper +`__. + +.. warning:: + Support for AEX-Notify in Gramine is not yet thoroughly tested and may + contain security vulnerabilities. This is temporary; the prefix + ``experimental_`` will be removed in the future after thorough validation. + Deprecated options ------------------ diff --git a/pal/src/host/linux-sgx/generated_offsets.c b/pal/src/host/linux-sgx/generated_offsets.c index 08e4ed9728..8ff89d06ec 100644 --- a/pal/src/host/linux-sgx/generated_offsets.c +++ b/pal/src/host/linux-sgx/generated_offsets.c @@ -15,6 +15,7 @@ const struct generated_offset generated_offsets[] = { /* defines from sgx_arch.h */ DEFINE(SGX_FLAGS_DEBUG, SGX_FLAGS_DEBUG), DEFINE(SGX_FLAGS_MODE64BIT, SGX_FLAGS_MODE64BIT), + DEFINE(SGX_FLAGS_AEXNOTIFY, SGX_FLAGS_AEXNOTIFY), DEFINE(SGX_XFRM_LEGACY, SGX_XFRM_LEGACY), DEFINE(SGX_XFRM_AVX, SGX_XFRM_AVX), DEFINE(SGX_XFRM_MPX, SGX_XFRM_MPX), @@ -54,6 +55,7 @@ const struct generated_offset generated_offsets[] = { OFFSET_T(SGX_GPR_RFLAGS, sgx_pal_gpr_t, rflags), OFFSET_T(SGX_GPR_RIP, sgx_pal_gpr_t, rip), OFFSET_T(SGX_GPR_EXITINFO, sgx_pal_gpr_t, exitinfo), + OFFSET_T(SGX_GPR_AEXNOTIFY, sgx_pal_gpr_t, aexnotify), DEFINE(SGX_GPR_SIZE, sizeof(sgx_pal_gpr_t)), /* sgx_cpu_context_t */ @@ -103,6 +105,7 @@ const struct generated_offset generated_offsets[] = { OFFSET(SGX_HEAP_MIN, pal_enclave_tcb, heap_min), OFFSET(SGX_HEAP_MAX, pal_enclave_tcb, heap_max), OFFSET(SGX_CLEAR_CHILD_TID, pal_enclave_tcb, clear_child_tid), + OFFSET(SGX_READY_FOR_AEX_NOTIFY, pal_enclave_tcb, ready_for_aex_notify), /* struct pal_host_tcb aka PAL_HOST_TCB */ OFFSET(PAL_HOST_TCB_TCS, pal_host_tcb, tcs), @@ -123,6 +126,7 @@ const struct generated_offset generated_offsets[] = { OFFSET_T(TCS_OFS_LIMIT, sgx_arch_tcs_t, ofs_limit), OFFSET_T(TCS_OGS_LIMIT, sgx_arch_tcs_t, ogs_limit), DEFINE(TCS_SIZE, sizeof(sgx_arch_tcs_t)), + DEFINE(TCS_FLAGS_AEXNOTIFY, TCS_FLAGS_AEXNOTIFY), /* sgx_attributes_t */ OFFSET_T(SGX_ATTRIBUTES_XFRM, sgx_attributes_t, xfrm), diff --git a/pal/src/host/linux-sgx/host_framework.c b/pal/src/host/linux-sgx/host_framework.c index c77f9bfb94..2e4e020793 100644 --- a/pal/src/host/linux-sgx/host_framework.c +++ b/pal/src/host/linux-sgx/host_framework.c @@ -175,6 +175,24 @@ bool is_wrfsbase_supported(void) { return true; } +bool is_aexnotify_supported(void) { + uint32_t cpuinfo[4]; + + cpuid(INTEL_SGX_LEAF, 1, cpuinfo); + if (!((cpuinfo[CPUID_WORD_EAX] >> 10) & 0x1)) { + log_error("AEX-Notify hardware feature is not supported."); + return false; + } + + cpuid(INTEL_SGX_LEAF, 0, cpuinfo); + if (!((cpuinfo[CPUID_WORD_EAX] >> 11) & 0x1)) { + log_error("ENCLU[EDECCSSA] leaf instruction is not supported."); + return false; + } + + return true; +} + int create_enclave(sgx_arch_secs_t* secs, sgx_arch_token_t* token) { assert(secs->size && IS_POWER_OF_2(secs->size)); assert(IS_ALIGNED(secs->base, secs->size)); diff --git a/pal/src/host/linux-sgx/host_internal.h b/pal/src/host/linux-sgx/host_internal.h index c6503725f6..775f029149 100644 --- a/pal/src/host/linux-sgx/host_internal.h +++ b/pal/src/host/linux-sgx/host_internal.h @@ -66,6 +66,7 @@ void* realloc(void* ptr, size_t new_size); int open_sgx_driver(void); bool is_wrfsbase_supported(void); +bool is_aexnotify_supported(void); int read_enclave_token(int token_file, sgx_arch_token_t* out_token); int create_dummy_enclave_token(sgx_sigstruct_t* sig, sgx_arch_token_t* out_token); diff --git a/pal/src/host/linux-sgx/host_main.c b/pal/src/host/linux-sgx/host_main.c index 33b29878c4..9dcc80dbd2 100644 --- a/pal/src/host/linux-sgx/host_main.c +++ b/pal/src/host/linux-sgx/host_main.c @@ -552,7 +552,10 @@ static int initialize_enclave(struct pal_enclave* enclave, const char* manifest_ tcs->ogs_base = tls_area->addr - enclave->baseaddr + t * g_page_size; tcs->ofs_limit = 0xfff; tcs->ogs_limit = 0xfff; + tcs->flags |= (enclave_token.body.attributes.flags & SGX_FLAGS_AEXNOTIFY) + ? TCS_FLAGS_AEXNOTIFY : 0; tcs_addrs[t] = (void*)tcs_area->addr + g_page_size * t; + } } else if (areas[i].data_src == BUF) { memcpy(data, areas[i].buf, areas[i].buf_size); @@ -690,6 +693,21 @@ static int parse_loader_config(char* manifest, struct pal_enclave* enclave_info, goto out; } + bool aex_notify_enabled; + ret = toml_bool_in(manifest_root, "sgx.experimental_enable_aex_notify", + /*defaultval=*/false, &aex_notify_enabled); + if (ret < 0) { + log_error("Cannot parse 'sgx.experimental_enable_aex_notify'"); + ret = -EINVAL; + goto out; + } + + if (aex_notify_enabled && !is_aexnotify_supported()) { + log_error("Cannot enable AEX-Notify on this platform (hardware doesn't support it)"); + ret = -EPERM; + goto out; + } + int64_t thread_num_int64; ret = toml_int_in(manifest_root, "sgx.max_threads", /*defaultval=*/-1, &thread_num_int64); if (ret < 0) { diff --git a/pal/src/host/linux-sgx/pal_exception.c b/pal/src/host/linux-sgx/pal_exception.c index 845b67526f..79edcb3046 100644 --- a/pal/src/host/linux-sgx/pal_exception.c +++ b/pal/src/host/linux-sgx/pal_exception.c @@ -22,6 +22,41 @@ #define ADDR_IN_PAL(addr) ((void*)(addr) > TEXT_START && (void*)(addr) < TEXT_END) +bool g_aex_notify_enabled = false; + +void init_aex_notify_for_thread(void) { + if (!g_aex_notify_enabled) + return; + + SET_ENCLAVE_TCB(ready_for_aex_notify, 1UL); + MB(); +#if 0 + /* + * FIXME: Re-enable in the following commit, when all AEX-Notify flows are added. + * Currently this would fail, as the untrusted runtime expects AEX-Notify flows but + * in-enclave runtime doesn't yet implement AEX-Notify flows. + */ + /* + * Note that AEX-Notify is enabled only for SSA[0] (regular context), and is always disabled + * for SSA[1] (stage-1 signal handling context). The disablement of AEX-Notify for SSA[1] is + * implicit (the AEX-Notify bit in the SSA[1]'s GPR region is by default zero). This disablement + * is important for stage-1 signal handling flows, see enclave_entry.S. + */ + GET_ENCLAVE_TCB(gpr)->aexnotify = 1U; + MB(); +#endif +} + +void fini_aex_notify_for_thread(void) { + if (!g_aex_notify_enabled) + return; + + SET_ENCLAVE_TCB(ready_for_aex_notify, 0UL); + MB(); + GET_ENCLAVE_TCB(gpr)->aexnotify = 0U; + MB(); +} + /* Restore an sgx_cpu_context_t as generated by .Lhandle_exception. Execution will * continue as specified by the rip in the context. */ __attribute_no_sanitize_address diff --git a/pal/src/host/linux-sgx/pal_linux.h b/pal/src/host/linux-sgx/pal_linux.h index f22e892993..97060e7bc3 100644 --- a/pal/src/host/linux-sgx/pal_linux.h +++ b/pal/src/host/linux-sgx/pal_linux.h @@ -95,6 +95,10 @@ void save_xregs(PAL_XREGS_STATE* xsave_area); void restore_xregs(const PAL_XREGS_STATE* xsave_area); noreturn void _restore_sgx_context(sgx_cpu_context_t* uc, PAL_XREGS_STATE* xsave_area); +extern bool g_aex_notify_enabled; +void init_aex_notify_for_thread(void); +void fini_aex_notify_for_thread(void); + void _PalExceptionHandler(uint32_t trusted_exit_info_, uint32_t untrusted_external_event, sgx_cpu_context_t* uc, PAL_XREGS_STATE* xregs_state, sgx_arch_exinfo_t* exinfo); diff --git a/pal/src/host/linux-sgx/pal_main.c b/pal/src/host/linux-sgx/pal_main.c index 77e5a11a34..ab9030cfe2 100644 --- a/pal/src/host/linux-sgx/pal_main.c +++ b/pal/src/host/linux-sgx/pal_main.c @@ -722,6 +722,13 @@ noreturn void pal_linux_main(void* uptr_libpal_uri, size_t libpal_uri_len, void* ocall_exit(1, /*is_exitgroup=*/true); } + ret = toml_bool_in(g_pal_public_state.manifest_root, "sgx.experimental_enable_aex_notify", + /*defaultval=*/false, &g_aex_notify_enabled); + if (ret < 0) { + log_error("Cannot parse 'sgx.experimental_enable_aex_notify'"); + ocall_exit(1, /*is_exitgroup=*/true); + } + /* Get host information about domain name configuration only for the first process. * This information will be checkpointed and restored during forking of the child * process(es). */ @@ -764,6 +771,8 @@ noreturn void pal_linux_main(void* uptr_libpal_uri, size_t libpal_uri_len, void* assert(!g_pal_linuxsgx_state.enclave_initialized); g_pal_linuxsgx_state.enclave_initialized = true; + init_aex_notify_for_thread(); + /* call main function */ pal_main(instance_id, parent, first_thread, arguments, environments, post_callback); } diff --git a/pal/src/host/linux-sgx/pal_tcb.h b/pal/src/host/linux-sgx/pal_tcb.h index d6c724d0c9..fa6886c029 100644 --- a/pal/src/host/linux-sgx/pal_tcb.h +++ b/pal/src/host/linux-sgx/pal_tcb.h @@ -42,6 +42,7 @@ struct pal_enclave_tcb { uint64_t ocall_exit_called; uint64_t thread_started; uint64_t ready_for_exceptions; + uint64_t ready_for_aex_notify; uint64_t manifest_size; void* heap_min; void* heap_max; diff --git a/pal/src/host/linux-sgx/pal_threading.c b/pal/src/host/linux-sgx/pal_threading.c index c28ffb2bbe..b2c28f76c9 100644 --- a/pal/src/host/linux-sgx/pal_threading.c +++ b/pal/src/host/linux-sgx/pal_threading.c @@ -90,6 +90,7 @@ static void init_dynamic_thread(void* addr) { tcs->ogs_base = (uint64_t)tcb - g_enclave_base; tcs->ofs_limit = 0xfff; tcs->ogs_limit = 0xfff; + tcs->flags |= g_aex_notify_enabled ? TCS_FLAGS_AEXNOTIFY : 0; } static int create_dynamic_tcs_if_none_available(void** out_tcs) { @@ -160,6 +161,8 @@ void pal_start_thread(void) { pal_set_tcb_stack_canary(stack_protector_canary); PAL_TCB* pal_tcb = pal_get_tcb(); memset(&pal_tcb->libos_tcb, 0, sizeof(pal_tcb->libos_tcb)); + init_aex_notify_for_thread(); + callback((void*)param); _PalThreadExit(/*clear_child_tid=*/NULL); /* UNREACHABLE */ @@ -227,6 +230,8 @@ void _PalThreadYieldExecution(void) { noreturn void _PalThreadExit(int* clear_child_tid) { struct pal_handle_thread* exiting_thread = GET_ENCLAVE_TCB(thread); + fini_aex_notify_for_thread(); + /* thread is ready to exit, must inform LibOS by erasing clear_child_tid; * note that we don't do it now (because this thread still occupies SGX * TCS slot) but during handle_thread_reset in assembly code */ diff --git a/pal/src/host/linux-sgx/sgx_arch.h b/pal/src/host/linux-sgx/sgx_arch.h index 1ac65c8eea..029b03d6e2 100644 --- a/pal/src/host/linux-sgx/sgx_arch.h +++ b/pal/src/host/linux-sgx/sgx_arch.h @@ -66,6 +66,7 @@ typedef uint8_t sgx_isvfamily_id_t[SGX_ISV_FAMILY_ID_SIZE]; #define SGX_FLAGS_MODE64BIT 0x04ULL #define SGX_FLAGS_PROVISION_KEY 0x10ULL #define SGX_FLAGS_LICENSE_KEY 0x20ULL +#define SGX_FLAGS_AEXNOTIFY 0x400ULL /* EINIT must verify *all* SECS.ATTRIBUTES[63..0] bits (FLAGS bits) against * SIGSTRUCT.ATTRIBUTES[63..0]. @@ -168,7 +169,8 @@ typedef struct { } sgx_arch_tcs_t; static_assert(sizeof(sgx_arch_tcs_t) == 4096, "incorrect struct size"); -#define TCS_FLAGS_DBGOPTIN (01ULL) +#define TCS_FLAGS_DBGOPTIN (01ULL) +#define TCS_FLAGS_AEXNOTIFY (02ULL) typedef struct { uint64_t rax; @@ -192,10 +194,12 @@ typedef struct { uint64_t ursp; uint64_t urbp; uint32_t exitinfo; - uint32_t reserved; + uint8_t reserved[3]; + uint8_t aexnotify; uint64_t fsbase; uint64_t gsbase; } sgx_pal_gpr_t; +static_assert(offsetof(sgx_pal_gpr_t, aexnotify) == 167, "Wrong offset of AEX-Notify bit in SSA"); typedef struct { uint64_t rax; @@ -443,6 +447,7 @@ static inline int enclu(uint32_t eax, uint64_t rbx, uint64_t rcx, uint64_t rdx) #define EACCEPT 5 #define EMODPE 6 #define EACCEPTCOPY 7 +#define EDECCSSA 9 #define SGX_LAUNCH_KEY 0 #define SGX_PROVISION_KEY 1 diff --git a/python/graminelibos/manifest.py b/python/graminelibos/manifest.py index 37829a8a2e..db41c5c2c1 100644 --- a/python/graminelibos/manifest.py +++ b/python/graminelibos/manifest.py @@ -309,6 +309,7 @@ def __init__(self, manifest_str): sgx.setdefault('debug', False) sgx.setdefault('enable_stats', False) sgx.setdefault('edmm_enable', False) + sgx.setdefault('experimental_enable_aex_notify', False) if sgx['edmm_enable']: sgx.setdefault('enclave_size', DEFAULT_ENCLAVE_SIZE_WITH_EDMM) diff --git a/python/graminelibos/manifest_check.py b/python/graminelibos/manifest_check.py index bfba896527..faff6fc855 100644 --- a/python/graminelibos/manifest_check.py +++ b/python/graminelibos/manifest_check.py @@ -81,6 +81,7 @@ }, 'debug': bool, 'edmm_enable': bool, + 'experimental_enable_aex_notify': bool, 'enable_stats': bool, 'enclave_size': _size, 'file_check_policy': Any('strict', 'allow_all_but_log'), diff --git a/python/graminelibos/sgx_sign.py b/python/graminelibos/sgx_sign.py index 17e2f6088e..cd5e49b318 100644 --- a/python/graminelibos/sgx_sign.py +++ b/python/graminelibos/sgx_sign.py @@ -100,6 +100,7 @@ def collect_cpu_feature_bits(manifest_cpu_features, options_dict, val, mask, sec def get_enclave_attributes(manifest_sgx): flags_dict = { 'debug': offs.SGX_FLAGS_DEBUG, + 'experimental_enable_aex_notify': offs.SGX_FLAGS_AEXNOTIFY, } flags = collect_bits(manifest_sgx, flags_dict) if ARCHITECTURE == 'amd64': @@ -271,6 +272,9 @@ def set_tls_field(t, offset, value): set_tcs_field(t, offs.TCS_OFS_LIMIT, '