Memory protection keys: Exception handler crashes if pkey0 is write-disabled

Question

Background: in-process isolation based on memory protection domains in x86/linux using memory protection keys (MPK) and the protection key register PKRU.

Setup: The program first executes administrator code that allocates a new protection key and associated memory, and moves the user stack pointer to that memory. This represents the switch to user code since it may operate only in that memory. If the user code causes an exception, I want the signal handler to pass execution back to the administrator -- this I got to work:

• (1) allocate protection key 1 pku=pkey_alloc() as well as associated memory to create a user stack ustack=mmap()+pkey_mprotect(ustack, ..., pku). Install exception handlers init_handler() for SEGV and FPE
• (2) switch rsp to user stack mov ustack, %%rsp
• (3a) We are now executing user code, which causes an exception e.g. *(int*)0=0. This triggers the signal handler via handler_asm()=>handler()=>unblock_signal(). Note, handler_asm() is necessary to allow handler() to operate on user stack
• (4) signal handler handler() returns to administrator, which switches rsp back to original stack

Minimal compilable source code demonstrating the administrator/user switch and signal handling is below, steps described as bullet points either happen in main or in functions as indicated by (). So far everything works as expected (Ubuntu 23.04)

Let's assume the user code needs to access only pre-allocated resources (i.e. does not need to malloc or call any c library functions). Now, to limit damage the user can do, I would like to disable write for all memory that does not belong to that user, in particular disabling write on pages with pkey 0 (setting PKRU.WD0=true i.e. PKRU=0x55555552). Thus, replacing 3a by either 3b or 3c

• (3b) set WD0=true via wrpkru // user causes no exception // set WD0=false => works fine
• (3c) WD0=true // user causes FPE or SEGV // WD0=false => system calls related to signal handling crash, see details below/or in code

How can I avoid this crash, is there a mistake I am making? Or is it generally not possible to successfully handle exceptions if WD0=true -- somewhat defeating the purpose of MPK?

Thank you for any help!!

---

Minimal implementation: Write-disable pkey0 and exception caused by user are controlled by two switches PROTECT_WD0 and INVOKE_SEGV, respectively. Switching between administrator/user happens in main()

PROTECT_WD0=0 PKRU for user code is 0x55555550

• INVOKE_SEGV=0: user causes FPE
• INVOKE_SEGV=1: user causes SEGV

PROTECT_WD0=1 PKRU for user code is 0x55555552

• INVOKE_SEGV=0: user causes FPE => crashes upon entering signal handler handler_asm()
• INVOKE_SEGV=1: user causes SEGV => crashes after unblock_signal() => sigprocmask() => __GI___sigprocmask => __GI___pthread_sigmask => syscall 14

Here, "crashes" means that when the kernel switches to my signal handler, it causes another SEGV at the specified location

#include <stdexcept>
#include <signal.h>
#include <unistd.h>
#include <sys/syscall.h>
#include <sys/mman.h>

// Save as "main.cpp" and compile via
// gcc -O0 -fexceptions -fnon-call-exceptions -g main.cpp
// gdb ./a.out
// after exception continue to handler via "signal SIGSEGV" or "signal SIGFPE"

// PROTECT_WD0=1 will write disable pkey 0
#define PROTECT_WD0 1
// INVOKE_SEGV=1: user code causes SEGV, =0: user code causes FPE
#define INVOKE_SEGV 1

uint8_t* ustack, *ostack;
void *ripret;

/////////////////////////////////////////////////////////////////////////////////////
// init_handler     installs signal handler "handler_asm"
// handler_asm      resets pkru before calling handler
// handler          modify RIP to return to after the error
// unblock_signal   ensure signal can be resent
// 
// modified from https://github.com/Plaristote/segvcatch
// except handler_asm, modified from https://github.com/IAIK/Donky
struct kernel_sigaction {
    void (*k_sa_sigaction)(int,siginfo_t *,void *);
    unsigned long k_sa_flags;
    void (*k_sa_restorer) (void);
    sigset_t k_sa_mask;
};

#  define RESTORE(name, syscall) RESTORE2 (name, syscall)
#  define RESTORE2(name, syscall)                     \
asm (                                                 \
   ".text\n"                                          \
   ".byte 0  # Yes, this really is necessary\n"       \
   ".align 16\n"                                      \
   "__" #name ":\n"                                   \
   "    movq $" #syscall ", %rax\n"                   \
   "    syscall\n"                                    \
   );

/* The return code for realtime-signals.  */
RESTORE (restore_rt, __NR_rt_sigreturn)
void restore_rt (void) asm ("__restore_rt")
  __attribute__ ((visibility ("hidden")));

static void unblock_signal(int signum __attribute__((__unused__))) {
    sigset_t sigs;
    sigemptyset(&sigs);
    sigaddset(&sigs, signum);
    // SIGSEGV crashes at sigprocmask => __GI___sigprocmask => __GI___pthread_sigmask => syscall 14
    sigprocmask(SIG_UNBLOCK, &sigs, NULL);
}

// Exception handler
void handler(int s, siginfo_t *, void *_p  __attribute__ ((__unused__))) {
  ucontext_t *_uc = (ucontext_t *)_p;                                             
  gregset_t &_gregs = _uc->uc_mcontext.gregs;                                   
  unblock_signal(s);
 _gregs[REG_RIP] = (greg_t)ripret;                               
}

// kernel resets pkru to 0x55555554: give full access before handling
void __attribute__((naked)) handler_asm(int, siginfo_t*, void *) {
    // SIGFPE crashes here
  __asm__ volatile(
    "mov %%rdx, %%r14\n"               //   save ucontext
    "xorl %%eax, %%eax; xorl %%ecx, %%ecx; xorl %%edx, %%edx; wrpkru;" // full access
    "mov %%r14, %%rdx\n"               //   restore ucontext
    "jmp %P0\n" :: "i"(handler));
} 

// install signal handlers
void init_handler(int signal) {
    struct kernel_sigaction act;                                        
    act.k_sa_sigaction = handler_asm;                     
    sigemptyset (&act.k_sa_mask);                                       
    act.k_sa_flags = SA_SIGINFO|0x4000000;                          
    act.k_sa_restorer = restore_rt;                                   
    syscall (SYS_rt_sigaction, signal, &act, NULL, _NSIG / 8);  
}

/////////////////////////////////////////////////////////////////////////////////////
int main(int argc, char *argv[]) {
    // allocate pkey (assumed:1) and associated stack ustack
    int pku=pkey_alloc(0,0);
    ustack = (uint8_t*)mmap(NULL, 0x10000, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS ,-1, 0);
    pkey_mprotect(ustack, 0x10000, PROT_READ | PROT_WRITE, pku); 
    ustack += 0xFFF0;

    // initialize handlers for SEGV and FPE
    // ripret is the address for the return from signal handler
    init_handler(SIGSEGV);
    init_handler(SIGFPE);
    ripret = &&ret;

    // ADMINISTRATOR: switch to user stack and write-disable pkey 0
    asm("mov %%rsp, %0; mov %1, %%rsp" : "=g" (ostack) : "g" (ustack));
    #if PROTECT_WD0
    asm("xorl %%ecx, %%ecx; rdpkru; xorl $2, %%eax; wrpkru" :::); 
    #endif

    // USER: causes SEGV or FPE
    #if INVOKE_SEGV
    *(int*) 0 = 0;
    #else
    ustack[0] = 0;
    ustack[0] = 10/ustack[0];
    #endif
ret:  
    // ADMINISTRATOR: write-enable pkey 0 and switch back to original stack
    #if PROTECT_WD0
    asm("xorl %%ecx, %%ecx; rdpkru; xorl $2, %%eax; wrpkru" :::);
    #endif
    asm("mov %0, %%rsp" : : "g" (ostack));

    printf("done\n");
    return 0;
}