x86: shadow stack support

compel/arch/aarch64/src/lib/infect.c

@@ -59,10 +59,9 @@ int sigreturn_prep_fpu_frame_plain(struct rt_sigframe *sigframe, struct rt_sigfr
 	return 0;
 }
 
-int compel_get_task_regs(pid_t pid, user_regs_struct_t *regs, user_fpregs_struct_t *ext_regs, save_regs_t save,
+int compel_get_task_regs(pid_t pid, user_regs_struct_t *regs, user_fpregs_struct_t *fpsimd, save_regs_t save,
 			 void *arg, __maybe_unused unsigned long flags)
 {
-	user_fpregs_struct_t tmp, *fpsimd = ext_regs ? ext_regs : &tmp;
 	struct iovec iov;
 	int ret;
 

compel/arch/arm/src/lib/infect.c

@@ -65,10 +65,9 @@ int sigreturn_prep_fpu_frame_plain(struct rt_sigframe *sigframe, struct rt_sigfr
 }
 
 #define PTRACE_GETVFPREGS 27
-int compel_get_task_regs(pid_t pid, user_regs_struct_t *regs, user_fpregs_struct_t *ext_regs, save_regs_t save,
+int compel_get_task_regs(pid_t pid, user_regs_struct_t *regs, user_fpregs_struct_t *vfp, save_regs_t save,
 			 void *arg, __maybe_unused unsigned long flags)
 {
-	user_fpregs_struct_t tmp, *vfp = ext_regs ? ext_regs : &tmp;
 	int ret = -1;
 
 	pr_info("Dumping GP/FPU registers for %d\n", pid);

compel/arch/mips/src/lib/infect.c

@@ -119,10 +119,9 @@ int sigreturn_prep_fpu_frame_plain(struct rt_sigframe *sigframe, struct rt_sigfr
 	return 0;
 }
 
-int compel_get_task_regs(pid_t pid, user_regs_struct_t *regs, user_fpregs_struct_t *ext_regs, save_regs_t save,
+int compel_get_task_regs(pid_t pid, user_regs_struct_t *regs, user_fpregs_struct_t *xs, save_regs_t save,
 			 void *arg, __maybe_unused unsigned long flags)
 {
-	user_fpregs_struct_t xsave = {}, *xs = ext_regs ? ext_regs : &xsave;
 	int ret = -1;
 
 	pr_info("Dumping GP/FPU registers for %d\n", pid);

compel/arch/ppc64/src/lib/infect.c

@@ -391,10 +391,9 @@ static int __get_task_regs(pid_t pid, user_regs_struct_t *regs, user_fpregs_stru
 	return 0;
 }
 
-int compel_get_task_regs(pid_t pid, user_regs_struct_t *regs, user_fpregs_struct_t *ext_regs, save_regs_t save,
+int compel_get_task_regs(pid_t pid, user_regs_struct_t *regs, user_fpregs_struct_t *fpregs, save_regs_t save,
 			 void *arg, __maybe_unused unsigned long flags)
 {
-	user_fpregs_struct_t tmp, *fpregs = ext_regs ? ext_regs : &tmp;
 	int ret;
 
 	ret = __get_task_regs(pid, regs, fpregs);

compel/arch/s390/src/lib/infect.c

@@ -293,10 +293,9 @@ static int s390_disable_ri_bit(pid_t pid, user_regs_struct_t *regs)
 /*
  * Prepare task registers for restart
  */
-int compel_get_task_regs(pid_t pid, user_regs_struct_t *regs, user_fpregs_struct_t *ext_regs, save_regs_t save,
+int compel_get_task_regs(pid_t pid, user_regs_struct_t *regs, user_fpregs_struct_t *fpregs, save_regs_t save,
 			 void *arg, __maybe_unused unsigned long flags)
 {
-	user_fpregs_struct_t tmp, *fpregs = ext_regs ? ext_regs : &tmp;
 	struct iovec iov;
 	int rewind;
 

compel/arch/x86/plugins/std/parasite-head.S

@@ -34,7 +34,21 @@ END(__export_parasite_head_start_compat)
 .code64
 #endif
 
+/*
+ * When parasite_service() runs in the daemon mode it will return the stack
+ * pointer for the sigreturn frame in %rax and we call sigreturn directly
+ * from here.
+ * Since a valid stack pointer is positive, it is safe to presume that
+ * return value <= 0 means that parasite_service() called parasite_trap_cmd()
+ * in non-daemon mode, and the parasite should stop at int3.
+ */
 ENTRY(__export_parasite_head_start)
 	call	parasite_service
+	cmp	$0, %rax
+	jle	1f
+	movq	%rax, %rsp
+	movq	$15, %rax
+	syscall
+1:
 	int	$0x03
 END(__export_parasite_head_start)

compel/arch/x86/src/lib/include/uapi/asm/cpu.h

@@ -244,6 +244,7 @@ enum cpuid_leafs {
 #define X86_FEATURE_PKU		     (11 * 32 + 3)  /* Protection Keys for Userspace */
 #define X86_FEATURE_OSPKE	     (11 * 32 + 4)  /* OS Protection Keys Enable */
 #define X86_FEATURE_AVX512_VBMI2     (11 * 32 + 6)  /* Additional AVX512 Vector Bit Manipulation Instructions */
+#define X86_FEATURE_SHSTK            (11 * 32 + 7)  /* Shadow Stack */
 #define X86_FEATURE_GFNI	     (11 * 32 + 8)  /* Galois Field New Instructions */
 #define X86_FEATURE_VAES	     (11 * 32 + 9)  /* Vector AES */
 #define X86_FEATURE_VPCLMULQDQ	     (11 * 32 + 10) /* Carry-Less Multiplication Double Quadword */

compel/arch/x86/src/lib/include/uapi/asm/fpu.h

@@ -245,6 +245,14 @@ struct pkru_state {
 	uint32_t pad;
 } __packed;
 
+/*
+ * State component 11 is Control-flow Enforcement user states
+ */
+struct cet_user_state {
+	uint64_t cet;			/* user control-flow settings */
+	uint64_t ssp;			/* user shadow stack pointer */
+};
+
 /*
  * This is our most modern FPU state format, as saved by the XSAVE
  * and restored by the XRSTOR instructions.
@@ -260,7 +268,7 @@ struct pkru_state {
  * Of course it was not ;-) Now using four pages...
  *
  */
-#define EXTENDED_STATE_AREA_SIZE (XSAVE_SIZE - sizeof(struct i387_fxsave_struct) - sizeof(struct xsave_hdr_struct))
+#define EXTENDED_STATE_AREA_SIZE (XSAVE_SIZE - sizeof(struct i387_fxsave_struct) - sizeof(struct xsave_hdr_struct) - sizeof(struct cet_user_state))
 
 /*
  * cpu requires it to be 64 byte aligned
@@ -276,6 +284,7 @@ struct xsave_struct {
 		struct ymmh_struct ymmh;
 		uint8_t extended_state_area[EXTENDED_STATE_AREA_SIZE];
 	};
+	struct cet_user_state cet;
 } __aligned(FP_MIN_ALIGN_BYTES) __packed;
 
 struct xsave_struct_ia32 {

compel/arch/x86/src/lib/include/uapi/asm/infect-types.h

@@ -143,4 +143,11 @@ typedef struct xsave_struct user_fpregs_struct_t;
  */
 #define __NR32_mmap __NR32_mmap2
 
+extern bool __compel_shstk_enabled(user_fpregs_struct_t *ext_regs);
+#define compel_shstk_enabled __compel_shstk_enabled
+
+extern int __parasite_setup_shstk(struct parasite_ctl *ctl,
+				user_fpregs_struct_t *ext_regs);
+#define parasite_setup_shstk __parasite_setup_shstk
+
 #endif /* UAPI_COMPEL_ASM_TYPES_H__ */

compel/arch/x86/src/lib/include/uapi/asm/sigframe.h

@@ -203,13 +203,21 @@ static inline void rt_sigframe_erase_sigset(struct rt_sigframe *sigframe)
 		: "rdi"(new_sp)						\
 		: "eax", "r8", "r9", "r10", "r11", "memory")
 
-#define ARCH_RT_SIGRETURN(new_sp, rt_sigframe)				\
+#define ARCH_RT_SIGRETURN_RST(new_sp, rt_sigframe)			\
 do {									\
 	if ((rt_sigframe)->is_native)					\
 		ARCH_RT_SIGRETURN_NATIVE(new_sp);			\
 	else								\
 		ARCH_RT_SIGRETURN_COMPAT(new_sp);			\
 } while (0)
+
+#define ARCH_RT_SIGRETURN_DUMP(new_sp, rt_sigframe)			\
+do {									\
+	if ((rt_sigframe)->is_native)					\
+		return new_sp;						\
+	else								\
+		ARCH_RT_SIGRETURN_COMPAT(new_sp);			\
+} while (0)
 /* clang-format off */
 
 int sigreturn_prep_fpu_frame(struct rt_sigframe *sigframe,

compel/arch/x86/src/lib/infect.c

@@ -26,6 +26,16 @@
 #ifndef NT_X86_XSTATE
 #define NT_X86_XSTATE 0x202 /* x86 extended state using xsave */
 #endif
+
+#ifndef NT_X86_SHSTK
+#define NT_X86_SHSTK 0x204	/* x86 shstk state */
+#endif
+
+#ifndef ARCH_SHSTK_STATUS
+#define ARCH_SHSTK_STATUS	0x5005
+#define ARCH_SHSTK_SHSTK	(1ULL << 0)
+#endif
+
 #ifndef NT_PRSTATUS
 #define NT_PRSTATUS 1 /* Contains copy of prstatus struct */
 #endif
@@ -250,7 +260,49 @@ static int get_task_xsave(pid_t pid, user_fpregs_struct_t *xsave)
 		// [1] Intel® 64 and IA-32 Architectures Software Developer's
 		//     Manual Volume 1: Basic Architecture
 		//     Section 13.6: Processor tracking of XSAVE-managed state
-		return get_task_fpregs(pid, xsave);
+		if (get_task_fpregs(pid, xsave))
+			return -1;
+	}
+
+	/*
+	 * xsave may be on stack, if we don't clear it explicitly we get
+	 * funky shadow stack state
+	 */
+	memset(&xsave->cet, 0, sizeof(xsave->cet));
+	if (compel_cpu_has_feature(X86_FEATURE_SHSTK)) {
+		unsigned long ssp = 0;
+		unsigned long features = 0;
+
+		if (ptrace(PTRACE_ARCH_PRCTL, pid, (unsigned long)&features, ARCH_SHSTK_STATUS)) {
+			/*
+			 * kernels that don't support shadow stack return
+			 * -EINVAL
+			 */
+			if (errno == EINVAL)
+				return 0;
+
+			pr_perror("shstk: can't get shadow stack status for %d", pid);
+			return -1;
+		}
+
+		if (!(features & ARCH_SHSTK_SHSTK))
+			return 0;
+
+		iov.iov_base = &ssp;
+		iov.iov_len = sizeof(ssp);
+
+		if (ptrace(PTRACE_GETREGSET, pid, (unsigned int)NT_X86_SHSTK, &iov) < 0) {
+			/* ENODEV means CET is not supported by the CPU  */
+			if (errno != ENODEV) {
+				pr_perror("shstk: can't get SSP for %d", pid);
+				return -1;
+			}
+		}
+
+		xsave->cet.cet = features;
+		xsave->cet.ssp = ssp;
+
+		pr_debug("%d: shstk: cet: %lx ssp: %lx\n", pid, xsave->cet.cet, xsave->cet.ssp);
 	}
 
 	return 0;
@@ -345,10 +397,9 @@ static int corrupt_extregs(pid_t pid)
 	return 0;
 }
 
-int compel_get_task_regs(pid_t pid, user_regs_struct_t *regs, user_fpregs_struct_t *ext_regs, save_regs_t save,
+int compel_get_task_regs(pid_t pid, user_regs_struct_t *regs, user_fpregs_struct_t *xs, save_regs_t save,
 			 void *arg, unsigned long flags)
 {
-	user_fpregs_struct_t xsave = {}, *xs = ext_regs ? ext_regs : &xsave;
 	int ret = -1;
 
 	pr_info("Dumping general registers for %d in %s mode\n", pid, user_regs_native(regs) ? "native" : "compat");
@@ -698,3 +749,59 @@ unsigned long compel_task_size(void)
 {
 	return TASK_SIZE;
 }
+
+bool __compel_shstk_enabled(user_fpregs_struct_t *ext_regs)
+{
+	if (!compel_cpu_has_feature(X86_FEATURE_SHSTK))
+		return false;
+
+	if (ext_regs->cet.cet & ARCH_SHSTK_SHSTK)
+		return true;
+
+	return false;
+}
+
+int parasite_setup_shstk(struct parasite_ctl *ctl, user_fpregs_struct_t *ext_regs)
+{
+	pid_t pid = ctl->rpid;
+	unsigned long sa_restorer = ctl->parasite_ip;
+	unsigned long long ssp;
+	unsigned long token;
+	struct iovec iov;
+
+	if (!compel_shstk_enabled(ext_regs))
+		return 0;
+
+	iov.iov_base = &ssp;
+	iov.iov_len = sizeof(ssp);
+	if (ptrace(PTRACE_GETREGSET, pid, (unsigned int)NT_X86_SHSTK, &iov) < 0) {
+		/* ENODEV means CET is not supported by the CPU  */
+		if (errno != ENODEV) {
+			pr_perror("shstk: %d: cannot get SSP", pid);
+			return -1;
+		}
+	}
+
+	/* The token is for 64-bit */
+	token = ALIGN_DOWN(ssp, 8);
+	token |= (1UL << 63);
+	ssp = ALIGN_DOWN(ssp, 8) - 8;
+	if (ptrace(PTRACE_POKEDATA, pid, (void *)ssp, token)) {
+		pr_perror("shstk: %d: failed to inject shadow stack token", pid);
+		return -1;
+	}
+
+	ssp = ssp - sizeof(uint64_t);
+	if (ptrace(PTRACE_POKEDATA, pid, (void *)ssp, sa_restorer)) {
+		pr_perror("shstk: %d: failed to inject restorer address", pid);
+		return -1;
+	}
+
+	ssp = ssp + sizeof(uint64_t);
+	if (ptrace(PTRACE_SETREGSET, pid, (unsigned int)NT_X86_SHSTK, &iov) < 0) {
+		pr_perror("shstk: %d: cannot write SSP", pid);
+		return -1;
+	}
+
+	return 0;
+}

compel/include/uapi/infect.h

@@ -182,4 +182,21 @@ void compel_set_thread_ip(struct parasite_thread_ctl *tctl, uint64_t v);
 
 extern void compel_get_stack(struct parasite_ctl *ctl, void **rstack, void **r_thread_stack);
 
+#ifndef compel_shstk_enabled
+static inline bool compel_shstk_enabled(user_fpregs_struct_t *ext_regs)
+{
+	return false;
+}
+#define compel_shstk_enabled
+#endif
+
+#ifndef parasite_setup_shstk
+static inline int parasite_setup_shstk(struct parasite_ctl *ctl,
+				       user_fpregs_struct_t *ext_regs)
+{
+	return 0;
+}
+#define parasite_setup_shstk parasite_setup_shstk
+#endif
+
 #endif

compel/plugins/include/uapi/std/infect.h

@@ -7,7 +7,7 @@ extern int parasite_get_rpc_sock(void);
 
 extern unsigned int __export_parasite_service_cmd;
 extern void *__export_parasite_service_args_ptr;
-extern int __must_check parasite_service(void);
+extern unsigned long __must_check parasite_service(void);
 
 /*
  * Must be supplied by user plugins.