commit: c7d7705101af05e259b3b84ffc59a60ff2b96142 Author: Alice Ferrazzi <alicef <AT> gentoo <DOT> org> AuthorDate: Fri Jan 5 15:05:38 2018 +0000 Commit: Alice Ferrazzi <alicef <AT> gentoo <DOT> org> CommitDate: Fri Jan 5 15:05:38 2018 +0000 URL: https://gitweb.gentoo.org/proj/linux-patches.git/commit/?id=c7d77051
linux kernel 4.4.110 0000_README | 4 + 1109_linux-4.4.110.patch | 2814 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 2818 insertions(+) diff --git a/0000_README b/0000_README index 3be106c..46149de 100644 --- a/0000_README +++ b/0000_README @@ -479,6 +479,10 @@ Patch: 1108_linux-4.4.109.patch From: http://www.kernel.org Desc: Linux 4.4.109 +Patch: 1109_linux-4.4.110.patch +From: http://www.kernel.org +Desc: Linux 4.4.110 + Patch: 1500_XATTR_USER_PREFIX.patch From: https://bugs.gentoo.org/show_bug.cgi?id=470644 Desc: Support for namespace user.pax.* on tmpfs. diff --git a/1109_linux-4.4.110.patch b/1109_linux-4.4.110.patch new file mode 100644 index 0000000..1c226ed --- /dev/null +++ b/1109_linux-4.4.110.patch @@ -0,0 +1,2814 @@ +diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt +index b4a83a490212..5977c4d71356 100644 +--- a/Documentation/kernel-parameters.txt ++++ b/Documentation/kernel-parameters.txt +@@ -2523,6 +2523,8 @@ bytes respectively. Such letter suffixes can also be entirely omitted. + + nojitter [IA-64] Disables jitter checking for ITC timers. + ++ nopti [X86-64] Disable KAISER isolation of kernel from user. ++ + no-kvmclock [X86,KVM] Disable paravirtualized KVM clock driver + + no-kvmapf [X86,KVM] Disable paravirtualized asynchronous page +@@ -3054,6 +3056,12 @@ bytes respectively. Such letter suffixes can also be entirely omitted. + pt. [PARIDE] + See Documentation/blockdev/paride.txt. + ++ pti= [X86_64] ++ Control KAISER user/kernel address space isolation: ++ on - enable ++ off - disable ++ auto - default setting ++ + pty.legacy_count= + [KNL] Number of legacy pty's. Overwrites compiled-in + default number. +diff --git a/Makefile b/Makefile +index 5d67056e24dd..b028c106535b 100644 +--- a/Makefile ++++ b/Makefile +@@ -1,6 +1,6 @@ + VERSION = 4 + PATCHLEVEL = 4 +-SUBLEVEL = 109 ++SUBLEVEL = 110 + EXTRAVERSION = + NAME = Blurry Fish Butt + +diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h +index 3783dc3e10b3..4abb284a5b9c 100644 +--- a/arch/x86/boot/compressed/misc.h ++++ b/arch/x86/boot/compressed/misc.h +@@ -9,6 +9,7 @@ + */ + #undef CONFIG_PARAVIRT + #undef CONFIG_PARAVIRT_SPINLOCKS ++#undef CONFIG_PAGE_TABLE_ISOLATION + #undef CONFIG_KASAN + + #include <linux/linkage.h> +diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S +index cc0f2f5da19b..952b23b5d4e9 100644 +--- a/arch/x86/entry/entry_64.S ++++ b/arch/x86/entry/entry_64.S +@@ -35,6 +35,7 @@ + #include <asm/asm.h> + #include <asm/smap.h> + #include <asm/pgtable_types.h> ++#include <asm/kaiser.h> + #include <linux/err.h> + + /* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */ +@@ -135,6 +136,7 @@ ENTRY(entry_SYSCALL_64) + * it is too small to ever cause noticeable irq latency. + */ + SWAPGS_UNSAFE_STACK ++ SWITCH_KERNEL_CR3_NO_STACK + /* + * A hypervisor implementation might want to use a label + * after the swapgs, so that it can do the swapgs +@@ -207,9 +209,17 @@ entry_SYSCALL_64_fastpath: + testl $_TIF_ALLWORK_MASK, ASM_THREAD_INFO(TI_flags, %rsp, SIZEOF_PTREGS) + jnz int_ret_from_sys_call_irqs_off /* Go to the slow path */ + +- RESTORE_C_REGS_EXCEPT_RCX_R11 + movq RIP(%rsp), %rcx + movq EFLAGS(%rsp), %r11 ++ RESTORE_C_REGS_EXCEPT_RCX_R11 ++ /* ++ * This opens a window where we have a user CR3, but are ++ * running in the kernel. This makes using the CS ++ * register useless for telling whether or not we need to ++ * switch CR3 in NMIs. Normal interrupts are OK because ++ * they are off here. ++ */ ++ SWITCH_USER_CR3 + movq RSP(%rsp), %rsp + /* + * 64-bit SYSRET restores rip from rcx, +@@ -347,10 +357,26 @@ GLOBAL(int_ret_from_sys_call) + syscall_return_via_sysret: + /* rcx and r11 are already restored (see code above) */ + RESTORE_C_REGS_EXCEPT_RCX_R11 ++ /* ++ * This opens a window where we have a user CR3, but are ++ * running in the kernel. This makes using the CS ++ * register useless for telling whether or not we need to ++ * switch CR3 in NMIs. Normal interrupts are OK because ++ * they are off here. ++ */ ++ SWITCH_USER_CR3 + movq RSP(%rsp), %rsp + USERGS_SYSRET64 + + opportunistic_sysret_failed: ++ /* ++ * This opens a window where we have a user CR3, but are ++ * running in the kernel. This makes using the CS ++ * register useless for telling whether or not we need to ++ * switch CR3 in NMIs. Normal interrupts are OK because ++ * they are off here. ++ */ ++ SWITCH_USER_CR3 + SWAPGS + jmp restore_c_regs_and_iret + END(entry_SYSCALL_64) +@@ -509,6 +535,7 @@ END(irq_entries_start) + * tracking that we're in kernel mode. + */ + SWAPGS ++ SWITCH_KERNEL_CR3 + + /* + * We need to tell lockdep that IRQs are off. We can't do this until +@@ -568,6 +595,7 @@ GLOBAL(retint_user) + mov %rsp,%rdi + call prepare_exit_to_usermode + TRACE_IRQS_IRETQ ++ SWITCH_USER_CR3 + SWAPGS + jmp restore_regs_and_iret + +@@ -625,6 +653,7 @@ native_irq_return_ldt: + pushq %rax + pushq %rdi + SWAPGS ++ SWITCH_KERNEL_CR3 + movq PER_CPU_VAR(espfix_waddr), %rdi + movq %rax, (0*8)(%rdi) /* RAX */ + movq (2*8)(%rsp), %rax /* RIP */ +@@ -640,6 +669,7 @@ native_irq_return_ldt: + andl $0xffff0000, %eax + popq %rdi + orq PER_CPU_VAR(espfix_stack), %rax ++ SWITCH_USER_CR3 + SWAPGS + movq %rax, %rsp + popq %rax +@@ -995,7 +1025,11 @@ idtentry machine_check has_error_code=0 paranoid=1 do_sym=*machine_check_vec + /* + * Save all registers in pt_regs, and switch gs if needed. + * Use slow, but surefire "are we in kernel?" check. +- * Return: ebx=0: need swapgs on exit, ebx=1: otherwise ++ * ++ * Return: ebx=0: needs swapgs but not SWITCH_USER_CR3 in paranoid_exit ++ * ebx=1: needs neither swapgs nor SWITCH_USER_CR3 in paranoid_exit ++ * ebx=2: needs both swapgs and SWITCH_USER_CR3 in paranoid_exit ++ * ebx=3: needs SWITCH_USER_CR3 but not swapgs in paranoid_exit + */ + ENTRY(paranoid_entry) + cld +@@ -1008,7 +1042,26 @@ ENTRY(paranoid_entry) + js 1f /* negative -> in kernel */ + SWAPGS + xorl %ebx, %ebx +-1: ret ++1: ++#ifdef CONFIG_PAGE_TABLE_ISOLATION ++ /* ++ * We might have come in between a swapgs and a SWITCH_KERNEL_CR3 ++ * on entry, or between a SWITCH_USER_CR3 and a swapgs on exit. ++ * Do a conditional SWITCH_KERNEL_CR3: this could safely be done ++ * unconditionally, but we need to find out whether the reverse ++ * should be done on return (conveyed to paranoid_exit in %ebx). ++ */ ++ ALTERNATIVE "jmp 2f", "movq %cr3, %rax", X86_FEATURE_KAISER ++ testl $KAISER_SHADOW_PGD_OFFSET, %eax ++ jz 2f ++ orl $2, %ebx ++ andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), %rax ++ /* If PCID enabled, set X86_CR3_PCID_NOFLUSH_BIT */ ++ ALTERNATIVE "", "bts $63, %rax", X86_FEATURE_PCID ++ movq %rax, %cr3 ++2: ++#endif ++ ret + END(paranoid_entry) + + /* +@@ -1021,19 +1074,26 @@ END(paranoid_entry) + * be complicated. Fortunately, we there's no good reason + * to try to handle preemption here. + * +- * On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it) ++ * On entry: ebx=0: needs swapgs but not SWITCH_USER_CR3 ++ * ebx=1: needs neither swapgs nor SWITCH_USER_CR3 ++ * ebx=2: needs both swapgs and SWITCH_USER_CR3 ++ * ebx=3: needs SWITCH_USER_CR3 but not swapgs + */ + ENTRY(paranoid_exit) + DISABLE_INTERRUPTS(CLBR_NONE) + TRACE_IRQS_OFF_DEBUG +- testl %ebx, %ebx /* swapgs needed? */ ++ TRACE_IRQS_IRETQ_DEBUG ++#ifdef CONFIG_PAGE_TABLE_ISOLATION ++ /* No ALTERNATIVE for X86_FEATURE_KAISER: paranoid_entry sets %ebx */ ++ testl $2, %ebx /* SWITCH_USER_CR3 needed? */ ++ jz paranoid_exit_no_switch ++ SWITCH_USER_CR3 ++paranoid_exit_no_switch: ++#endif ++ testl $1, %ebx /* swapgs needed? */ + jnz paranoid_exit_no_swapgs +- TRACE_IRQS_IRETQ + SWAPGS_UNSAFE_STACK +- jmp paranoid_exit_restore + paranoid_exit_no_swapgs: +- TRACE_IRQS_IRETQ_DEBUG +-paranoid_exit_restore: + RESTORE_EXTRA_REGS + RESTORE_C_REGS + REMOVE_PT_GPREGS_FROM_STACK 8 +@@ -1048,6 +1108,13 @@ ENTRY(error_entry) + cld + SAVE_C_REGS 8 + SAVE_EXTRA_REGS 8 ++ /* ++ * error_entry() always returns with a kernel gsbase and ++ * CR3. We must also have a kernel CR3/gsbase before ++ * calling TRACE_IRQS_*. Just unconditionally switch to ++ * the kernel CR3 here. ++ */ ++ SWITCH_KERNEL_CR3 + xorl %ebx, %ebx + testb $3, CS+8(%rsp) + jz .Lerror_kernelspace +@@ -1210,6 +1277,10 @@ ENTRY(nmi) + */ + + SWAPGS_UNSAFE_STACK ++ /* ++ * percpu variables are mapped with user CR3, so no need ++ * to switch CR3 here. ++ */ + cld + movq %rsp, %rdx + movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp +@@ -1243,12 +1314,34 @@ ENTRY(nmi) + + movq %rsp, %rdi + movq $-1, %rsi ++#ifdef CONFIG_PAGE_TABLE_ISOLATION ++ /* Unconditionally use kernel CR3 for do_nmi() */ ++ /* %rax is saved above, so OK to clobber here */ ++ ALTERNATIVE "jmp 2f", "movq %cr3, %rax", X86_FEATURE_KAISER ++ /* If PCID enabled, NOFLUSH now and NOFLUSH on return */ ++ ALTERNATIVE "", "bts $63, %rax", X86_FEATURE_PCID ++ pushq %rax ++ /* mask off "user" bit of pgd address and 12 PCID bits: */ ++ andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), %rax ++ movq %rax, %cr3 ++2: ++#endif + call do_nmi + ++#ifdef CONFIG_PAGE_TABLE_ISOLATION ++ /* ++ * Unconditionally restore CR3. I know we return to ++ * kernel code that needs user CR3, but do we ever return ++ * to "user mode" where we need the kernel CR3? ++ */ ++ ALTERNATIVE "", "popq %rax; movq %rax, %cr3", X86_FEATURE_KAISER ++#endif ++ + /* + * Return back to user mode. We must *not* do the normal exit +- * work, because we don't want to enable interrupts. Fortunately, +- * do_nmi doesn't modify pt_regs. ++ * work, because we don't want to enable interrupts. Do not ++ * switch to user CR3: we might be going back to kernel code ++ * that had a user CR3 set. + */ + SWAPGS + jmp restore_c_regs_and_iret +@@ -1445,22 +1538,55 @@ end_repeat_nmi: + ALLOC_PT_GPREGS_ON_STACK + + /* +- * Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit +- * as we should not be calling schedule in NMI context. +- * Even with normal interrupts enabled. An NMI should not be +- * setting NEED_RESCHED or anything that normal interrupts and +- * exceptions might do. ++ * Use the same approach as paranoid_entry to handle SWAPGS, but ++ * without CR3 handling since we do that differently in NMIs. No ++ * need to use paranoid_exit as we should not be calling schedule ++ * in NMI context. Even with normal interrupts enabled. An NMI ++ * should not be setting NEED_RESCHED or anything that normal ++ * interrupts and exceptions might do. + */ +- call paranoid_entry +- +- /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */ ++ cld ++ SAVE_C_REGS ++ SAVE_EXTRA_REGS ++ movl $1, %ebx ++ movl $MSR_GS_BASE, %ecx ++ rdmsr ++ testl %edx, %edx ++ js 1f /* negative -> in kernel */ ++ SWAPGS ++ xorl %ebx, %ebx ++1: + movq %rsp, %rdi + movq $-1, %rsi ++#ifdef CONFIG_PAGE_TABLE_ISOLATION ++ /* Unconditionally use kernel CR3 for do_nmi() */ ++ /* %rax is saved above, so OK to clobber here */ ++ ALTERNATIVE "jmp 2f", "movq %cr3, %rax", X86_FEATURE_KAISER ++ /* If PCID enabled, NOFLUSH now and NOFLUSH on return */ ++ ALTERNATIVE "", "bts $63, %rax", X86_FEATURE_PCID ++ pushq %rax ++ /* mask off "user" bit of pgd address and 12 PCID bits: */ ++ andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), %rax ++ movq %rax, %cr3 ++2: ++#endif ++ ++ /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */ + call do_nmi + ++#ifdef CONFIG_PAGE_TABLE_ISOLATION ++ /* ++ * Unconditionally restore CR3. We might be returning to ++ * kernel code that needs user CR3, like just just before ++ * a sysret. ++ */ ++ ALTERNATIVE "", "popq %rax; movq %rax, %cr3", X86_FEATURE_KAISER ++#endif ++ + testl %ebx, %ebx /* swapgs needed? */ + jnz nmi_restore + nmi_swapgs: ++ /* We fixed up CR3 above, so no need to switch it here */ + SWAPGS_UNSAFE_STACK + nmi_restore: + RESTORE_EXTRA_REGS +diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_compat.S +index 15cfebaa7688..d03bf0e28b8b 100644 +--- a/arch/x86/entry/entry_64_compat.S ++++ b/arch/x86/entry/entry_64_compat.S +@@ -13,6 +13,8 @@ + #include <asm/irqflags.h> + #include <asm/asm.h> + #include <asm/smap.h> ++#include <asm/pgtable_types.h> ++#include <asm/kaiser.h> + #include <linux/linkage.h> + #include <linux/err.h> + +@@ -50,6 +52,7 @@ ENDPROC(native_usergs_sysret32) + ENTRY(entry_SYSENTER_compat) + /* Interrupts are off on entry. */ + SWAPGS_UNSAFE_STACK ++ SWITCH_KERNEL_CR3_NO_STACK + movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp + + /* +@@ -161,6 +164,7 @@ ENDPROC(entry_SYSENTER_compat) + ENTRY(entry_SYSCALL_compat) + /* Interrupts are off on entry. */ + SWAPGS_UNSAFE_STACK ++ SWITCH_KERNEL_CR3_NO_STACK + + /* Stash user ESP and switch to the kernel stack. */ + movl %esp, %r8d +@@ -208,6 +212,7 @@ ENTRY(entry_SYSCALL_compat) + /* Opportunistic SYSRET */ + sysret32_from_system_call: + TRACE_IRQS_ON /* User mode traces as IRQs on. */ ++ SWITCH_USER_CR3 + movq RBX(%rsp), %rbx /* pt_regs->rbx */ + movq RBP(%rsp), %rbp /* pt_regs->rbp */ + movq EFLAGS(%rsp), %r11 /* pt_regs->flags (in r11) */ +@@ -269,6 +274,7 @@ ENTRY(entry_INT80_compat) + PARAVIRT_ADJUST_EXCEPTION_FRAME + ASM_CLAC /* Do this early to minimize exposure */ + SWAPGS ++ SWITCH_KERNEL_CR3_NO_STACK + + /* + * User tracing code (ptrace or signal handlers) might assume that +@@ -311,6 +317,7 @@ ENTRY(entry_INT80_compat) + + /* Go back to user mode. */ + TRACE_IRQS_ON ++ SWITCH_USER_CR3 + SWAPGS + jmp restore_regs_and_iret + END(entry_INT80_compat) +diff --git a/arch/x86/entry/vdso/vclock_gettime.c b/arch/x86/entry/vdso/vclock_gettime.c +index ca94fa649251..5dd363d54348 100644 +--- a/arch/x86/entry/vdso/vclock_gettime.c ++++ b/arch/x86/entry/vdso/vclock_gettime.c +@@ -36,6 +36,11 @@ static notrace cycle_t vread_hpet(void) + } + #endif + ++#ifdef CONFIG_PARAVIRT_CLOCK ++extern u8 pvclock_page ++ __attribute__((visibility("hidden"))); ++#endif ++ + #ifndef BUILD_VDSO32 + + #include <linux/kernel.h> +@@ -62,63 +67,65 @@ notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz) + + #ifdef CONFIG_PARAVIRT_CLOCK + +-static notrace const struct pvclock_vsyscall_time_info *get_pvti(int cpu) ++static notrace const struct pvclock_vsyscall_time_info *get_pvti0(void) + { +- const struct pvclock_vsyscall_time_info *pvti_base; +- int idx = cpu / (PAGE_SIZE/PVTI_SIZE); +- int offset = cpu % (PAGE_SIZE/PVTI_SIZE); +- +- BUG_ON(PVCLOCK_FIXMAP_BEGIN + idx > PVCLOCK_FIXMAP_END); +- +- pvti_base = (struct pvclock_vsyscall_time_info *) +- __fix_to_virt(PVCLOCK_FIXMAP_BEGIN+idx); +- +- return &pvti_base[offset]; ++ return (const struct pvclock_vsyscall_time_info *)&pvclock_page; + } + + static notrace cycle_t vread_pvclock(int *mode) + { +- const struct pvclock_vsyscall_time_info *pvti; ++ const struct pvclock_vcpu_time_info *pvti = &get_pvti0()->pvti; + cycle_t ret; +- u64 last; +- u32 version; +- u8 flags; +- unsigned cpu, cpu1; +- ++ u64 tsc, pvti_tsc; ++ u64 last, delta, pvti_system_time; ++ u32 version, pvti_tsc_to_system_mul, pvti_tsc_shift; + + /* +- * Note: hypervisor must guarantee that: +- * 1. cpu ID number maps 1:1 to per-CPU pvclock time info. +- * 2. that per-CPU pvclock time info is updated if the +- * underlying CPU changes. +- * 3. that version is increased whenever underlying CPU +- * changes. ++ * Note: The kernel and hypervisor must guarantee that cpu ID ++ * number maps 1:1 to per-CPU pvclock time info. ++ * ++ * Because the hypervisor is entirely unaware of guest userspace ++ * preemption, it cannot guarantee that per-CPU pvclock time ++ * info is updated if the underlying CPU changes or that that ++ * version is increased whenever underlying CPU changes. + * ++ * On KVM, we are guaranteed that pvti updates for any vCPU are ++ * atomic as seen by *all* vCPUs. This is an even stronger ++ * guarantee than we get with a normal seqlock. ++ * ++ * On Xen, we don't appear to have that guarantee, but Xen still ++ * supplies a valid seqlock using the version field. ++ ++ * We only do pvclock vdso timing at all if ++ * PVCLOCK_TSC_STABLE_BIT is set, and we interpret that bit to ++ * mean that all vCPUs have matching pvti and that the TSC is ++ * synced, so we can just look at vCPU 0's pvti. + */ +- do { +- cpu = __getcpu() & VGETCPU_CPU_MASK; +- /* TODO: We can put vcpu id into higher bits of pvti.version. +- * This will save a couple of cycles by getting rid of +- * __getcpu() calls (Gleb). +- */ +- +- pvti = get_pvti(cpu); +- +- version = __pvclock_read_cycles(&pvti->pvti, &ret, &flags); +- +- /* +- * Test we're still on the cpu as well as the version. +- * We could have been migrated just after the first +- * vgetcpu but before fetching the version, so we +- * wouldn't notice a version change. +- */ +- cpu1 = __getcpu() & VGETCPU_CPU_MASK; +- } while (unlikely(cpu != cpu1 || +- (pvti->pvti.version & 1) || +- pvti->pvti.version != version)); +- +- if (unlikely(!(flags & PVCLOCK_TSC_STABLE_BIT))) ++ ++ if (unlikely(!(pvti->flags & PVCLOCK_TSC_STABLE_BIT))) { + *mode = VCLOCK_NONE; ++ return 0; ++ } ++ ++ do { ++ version = pvti->version; ++ ++ /* This is also a read barrier, so we'll read version first. */ ++ tsc = rdtsc_ordered(); ++ ++ pvti_tsc_to_system_mul = pvti->tsc_to_system_mul; ++ pvti_tsc_shift = pvti->tsc_shift; ++ pvti_system_time = pvti->system_time; ++ pvti_tsc = pvti->tsc_timestamp; ++ ++ /* Make sure that the version double-check is last. */ ++ smp_rmb(); ++ } while (unlikely((version & 1) || version != pvti->version)); ++ ++ delta = tsc - pvti_tsc; ++ ret = pvti_system_time + ++ pvclock_scale_delta(delta, pvti_tsc_to_system_mul, ++ pvti_tsc_shift); + + /* refer to tsc.c read_tsc() comment for rationale */ + last = gtod->cycle_last; +diff --git a/arch/x86/entry/vdso/vdso-layout.lds.S b/arch/x86/entry/vdso/vdso-layout.lds.S +index de2c921025f5..4158acc17df0 100644 +--- a/arch/x86/entry/vdso/vdso-layout.lds.S ++++ b/arch/x86/entry/vdso/vdso-layout.lds.S +@@ -25,7 +25,7 @@ SECTIONS + * segment. + */ + +- vvar_start = . - 2 * PAGE_SIZE; ++ vvar_start = . - 3 * PAGE_SIZE; + vvar_page = vvar_start; + + /* Place all vvars at the offsets in asm/vvar.h. */ +@@ -36,6 +36,7 @@ SECTIONS + #undef EMIT_VVAR + + hpet_page = vvar_start + PAGE_SIZE; ++ pvclock_page = vvar_start + 2 * PAGE_SIZE; + + . = SIZEOF_HEADERS; + +diff --git a/arch/x86/entry/vdso/vdso2c.c b/arch/x86/entry/vdso/vdso2c.c +index 785d9922b106..491020b2826d 100644 +--- a/arch/x86/entry/vdso/vdso2c.c ++++ b/arch/x86/entry/vdso/vdso2c.c +@@ -73,6 +73,7 @@ enum { + sym_vvar_start, + sym_vvar_page, + sym_hpet_page, ++ sym_pvclock_page, + sym_VDSO_FAKE_SECTION_TABLE_START, + sym_VDSO_FAKE_SECTION_TABLE_END, + }; +@@ -80,6 +81,7 @@ enum { + const int special_pages[] = { + sym_vvar_page, + sym_hpet_page, ++ sym_pvclock_page, + }; + + struct vdso_sym { +@@ -91,6 +93,7 @@ struct vdso_sym required_syms[] = { + [sym_vvar_start] = {"vvar_start", true}, + [sym_vvar_page] = {"vvar_page", true}, + [sym_hpet_page] = {"hpet_page", true}, ++ [sym_pvclock_page] = {"pvclock_page", true}, + [sym_VDSO_FAKE_SECTION_TABLE_START] = { + "VDSO_FAKE_SECTION_TABLE_START", false + }, +diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c +index 64df47148160..aa828191c654 100644 +--- a/arch/x86/entry/vdso/vma.c ++++ b/arch/x86/entry/vdso/vma.c +@@ -100,6 +100,7 @@ static int map_vdso(const struct vdso_image *image, bool calculate_addr) + .name = "[vvar]", + .pages = no_pages, + }; ++ struct pvclock_vsyscall_time_info *pvti; + + if (calculate_addr) { + addr = vdso_addr(current->mm->start_stack, +@@ -169,6 +170,18 @@ static int map_vdso(const struct vdso_image *image, bool calculate_addr) + } + #endif + ++ pvti = pvclock_pvti_cpu0_va(); ++ if (pvti && image->sym_pvclock_page) { ++ ret = remap_pfn_range(vma, ++ text_start + image->sym_pvclock_page, ++ __pa(pvti) >> PAGE_SHIFT, ++ PAGE_SIZE, ++ PAGE_READONLY); ++ ++ if (ret) ++ goto up_fail; ++ } ++ + up_fail: + if (ret) + current->mm->context.vdso = NULL; +diff --git a/arch/x86/include/asm/cmdline.h b/arch/x86/include/asm/cmdline.h +index e01f7f7ccb0c..84ae170bc3d0 100644 +--- a/arch/x86/include/asm/cmdline.h ++++ b/arch/x86/include/asm/cmdline.h +@@ -2,5 +2,7 @@ + #define _ASM_X86_CMDLINE_H + + int cmdline_find_option_bool(const char *cmdline_ptr, const char *option); ++int cmdline_find_option(const char *cmdline_ptr, const char *option, ++ char *buffer, int bufsize); + + #endif /* _ASM_X86_CMDLINE_H */ +diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h +index f7ba9fbf12ee..f6605712ca90 100644 +--- a/arch/x86/include/asm/cpufeature.h ++++ b/arch/x86/include/asm/cpufeature.h +@@ -187,6 +187,7 @@ + #define X86_FEATURE_ARAT ( 7*32+ 1) /* Always Running APIC Timer */ + #define X86_FEATURE_CPB ( 7*32+ 2) /* AMD Core Performance Boost */ + #define X86_FEATURE_EPB ( 7*32+ 3) /* IA32_ENERGY_PERF_BIAS support */ ++#define X86_FEATURE_INVPCID_SINGLE ( 7*32+ 4) /* Effectively INVPCID && CR4.PCIDE=1 */ + #define X86_FEATURE_PLN ( 7*32+ 5) /* Intel Power Limit Notification */ + #define X86_FEATURE_PTS ( 7*32+ 6) /* Intel Package Thermal Status */ + #define X86_FEATURE_DTHERM ( 7*32+ 7) /* Digital Thermal Sensor */ +@@ -199,6 +200,9 @@ + #define X86_FEATURE_HWP_PKG_REQ ( 7*32+14) /* Intel HWP_PKG_REQ */ + #define X86_FEATURE_INTEL_PT ( 7*32+15) /* Intel Processor Trace */ + ++/* Because the ALTERNATIVE scheme is for members of the X86_FEATURE club... */ ++#define X86_FEATURE_KAISER ( 7*32+31) /* CONFIG_PAGE_TABLE_ISOLATION w/o nokaiser */ ++ + /* Virtualization flags: Linux defined, word 8 */ + #define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */ + #define X86_FEATURE_VNMI ( 8*32+ 1) /* Intel Virtual NMI */ +diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h +index 4e10d73cf018..880db91d9457 100644 +--- a/arch/x86/include/asm/desc.h ++++ b/arch/x86/include/asm/desc.h +@@ -43,7 +43,7 @@ struct gdt_page { + struct desc_struct gdt[GDT_ENTRIES]; + } __attribute__((aligned(PAGE_SIZE))); + +-DECLARE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page); ++DECLARE_PER_CPU_PAGE_ALIGNED_USER_MAPPED(struct gdt_page, gdt_page); + + static inline struct desc_struct *get_cpu_gdt_table(unsigned int cpu) + { +diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h +index 59caa55fb9b5..ee52ff858699 100644 +--- a/arch/x86/include/asm/hw_irq.h ++++ b/arch/x86/include/asm/hw_irq.h +@@ -187,7 +187,7 @@ extern char irq_entries_start[]; + #define VECTOR_RETRIGGERED ((void *)~0UL) + + typedef struct irq_desc* vector_irq_t[NR_VECTORS]; +-DECLARE_PER_CPU(vector_irq_t, vector_irq); ++DECLARE_PER_CPU_USER_MAPPED(vector_irq_t, vector_irq); + + #endif /* !ASSEMBLY_ */ + +diff --git a/arch/x86/include/asm/kaiser.h b/arch/x86/include/asm/kaiser.h +new file mode 100644 +index 000000000000..802bbbdfe143 +--- /dev/null ++++ b/arch/x86/include/asm/kaiser.h +@@ -0,0 +1,141 @@ ++#ifndef _ASM_X86_KAISER_H ++#define _ASM_X86_KAISER_H ++ ++#include <uapi/asm/processor-flags.h> /* For PCID constants */ ++ ++/* ++ * This file includes the definitions for the KAISER feature. ++ * KAISER is a counter measure against x86_64 side channel attacks on ++ * the kernel virtual memory. It has a shadow pgd for every process: the ++ * shadow pgd has a minimalistic kernel-set mapped, but includes the whole ++ * user memory. Within a kernel context switch, or when an interrupt is handled, ++ * the pgd is switched to the normal one. When the system switches to user mode, ++ * the shadow pgd is enabled. By this, the virtual memory caches are freed, ++ * and the user may not attack the whole kernel memory. ++ * ++ * A minimalistic kernel mapping holds the parts needed to be mapped in user ++ * mode, such as the entry/exit functions of the user space, or the stacks. ++ */ ++ ++#define KAISER_SHADOW_PGD_OFFSET 0x1000 ++ ++#ifdef __ASSEMBLY__ ++#ifdef CONFIG_PAGE_TABLE_ISOLATION ++ ++.macro _SWITCH_TO_KERNEL_CR3 reg ++movq %cr3, \reg ++andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), \reg ++/* If PCID enabled, set X86_CR3_PCID_NOFLUSH_BIT */ ++ALTERNATIVE "", "bts $63, \reg", X86_FEATURE_PCID ++movq \reg, %cr3 ++.endm ++ ++.macro _SWITCH_TO_USER_CR3 reg regb ++/* ++ * regb must be the low byte portion of reg: because we have arranged ++ * for the low byte of the user PCID to serve as the high byte of NOFLUSH ++ * (0x80 for each when PCID is enabled, or 0x00 when PCID and NOFLUSH are ++ * not enabled): so that the one register can update both memory and cr3. ++ */ ++movq %cr3, \reg ++orq PER_CPU_VAR(x86_cr3_pcid_user), \reg ++js 9f ++/* If PCID enabled, FLUSH this time, reset to NOFLUSH for next time */ ++movb \regb, PER_CPU_VAR(x86_cr3_pcid_user+7) ++9: ++movq \reg, %cr3 ++.endm ++ ++.macro SWITCH_KERNEL_CR3 ++ALTERNATIVE "jmp 8f", "pushq %rax", X86_FEATURE_KAISER ++_SWITCH_TO_KERNEL_CR3 %rax ++popq %rax ++8: ++.endm ++ ++.macro SWITCH_USER_CR3 ++ALTERNATIVE "jmp 8f", "pushq %rax", X86_FEATURE_KAISER ++_SWITCH_TO_USER_CR3 %rax %al ++popq %rax ++8: ++.endm ++ ++.macro SWITCH_KERNEL_CR3_NO_STACK ++ALTERNATIVE "jmp 8f", \ ++ __stringify(movq %rax, PER_CPU_VAR(unsafe_stack_register_backup)), \ ++ X86_FEATURE_KAISER ++_SWITCH_TO_KERNEL_CR3 %rax ++movq PER_CPU_VAR(unsafe_stack_register_backup), %rax ++8: ++.endm ++ ++#else /* CONFIG_PAGE_TABLE_ISOLATION */ ++ ++.macro SWITCH_KERNEL_CR3 ++.endm ++.macro SWITCH_USER_CR3 ++.endm ++.macro SWITCH_KERNEL_CR3_NO_STACK ++.endm ++ ++#endif /* CONFIG_PAGE_TABLE_ISOLATION */ ++ ++#else /* __ASSEMBLY__ */ ++ ++#ifdef CONFIG_PAGE_TABLE_ISOLATION ++/* ++ * Upon kernel/user mode switch, it may happen that the address ++ * space has to be switched before the registers have been ++ * stored. To change the address space, another register is ++ * needed. A register therefore has to be stored/restored. ++*/ ++DECLARE_PER_CPU_USER_MAPPED(unsigned long, unsafe_stack_register_backup); ++ ++DECLARE_PER_CPU(unsigned long, x86_cr3_pcid_user); ++ ++extern char __per_cpu_user_mapped_start[], __per_cpu_user_mapped_end[]; ++ ++extern int kaiser_enabled; ++extern void __init kaiser_check_boottime_disable(void); ++#else ++#define kaiser_enabled 0 ++static inline void __init kaiser_check_boottime_disable(void) {} ++#endif /* CONFIG_PAGE_TABLE_ISOLATION */ ++ ++/* ++ * Kaiser function prototypes are needed even when CONFIG_PAGE_TABLE_ISOLATION is not set, ++ * so as to build with tests on kaiser_enabled instead of #ifdefs. ++ */ ++ ++/** ++ * kaiser_add_mapping - map a virtual memory part to the shadow (user) mapping ++ * @addr: the start address of the range ++ * @size: the size of the range ++ * @flags: The mapping flags of the pages ++ * ++ * The mapping is done on a global scope, so no bigger ++ * synchronization has to be done. the pages have to be ++ * manually unmapped again when they are not needed any longer. ++ */ ++extern int kaiser_add_mapping(unsigned long addr, unsigned long size, unsigned long flags); ++ ++/** ++ * kaiser_remove_mapping - unmap a virtual memory part of the shadow mapping ++ * @addr: the start address of the range ++ * @size: the size of the range ++ */ ++extern void kaiser_remove_mapping(unsigned long start, unsigned long size); ++ ++/** ++ * kaiser_init - Initialize the shadow mapping ++ * ++ * Most parts of the shadow mapping can be mapped upon boot ++ * time. Only per-process things like the thread stacks ++ * or a new LDT have to be mapped at runtime. These boot- ++ * time mappings are permanent and never unmapped. ++ */ ++extern void kaiser_init(void); ++ ++#endif /* __ASSEMBLY */ ++ ++#endif /* _ASM_X86_KAISER_H */ +diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h +index 6ec0c8b2e9df..84c62d950023 100644 +--- a/arch/x86/include/asm/pgtable.h ++++ b/arch/x86/include/asm/pgtable.h +@@ -18,6 +18,12 @@ + #ifndef __ASSEMBLY__ + #include <asm/x86_init.h> + ++#ifdef CONFIG_PAGE_TABLE_ISOLATION ++extern int kaiser_enabled; ++#else ++#define kaiser_enabled 0 ++#endif ++ + void ptdump_walk_pgd_level(struct seq_file *m, pgd_t *pgd); + void ptdump_walk_pgd_level_checkwx(void); + +@@ -653,7 +659,17 @@ static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address) + + static inline int pgd_bad(pgd_t pgd) + { +- return (pgd_flags(pgd) & ~_PAGE_USER) != _KERNPG_TABLE; ++ pgdval_t ignore_flags = _PAGE_USER; ++ /* ++ * We set NX on KAISER pgds that map userspace memory so ++ * that userspace can not meaningfully use the kernel ++ * page table by accident; it will fault on the first ++ * instruction it tries to run. See native_set_pgd(). ++ */ ++ if (kaiser_enabled) ++ ignore_flags |= _PAGE_NX; ++ ++ return (pgd_flags(pgd) & ~ignore_flags) != _KERNPG_TABLE; + } + + static inline int pgd_none(pgd_t pgd) +@@ -855,7 +871,15 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, + */ + static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count) + { +- memcpy(dst, src, count * sizeof(pgd_t)); ++ memcpy(dst, src, count * sizeof(pgd_t)); ++#ifdef CONFIG_PAGE_TABLE_ISOLATION ++ if (kaiser_enabled) { ++ /* Clone the shadow pgd part as well */ ++ memcpy(native_get_shadow_pgd(dst), ++ native_get_shadow_pgd(src), ++ count * sizeof(pgd_t)); ++ } ++#endif + } + + #define PTE_SHIFT ilog2(PTRS_PER_PTE) +diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h +index 2ee781114d34..c810226e741a 100644 +--- a/arch/x86/include/asm/pgtable_64.h ++++ b/arch/x86/include/asm/pgtable_64.h +@@ -106,9 +106,32 @@ static inline void native_pud_clear(pud_t *pud) + native_set_pud(pud, native_make_pud(0)); + } + ++#ifdef CONFIG_PAGE_TABLE_ISOLATION ++extern pgd_t kaiser_set_shadow_pgd(pgd_t *pgdp, pgd_t pgd); ++ ++static inline pgd_t *native_get_shadow_pgd(pgd_t *pgdp) ++{ ++#ifdef CONFIG_DEBUG_VM ++ /* linux/mmdebug.h may not have been included at this point */ ++ BUG_ON(!kaiser_enabled); ++#endif ++ return (pgd_t *)((unsigned long)pgdp | (unsigned long)PAGE_SIZE); ++} ++#else ++static inline pgd_t kaiser_set_shadow_pgd(pgd_t *pgdp, pgd_t pgd) ++{ ++ return pgd; ++} ++static inline pgd_t *native_get_shadow_pgd(pgd_t *pgdp) ++{ ++ BUILD_BUG_ON(1); ++ return NULL; ++} ++#endif /* CONFIG_PAGE_TABLE_ISOLATION */ ++ + static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd) + { +- *pgdp = pgd; ++ *pgdp = kaiser_set_shadow_pgd(pgdp, pgd); + } + + static inline void native_pgd_clear(pgd_t *pgd) +diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h +index 79c91853e50e..8dba273da25a 100644 +--- a/arch/x86/include/asm/pgtable_types.h ++++ b/arch/x86/include/asm/pgtable_types.h +@@ -89,7 +89,7 @@ + #define _PAGE_NX (_AT(pteval_t, 0)) + #endif + +-#define _PAGE_PROTNONE (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE) ++#define _PAGE_PROTNONE (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE) + + #define _PAGE_TABLE (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | \ + _PAGE_ACCESSED | _PAGE_DIRTY) +@@ -102,6 +102,33 @@ + _PAGE_SOFT_DIRTY) + #define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE) + ++/* The ASID is the lower 12 bits of CR3 */ ++#define X86_CR3_PCID_ASID_MASK (_AC((1<<12)-1,UL)) ++ ++/* Mask for all the PCID-related bits in CR3: */ ++#define X86_CR3_PCID_MASK (X86_CR3_PCID_NOFLUSH | X86_CR3_PCID_ASID_MASK) ++#define X86_CR3_PCID_ASID_KERN (_AC(0x0,UL)) ++ ++#if defined(CONFIG_PAGE_TABLE_ISOLATION) && defined(CONFIG_X86_64) ++/* Let X86_CR3_PCID_ASID_USER be usable for the X86_CR3_PCID_NOFLUSH bit */ ++#define X86_CR3_PCID_ASID_USER (_AC(0x80,UL)) ++ ++#define X86_CR3_PCID_KERN_FLUSH (X86_CR3_PCID_ASID_KERN) ++#define X86_CR3_PCID_USER_FLUSH (X86_CR3_PCID_ASID_USER) ++#define X86_CR3_PCID_KERN_NOFLUSH (X86_CR3_PCID_NOFLUSH | X86_CR3_PCID_ASID_KERN) ++#define X86_CR3_PCID_USER_NOFLUSH (X86_CR3_PCID_NOFLUSH | X86_CR3_PCID_ASID_USER) ++#else ++#define X86_CR3_PCID_ASID_USER (_AC(0x0,UL)) ++/* ++ * PCIDs are unsupported on 32-bit and none of these bits can be ++ * set in CR3: ++ */ ++#define X86_CR3_PCID_KERN_FLUSH (0) ++#define X86_CR3_PCID_USER_FLUSH (0) ++#define X86_CR3_PCID_KERN_NOFLUSH (0) ++#define X86_CR3_PCID_USER_NOFLUSH (0) ++#endif ++ + /* + * The cache modes defined here are used to translate between pure SW usage + * and the HW defined cache mode bits and/or PAT entries. +diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h +index 2d5a50cb61a2..f3bdaed0188f 100644 +--- a/arch/x86/include/asm/processor.h ++++ b/arch/x86/include/asm/processor.h +@@ -305,7 +305,7 @@ struct tss_struct { + + } ____cacheline_aligned; + +-DECLARE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss); ++DECLARE_PER_CPU_SHARED_ALIGNED_USER_MAPPED(struct tss_struct, cpu_tss); + + #ifdef CONFIG_X86_32 + DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack); +diff --git a/arch/x86/include/asm/pvclock.h b/arch/x86/include/asm/pvclock.h +index baad72e4c100..6045cef376c2 100644 +--- a/arch/x86/include/asm/pvclock.h ++++ b/arch/x86/include/asm/pvclock.h +@@ -4,6 +4,15 @@ + #include <linux/clocksource.h> + #include <asm/pvclock-abi.h> + ++#ifdef CONFIG_PARAVIRT_CLOCK ++extern struct pvclock_vsyscall_time_info *pvclock_pvti_cpu0_va(void); ++#else ++static inline struct pvclock_vsyscall_time_info *pvclock_pvti_cpu0_va(void) ++{ ++ return NULL; ++} ++#endif ++ + /* some helper functions for xen and kvm pv clock sources */ + cycle_t pvclock_clocksource_read(struct pvclock_vcpu_time_info *src); + u8 pvclock_read_flags(struct pvclock_vcpu_time_info *src); +diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h +index 9fc5968da820..a691b66cc40a 100644 +--- a/arch/x86/include/asm/tlbflush.h ++++ b/arch/x86/include/asm/tlbflush.h +@@ -131,6 +131,24 @@ static inline void cr4_set_bits_and_update_boot(unsigned long mask) + cr4_set_bits(mask); + } + ++/* ++ * Declare a couple of kaiser interfaces here for convenience, ++ * to avoid the need for asm/kaiser.h in unexpected places. ++ */ ++#ifdef CONFIG_PAGE_TABLE_ISOLATION ++extern int kaiser_enabled; ++extern void kaiser_setup_pcid(void); ++extern void kaiser_flush_tlb_on_return_to_user(void); ++#else ++#define kaiser_enabled 0 ++static inline void kaiser_setup_pcid(void) ++{ ++} ++static inline void kaiser_flush_tlb_on_return_to_user(void) ++{ ++} ++#endif ++ + static inline void __native_flush_tlb(void) + { + /* +@@ -139,6 +157,8 @@ static inline void __native_flush_tlb(void) + * back: + */ + preempt_disable(); ++ if (kaiser_enabled) ++ kaiser_flush_tlb_on_return_to_user(); + native_write_cr3(native_read_cr3()); + preempt_enable(); + } +@@ -148,20 +168,27 @@ static inline void __native_flush_tlb_global_irq_disabled(void) + unsigned long cr4; + + cr4 = this_cpu_read(cpu_tlbstate.cr4); +- /* clear PGE */ +- native_write_cr4(cr4 & ~X86_CR4_PGE); +- /* write old PGE again and flush TLBs */ +- native_write_cr4(cr4); ++ if (cr4 & X86_CR4_PGE) { ++ /* clear PGE and flush TLB of all entries */ ++ native_write_cr4(cr4 & ~X86_CR4_PGE); ++ /* restore PGE as it was before */ ++ native_write_cr4(cr4); ++ } else { ++ /* do it with cr3, letting kaiser flush user PCID */ ++ __native_flush_tlb(); ++ } + } + + static inline void __native_flush_tlb_global(void) + { + unsigned long flags; + +- if (static_cpu_has(X86_FEATURE_INVPCID)) { ++ if (this_cpu_has(X86_FEATURE_INVPCID)) { + /* + * Using INVPCID is considerably faster than a pair of writes + * to CR4 sandwiched inside an IRQ flag save/restore. ++ * ++ * Note, this works with CR4.PCIDE=0 or 1. + */ + invpcid_flush_all(); + return; +@@ -173,24 +200,45 @@ static inline void __native_flush_tlb_global(void) + * be called from deep inside debugging code.) + */ + raw_local_irq_save(flags); +- + __native_flush_tlb_global_irq_disabled(); +- + raw_local_irq_restore(flags); + } + + static inline void __native_flush_tlb_single(unsigned long addr) + { +- asm volatile("invlpg (%0)" ::"r" (addr) : "memory"); ++ /* ++ * SIMICS #GP's if you run INVPCID with type 2/3 ++ * and X86_CR4_PCIDE clear. Shame! ++ * ++ * The ASIDs used below are hard-coded. But, we must not ++ * call invpcid(type=1/2) before CR4.PCIDE=1. Just call ++ * invlpg in the case we are called early. ++ */ ++ ++ if (!this_cpu_has(X86_FEATURE_INVPCID_SINGLE)) { ++ if (kaiser_enabled) ++ kaiser_flush_tlb_on_return_to_user(); ++ asm volatile("invlpg (%0)" ::"r" (addr) : "memory"); ++ return; ++ } ++ /* Flush the address out of both PCIDs. */ ++ /* ++ * An optimization here might be to determine addresses ++ * that are only kernel-mapped and only flush the kernel ++ * ASID. But, userspace flushes are probably much more ++ * important performance-wise. ++ * ++ * Make sure to do only a single invpcid when KAISER is ++ * disabled and we have only a single ASID. ++ */ ++ if (kaiser_enabled) ++ invpcid_flush_one(X86_CR3_PCID_ASID_USER, addr); ++ invpcid_flush_one(X86_CR3_PCID_ASID_KERN, addr); + } + + static inline void __flush_tlb_all(void) + { +- if (cpu_has_pge) +- __flush_tlb_global(); +- else +- __flush_tlb(); +- ++ __flush_tlb_global(); + /* + * Note: if we somehow had PCID but not PGE, then this wouldn't work -- + * we'd end up flushing kernel translations for the current ASID but +diff --git a/arch/x86/include/asm/vdso.h b/arch/x86/include/asm/vdso.h +index 756de9190aec..deabaf9759b6 100644 +--- a/arch/x86/include/asm/vdso.h ++++ b/arch/x86/include/asm/vdso.h +@@ -22,6 +22,7 @@ struct vdso_image { + + long sym_vvar_page; + long sym_hpet_page; ++ long sym_pvclock_page; + long sym_VDSO32_NOTE_MASK; + long sym___kernel_sigreturn; + long sym___kernel_rt_sigreturn; +diff --git a/arch/x86/include/uapi/asm/processor-flags.h b/arch/x86/include/uapi/asm/processor-flags.h +index 79887abcb5e1..1361779f44fe 100644 +--- a/arch/x86/include/uapi/asm/processor-flags.h ++++ b/arch/x86/include/uapi/asm/processor-flags.h +@@ -77,7 +77,8 @@ + #define X86_CR3_PWT _BITUL(X86_CR3_PWT_BIT) + #define X86_CR3_PCD_BIT 4 /* Page Cache Disable */ + #define X86_CR3_PCD _BITUL(X86_CR3_PCD_BIT) +-#define X86_CR3_PCID_MASK _AC(0x00000fff,UL) /* PCID Mask */ ++#define X86_CR3_PCID_NOFLUSH_BIT 63 /* Preserve old PCID */ ++#define X86_CR3_PCID_NOFLUSH _BITULL(X86_CR3_PCID_NOFLUSH_BIT) + + /* + * Intel CPU features in CR4 +diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c +index aa1e7246b06b..cc154ac64f00 100644 +--- a/arch/x86/kernel/cpu/common.c ++++ b/arch/x86/kernel/cpu/common.c +@@ -92,7 +92,7 @@ static const struct cpu_dev default_cpu = { + + static const struct cpu_dev *this_cpu = &default_cpu; + +-DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = { ++DEFINE_PER_CPU_PAGE_ALIGNED_USER_MAPPED(struct gdt_page, gdt_page) = { .gdt = { + #ifdef CONFIG_X86_64 + /* + * We need valid kernel segments for data and code in long mode too +@@ -324,8 +324,21 @@ static __always_inline void setup_smap(struct cpuinfo_x86 *c) + static void setup_pcid(struct cpuinfo_x86 *c) + { + if (cpu_has(c, X86_FEATURE_PCID)) { +- if (cpu_has(c, X86_FEATURE_PGE)) { ++ if (cpu_has(c, X86_FEATURE_PGE) || kaiser_enabled) { + cr4_set_bits(X86_CR4_PCIDE); ++ /* ++ * INVPCID has two "groups" of types: ++ * 1/2: Invalidate an individual address ++ * 3/4: Invalidate all contexts ++ * ++ * 1/2 take a PCID, but 3/4 do not. So, 3/4 ++ * ignore the PCID argument in the descriptor. ++ * But, we have to be careful not to call 1/2 ++ * with an actual non-zero PCID in them before ++ * we do the above cr4_set_bits(). ++ */ ++ if (cpu_has(c, X86_FEATURE_INVPCID)) ++ set_cpu_cap(c, X86_FEATURE_INVPCID_SINGLE); + } else { + /* + * flush_tlb_all(), as currently implemented, won't +@@ -338,6 +351,7 @@ static void setup_pcid(struct cpuinfo_x86 *c) + clear_cpu_cap(c, X86_FEATURE_PCID); + } + } ++ kaiser_setup_pcid(); + } + + /* +@@ -1229,7 +1243,7 @@ static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = { + [DEBUG_STACK - 1] = DEBUG_STKSZ + }; + +-static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks ++DEFINE_PER_CPU_PAGE_ALIGNED_USER_MAPPED(char, exception_stacks + [(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]); + + /* May not be marked __init: used by software suspend */ +@@ -1392,6 +1406,14 @@ void cpu_init(void) + * try to read it. + */ + cr4_init_shadow(); ++ if (!kaiser_enabled) { ++ /* ++ * secondary_startup_64() deferred setting PGE in cr4: ++ * probe_page_size_mask() sets it on the boot cpu, ++ * but it needs to be set on each secondary cpu. ++ */ ++ cr4_set_bits(X86_CR4_PGE); ++ } + + /* + * Load microcode on this cpu if a valid microcode is available. +diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c +index 1e7de3cefc9c..f01b3a12dce0 100644 +--- a/arch/x86/kernel/cpu/perf_event_intel_ds.c ++++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c +@@ -2,11 +2,15 @@ + #include <linux/types.h> + #include <linux/slab.h> + ++#include <asm/kaiser.h> + #include <asm/perf_event.h> + #include <asm/insn.h> + + #include "perf_event.h" + ++static ++DEFINE_PER_CPU_SHARED_ALIGNED_USER_MAPPED(struct debug_store, cpu_debug_store); ++ + /* The size of a BTS record in bytes: */ + #define BTS_RECORD_SIZE 24 + +@@ -268,6 +272,39 @@ void fini_debug_store_on_cpu(int cpu) + + static DEFINE_PER_CPU(void *, insn_buffer); + ++static void *dsalloc(size_t size, gfp_t flags, int node) ++{ ++#ifdef CONFIG_PAGE_TABLE_ISOLATION ++ unsigned int order = get_order(size); ++ struct page *page; ++ unsigned long addr; ++ ++ page = __alloc_pages_node(node, flags | __GFP_ZERO, order); ++ if (!page) ++ return NULL; ++ addr = (unsigned long)page_address(page); ++ if (kaiser_add_mapping(addr, size, __PAGE_KERNEL) < 0) { ++ __free_pages(page, order); ++ addr = 0; ++ } ++ return (void *)addr; ++#else ++ return kmalloc_node(size, flags | __GFP_ZERO, node); ++#endif ++} ++ ++static void dsfree(const void *buffer, size_t size) ++{ ++#ifdef CONFIG_PAGE_TABLE_ISOLATION ++ if (!buffer) ++ return; ++ kaiser_remove_mapping((unsigned long)buffer, size); ++ free_pages((unsigned long)buffer, get_order(size)); ++#else ++ kfree(buffer); ++#endif ++} ++ + static int alloc_pebs_buffer(int cpu) + { + struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds; +@@ -278,7 +315,7 @@ static int alloc_pebs_buffer(int cpu) + if (!x86_pmu.pebs) + return 0; + +- buffer = kzalloc_node(x86_pmu.pebs_buffer_size, GFP_KERNEL, node); ++ buffer = dsalloc(x86_pmu.pebs_buffer_size, GFP_KERNEL, node); + if (unlikely(!buffer)) + return -ENOMEM; + +@@ -289,7 +326,7 @@ static int alloc_pebs_buffer(int cpu) + if (x86_pmu.intel_cap.pebs_format < 2) { + ibuffer = kzalloc_node(PEBS_FIXUP_SIZE, GFP_KERNEL, node); + if (!ibuffer) { +- kfree(buffer); ++ dsfree(buffer, x86_pmu.pebs_buffer_size); + return -ENOMEM; + } + per_cpu(insn_buffer, cpu) = ibuffer; +@@ -315,7 +352,8 @@ static void release_pebs_buffer(int cpu) + kfree(per_cpu(insn_buffer, cpu)); + per_cpu(insn_buffer, cpu) = NULL; + +- kfree((void *)(unsigned long)ds->pebs_buffer_base); ++ dsfree((void *)(unsigned long)ds->pebs_buffer_base, ++ x86_pmu.pebs_buffer_size); + ds->pebs_buffer_base = 0; + } + +@@ -329,7 +367,7 @@ static int alloc_bts_buffer(int cpu) + if (!x86_pmu.bts) + return 0; + +- buffer = kzalloc_node(BTS_BUFFER_SIZE, GFP_KERNEL | __GFP_NOWARN, node); ++ buffer = dsalloc(BTS_BUFFER_SIZE, GFP_KERNEL | __GFP_NOWARN, node); + if (unlikely(!buffer)) { + WARN_ONCE(1, "%s: BTS buffer allocation failure\n", __func__); + return -ENOMEM; +@@ -355,19 +393,15 @@ static void release_bts_buffer(int cpu) + if (!ds || !x86_pmu.bts) + return; + +- kfree((void *)(unsigned long)ds->bts_buffer_base); ++ dsfree((void *)(unsigned long)ds->bts_buffer_base, BTS_BUFFER_SIZE); + ds->bts_buffer_base = 0; + } + + static int alloc_ds_buffer(int cpu) + { +- int node = cpu_to_node(cpu); +- struct debug_store *ds; +- +- ds = kzalloc_node(sizeof(*ds), GFP_KERNEL, node); +- if (unlikely(!ds)) +- return -ENOMEM; ++ struct debug_store *ds = per_cpu_ptr(&cpu_debug_store, cpu); + ++ memset(ds, 0, sizeof(*ds)); + per_cpu(cpu_hw_events, cpu).ds = ds; + + return 0; +@@ -381,7 +415,6 @@ static void release_ds_buffer(int cpu) + return; + + per_cpu(cpu_hw_events, cpu).ds = NULL; +- kfree(ds); + } + + void release_ds_buffers(void) +diff --git a/arch/x86/kernel/espfix_64.c b/arch/x86/kernel/espfix_64.c +index 4d38416e2a7f..b02cb2ec6726 100644 +--- a/arch/x86/kernel/espfix_64.c ++++ b/arch/x86/kernel/espfix_64.c +@@ -41,6 +41,7 @@ + #include <asm/pgalloc.h> + #include <asm/setup.h> + #include <asm/espfix.h> ++#include <asm/kaiser.h> + + /* + * Note: we only need 6*8 = 48 bytes for the espfix stack, but round +@@ -126,6 +127,15 @@ void __init init_espfix_bsp(void) + /* Install the espfix pud into the kernel page directory */ + pgd_p = &init_level4_pgt[pgd_index(ESPFIX_BASE_ADDR)]; + pgd_populate(&init_mm, pgd_p, (pud_t *)espfix_pud_page); ++ /* ++ * Just copy the top-level PGD that is mapping the espfix ++ * area to ensure it is mapped into the shadow user page ++ * tables. ++ */ ++ if (kaiser_enabled) { ++ set_pgd(native_get_shadow_pgd(pgd_p), ++ __pgd(_KERNPG_TABLE | __pa((pud_t *)espfix_pud_page))); ++ } + + /* Randomize the locations */ + init_espfix_random(); +diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S +index ffdc0e860390..4034e905741a 100644 +--- a/arch/x86/kernel/head_64.S ++++ b/arch/x86/kernel/head_64.S +@@ -183,8 +183,8 @@ ENTRY(secondary_startup_64) + movq $(init_level4_pgt - __START_KERNEL_map), %rax + 1: + +- /* Enable PAE mode and PGE */ +- movl $(X86_CR4_PAE | X86_CR4_PGE), %ecx ++ /* Enable PAE and PSE, but defer PGE until kaiser_enabled is decided */ ++ movl $(X86_CR4_PAE | X86_CR4_PSE), %ecx + movq %rcx, %cr4 + + /* Setup early boot stage 4 level pagetables. */ +@@ -441,6 +441,27 @@ early_idt_ripmsg: + .balign PAGE_SIZE; \ + GLOBAL(name) + ++#ifdef CONFIG_PAGE_TABLE_ISOLATION ++/* ++ * Each PGD needs to be 8k long and 8k aligned. We do not ++ * ever go out to userspace with these, so we do not ++ * strictly *need* the second page, but this allows us to ++ * have a single set_pgd() implementation that does not ++ * need to worry about whether it has 4k or 8k to work ++ * with. ++ * ++ * This ensures PGDs are 8k long: ++ */ ++#define KAISER_USER_PGD_FILL 512 ++/* This ensures they are 8k-aligned: */ ++#define NEXT_PGD_PAGE(name) \ ++ .balign 2 * PAGE_SIZE; \ ++GLOBAL(name) ++#else ++#define NEXT_PGD_PAGE(name) NEXT_PAGE(name) ++#define KAISER_USER_PGD_FILL 0 ++#endif ++ + /* Automate the creation of 1 to 1 mapping pmd entries */ + #define PMDS(START, PERM, COUNT) \ + i = 0 ; \ +@@ -450,9 +471,10 @@ GLOBAL(name) + .endr + + __INITDATA +-NEXT_PAGE(early_level4_pgt) ++NEXT_PGD_PAGE(early_level4_pgt) + .fill 511,8,0 + .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE ++ .fill KAISER_USER_PGD_FILL,8,0 + + NEXT_PAGE(early_dynamic_pgts) + .fill 512*EARLY_DYNAMIC_PAGE_TABLES,8,0 +@@ -460,16 +482,18 @@ NEXT_PAGE(early_dynamic_pgts) + .data + + #ifndef CONFIG_XEN +-NEXT_PAGE(init_level4_pgt) ++NEXT_PGD_PAGE(init_level4_pgt) + .fill 512,8,0 ++ .fill KAISER_USER_PGD_FILL,8,0 + #else +-NEXT_PAGE(init_level4_pgt) ++NEXT_PGD_PAGE(init_level4_pgt) + .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE + .org init_level4_pgt + L4_PAGE_OFFSET*8, 0 + .quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE + .org init_level4_pgt + L4_START_KERNEL*8, 0 + /* (2^48-(2*1024*1024*1024))/(2^39) = 511 */ + .quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE ++ .fill KAISER_USER_PGD_FILL,8,0 + + NEXT_PAGE(level3_ident_pgt) + .quad level2_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE +@@ -480,6 +504,7 @@ NEXT_PAGE(level2_ident_pgt) + */ + PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, PTRS_PER_PMD) + #endif ++ .fill KAISER_USER_PGD_FILL,8,0 + + NEXT_PAGE(level3_kernel_pgt) + .fill L3_START_KERNEL,8,0 +diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c +index 1423ab1b0312..f480b38a03c3 100644 +--- a/arch/x86/kernel/irqinit.c ++++ b/arch/x86/kernel/irqinit.c +@@ -51,7 +51,7 @@ static struct irqaction irq2 = { + .flags = IRQF_NO_THREAD, + }; + +-DEFINE_PER_CPU(vector_irq_t, vector_irq) = { ++DEFINE_PER_CPU_USER_MAPPED(vector_irq_t, vector_irq) = { + [0 ... NR_VECTORS - 1] = VECTOR_UNUSED, + }; + +diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c +index 2bd81e302427..ec1b06dc82d2 100644 +--- a/arch/x86/kernel/kvmclock.c ++++ b/arch/x86/kernel/kvmclock.c +@@ -45,6 +45,11 @@ early_param("no-kvmclock", parse_no_kvmclock); + static struct pvclock_vsyscall_time_info *hv_clock; + static struct pvclock_wall_clock wall_clock; + ++struct pvclock_vsyscall_time_info *pvclock_pvti_cpu0_va(void) ++{ ++ return hv_clock; ++} ++ + /* + * The wallclock is the time of day when we booted. Since then, some time may + * have elapsed since the hypervisor wrote the data. So we try to account for +diff --git a/arch/x86/kernel/ldt.c b/arch/x86/kernel/ldt.c +index d6279593bcdd..bc429365b72a 100644 +--- a/arch/x86/kernel/ldt.c ++++ b/arch/x86/kernel/ldt.c +@@ -16,6 +16,7 @@ + #include <linux/slab.h> + #include <linux/vmalloc.h> + #include <linux/uaccess.h> ++#include <linux/kaiser.h> + + #include <asm/ldt.h> + #include <asm/desc.h> +@@ -34,11 +35,21 @@ static void flush_ldt(void *current_mm) + set_ldt(pc->ldt->entries, pc->ldt->size); + } + ++static void __free_ldt_struct(struct ldt_struct *ldt) ++{ ++ if (ldt->size * LDT_ENTRY_SIZE > PAGE_SIZE) ++ vfree(ldt->entries); ++ else ++ free_page((unsigned long)ldt->entries); ++ kfree(ldt); ++} ++ + /* The caller must call finalize_ldt_struct on the result. LDT starts zeroed. */ + static struct ldt_struct *alloc_ldt_struct(int size) + { + struct ldt_struct *new_ldt; + int alloc_size; ++ int ret; + + if (size > LDT_ENTRIES) + return NULL; +@@ -66,7 +77,13 @@ static struct ldt_struct *alloc_ldt_struct(int size) + return NULL; + } + ++ ret = kaiser_add_mapping((unsigned long)new_ldt->entries, alloc_size, ++ __PAGE_KERNEL); + new_ldt->size = size; ++ if (ret) { ++ __free_ldt_struct(new_ldt); ++ return NULL; ++ } + return new_ldt; + } + +@@ -92,12 +109,10 @@ static void free_ldt_struct(struct ldt_struct *ldt) + if (likely(!ldt)) + return; + ++ kaiser_remove_mapping((unsigned long)ldt->entries, ++ ldt->size * LDT_ENTRY_SIZE); + paravirt_free_ldt(ldt->entries, ldt->size); +- if (ldt->size * LDT_ENTRY_SIZE > PAGE_SIZE) +- vfree(ldt->entries); +- else +- free_page((unsigned long)ldt->entries); +- kfree(ldt); ++ __free_ldt_struct(ldt); + } + + /* +diff --git a/arch/x86/kernel/paravirt_patch_64.c b/arch/x86/kernel/paravirt_patch_64.c +index 8aa05583bc42..0677bf8d3a42 100644 +--- a/arch/x86/kernel/paravirt_patch_64.c ++++ b/arch/x86/kernel/paravirt_patch_64.c +@@ -9,7 +9,6 @@ DEF_NATIVE(pv_irq_ops, save_fl, "pushfq; popq %rax"); + DEF_NATIVE(pv_mmu_ops, read_cr2, "movq %cr2, %rax"); + DEF_NATIVE(pv_mmu_ops, read_cr3, "movq %cr3, %rax"); + DEF_NATIVE(pv_mmu_ops, write_cr3, "movq %rdi, %cr3"); +-DEF_NATIVE(pv_mmu_ops, flush_tlb_single, "invlpg (%rdi)"); + DEF_NATIVE(pv_cpu_ops, clts, "clts"); + DEF_NATIVE(pv_cpu_ops, wbinvd, "wbinvd"); + +@@ -62,7 +61,6 @@ unsigned native_patch(u8 type, u16 clobbers, void *ibuf, + PATCH_SITE(pv_mmu_ops, read_cr3); + PATCH_SITE(pv_mmu_ops, write_cr3); + PATCH_SITE(pv_cpu_ops, clts); +- PATCH_SITE(pv_mmu_ops, flush_tlb_single); + PATCH_SITE(pv_cpu_ops, wbinvd); + #if defined(CONFIG_PARAVIRT_SPINLOCKS) && defined(CONFIG_QUEUED_SPINLOCKS) + case PARAVIRT_PATCH(pv_lock_ops.queued_spin_unlock): +diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c +index 9f7c21c22477..7c5c5dc90ffa 100644 +--- a/arch/x86/kernel/process.c ++++ b/arch/x86/kernel/process.c +@@ -39,7 +39,7 @@ + * section. Since TSS's are completely CPU-local, we want them + * on exact cacheline boundaries, to eliminate cacheline ping-pong. + */ +-__visible DEFINE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss) = { ++__visible DEFINE_PER_CPU_SHARED_ALIGNED_USER_MAPPED(struct tss_struct, cpu_tss) = { + .x86_tss = { + .sp0 = TOP_OF_INIT_STACK, + #ifdef CONFIG_X86_32 +diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c +index e67b834279b2..bbaae4cf9e8e 100644 +--- a/arch/x86/kernel/setup.c ++++ b/arch/x86/kernel/setup.c +@@ -112,6 +112,7 @@ + #include <asm/alternative.h> + #include <asm/prom.h> + #include <asm/microcode.h> ++#include <asm/kaiser.h> + + /* + * max_low_pfn_mapped: highest direct mapped pfn under 4GB +@@ -1016,6 +1017,12 @@ void __init setup_arch(char **cmdline_p) + */ + init_hypervisor_platform(); + ++ /* ++ * This needs to happen right after XENPV is set on xen and ++ * kaiser_enabled is checked below in cleanup_highmap(). ++ */ ++ kaiser_check_boottime_disable(); ++ + x86_init.resources.probe_roms(); + + /* after parse_early_param, so could debug it */ +diff --git a/arch/x86/kernel/tracepoint.c b/arch/x86/kernel/tracepoint.c +index 1c113db9ed57..2bb5ee464df3 100644 +--- a/arch/x86/kernel/tracepoint.c ++++ b/arch/x86/kernel/tracepoint.c +@@ -9,10 +9,12 @@ + #include <linux/atomic.h> + + atomic_t trace_idt_ctr = ATOMIC_INIT(0); ++__aligned(PAGE_SIZE) + struct desc_ptr trace_idt_descr = { NR_VECTORS * 16 - 1, + (unsigned long) trace_idt_table }; + + /* No need to be aligned, but done to keep all IDTs defined the same way. */ ++__aligned(PAGE_SIZE) + gate_desc trace_idt_table[NR_VECTORS] __page_aligned_bss; + + static int trace_irq_vector_refcount; +diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c +index 796f1ec67469..ccf17dbfea09 100644 +--- a/arch/x86/kvm/x86.c ++++ b/arch/x86/kvm/x86.c +@@ -759,7 +759,8 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) + return 1; + + /* PCID can not be enabled when cr3[11:0]!=000H or EFER.LMA=0 */ +- if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_MASK) || !is_long_mode(vcpu)) ++ if ((kvm_read_cr3(vcpu) & X86_CR3_PCID_ASID_MASK) || ++ !is_long_mode(vcpu)) + return 1; + } + +diff --git a/arch/x86/lib/cmdline.c b/arch/x86/lib/cmdline.c +index 422db000d727..a744506856b1 100644 +--- a/arch/x86/lib/cmdline.c ++++ b/arch/x86/lib/cmdline.c +@@ -82,3 +82,108 @@ int cmdline_find_option_bool(const char *cmdline, const char *option) + + return 0; /* Buffer overrun */ + } ++ ++/* ++ * Find a non-boolean option (i.e. option=argument). In accordance with ++ * standard Linux practice, if this option is repeated, this returns the ++ * last instance on the command line. ++ * ++ * @cmdline: the cmdline string ++ * @max_cmdline_size: the maximum size of cmdline ++ * @option: option string to look for ++ * @buffer: memory buffer to return the option argument ++ * @bufsize: size of the supplied memory buffer ++ * ++ * Returns the length of the argument (regardless of if it was ++ * truncated to fit in the buffer), or -1 on not found. ++ */ ++static int ++__cmdline_find_option(const char *cmdline, int max_cmdline_size, ++ const char *option, char *buffer, int bufsize) ++{ ++ char c; ++ int pos = 0, len = -1; ++ const char *opptr = NULL; ++ char *bufptr = buffer; ++ enum { ++ st_wordstart = 0, /* Start of word/after whitespace */ ++ st_wordcmp, /* Comparing this word */ ++ st_wordskip, /* Miscompare, skip */ ++ st_bufcpy, /* Copying this to buffer */ ++ } state = st_wordstart; ++ ++ if (!cmdline) ++ return -1; /* No command line */ ++ ++ /* ++ * This 'pos' check ensures we do not overrun ++ * a non-NULL-terminated 'cmdline' ++ */ ++ while (pos++ < max_cmdline_size) { ++ c = *(char *)cmdline++; ++ if (!c) ++ break; ++ ++ switch (state) { ++ case st_wordstart: ++ if (myisspace(c)) ++ break; ++ ++ state = st_wordcmp; ++ opptr = option; ++ /* fall through */ ++ ++ case st_wordcmp: ++ if ((c == '=') && !*opptr) { ++ /* ++ * We matched all the way to the end of the ++ * option we were looking for, prepare to ++ * copy the argument. ++ */ ++ len = 0; ++ bufptr = buffer; ++ state = st_bufcpy; ++ break; ++ } else if (c == *opptr++) { ++ /* ++ * We are currently matching, so continue ++ * to the next character on the cmdline. ++ */ ++ break; ++ } ++ state = st_wordskip; ++ /* fall through */ ++ ++ case st_wordskip: ++ if (myisspace(c)) ++ state = st_wordstart; ++ break; ++ ++ case st_bufcpy: ++ if (myisspace(c)) { ++ state = st_wordstart; ++ } else { ++ /* ++ * Increment len, but don't overrun the ++ * supplied buffer and leave room for the ++ * NULL terminator. ++ */ ++ if (++len < bufsize) ++ *bufptr++ = c; ++ } ++ break; ++ } ++ } ++ ++ if (bufsize) ++ *bufptr = '\0'; ++ ++ return len; ++} ++ ++int cmdline_find_option(const char *cmdline, const char *option, char *buffer, ++ int bufsize) ++{ ++ return __cmdline_find_option(cmdline, COMMAND_LINE_SIZE, option, ++ buffer, bufsize); ++} +diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile +index 1ae7c141f778..61e6cead9c4a 100644 +--- a/arch/x86/mm/Makefile ++++ b/arch/x86/mm/Makefile +@@ -32,3 +32,4 @@ obj-$(CONFIG_ACPI_NUMA) += srat.o + obj-$(CONFIG_NUMA_EMU) += numa_emulation.o + + obj-$(CONFIG_X86_INTEL_MPX) += mpx.o ++obj-$(CONFIG_PAGE_TABLE_ISOLATION) += kaiser.o +diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c +index ed4b372860e4..2bd45ae91eb3 100644 +--- a/arch/x86/mm/init.c ++++ b/arch/x86/mm/init.c +@@ -165,7 +165,7 @@ static void __init probe_page_size_mask(void) + cr4_set_bits_and_update_boot(X86_CR4_PSE); + + /* Enable PGE if available */ +- if (cpu_has_pge) { ++ if (cpu_has_pge && !kaiser_enabled) { + cr4_set_bits_and_update_boot(X86_CR4_PGE); + __supported_pte_mask |= _PAGE_GLOBAL; + } else +diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c +index ec081fe0ce2c..d76ec9348cff 100644 +--- a/arch/x86/mm/init_64.c ++++ b/arch/x86/mm/init_64.c +@@ -395,6 +395,16 @@ void __init cleanup_highmap(void) + continue; + if (vaddr < (unsigned long) _text || vaddr > end) + set_pmd(pmd, __pmd(0)); ++ else if (kaiser_enabled) { ++ /* ++ * level2_kernel_pgt is initialized with _PAGE_GLOBAL: ++ * clear that now. This is not important, so long as ++ * CR4.PGE remains clear, but it removes an anomaly. ++ * Physical mapping setup below avoids _PAGE_GLOBAL ++ * by use of massage_pgprot() inside pfn_pte() etc. ++ */ ++ set_pmd(pmd, pmd_clear_flags(*pmd, _PAGE_GLOBAL)); ++ } + } + } + +diff --git a/arch/x86/mm/kaiser.c b/arch/x86/mm/kaiser.c +new file mode 100644 +index 000000000000..b0b3a69f1c7f +--- /dev/null ++++ b/arch/x86/mm/kaiser.c +@@ -0,0 +1,456 @@ ++#include <linux/bug.h> ++#include <linux/kernel.h> ++#include <linux/errno.h> ++#include <linux/string.h> ++#include <linux/types.h> ++#include <linux/bug.h> ++#include <linux/init.h> ++#include <linux/interrupt.h> ++#include <linux/spinlock.h> ++#include <linux/mm.h> ++#include <linux/uaccess.h> ++#include <linux/ftrace.h> ++ ++#undef pr_fmt ++#define pr_fmt(fmt) "Kernel/User page tables isolation: " fmt ++ ++#include <asm/kaiser.h> ++#include <asm/tlbflush.h> /* to verify its kaiser declarations */ ++#include <asm/pgtable.h> ++#include <asm/pgalloc.h> ++#include <asm/desc.h> ++#include <asm/cmdline.h> ++ ++int kaiser_enabled __read_mostly = 1; ++EXPORT_SYMBOL(kaiser_enabled); /* for inlined TLB flush functions */ ++ ++__visible ++DEFINE_PER_CPU_USER_MAPPED(unsigned long, unsafe_stack_register_backup); ++ ++/* ++ * These can have bit 63 set, so we can not just use a plain "or" ++ * instruction to get their value or'd into CR3. It would take ++ * another register. So, we use a memory reference to these instead. ++ * ++ * This is also handy because systems that do not support PCIDs ++ * just end up or'ing a 0 into their CR3, which does no harm. ++ */ ++DEFINE_PER_CPU(unsigned long, x86_cr3_pcid_user); ++ ++/* ++ * At runtime, the only things we map are some things for CPU ++ * hotplug, and stacks for new processes. No two CPUs will ever ++ * be populating the same addresses, so we only need to ensure ++ * that we protect between two CPUs trying to allocate and ++ * populate the same page table page. ++ * ++ * Only take this lock when doing a set_p[4um]d(), but it is not ++ * needed for doing a set_pte(). We assume that only the *owner* ++ * of a given allocation will be doing this for _their_ ++ * allocation. ++ * ++ * This ensures that once a system has been running for a while ++ * and there have been stacks all over and these page tables ++ * are fully populated, there will be no further acquisitions of ++ * this lock. ++ */ ++static DEFINE_SPINLOCK(shadow_table_allocation_lock); ++ ++/* ++ * Returns -1 on error. ++ */ ++static inline unsigned long get_pa_from_mapping(unsigned long vaddr) ++{ ++ pgd_t *pgd; ++ pud_t *pud; ++ pmd_t *pmd; ++ pte_t *pte; ++ ++ pgd = pgd_offset_k(vaddr); ++ /* ++ * We made all the kernel PGDs present in kaiser_init(). ++ * We expect them to stay that way. ++ */ ++ BUG_ON(pgd_none(*pgd)); ++ /* ++ * PGDs are either 512GB or 128TB on all x86_64 ++ * configurations. We don't handle these. ++ */ ++ BUG_ON(pgd_large(*pgd)); ++ ++ pud = pud_offset(pgd, vaddr); ++ if (pud_none(*pud)) { ++ WARN_ON_ONCE(1); ++ return -1; ++ } ++ ++ if (pud_large(*pud)) ++ return (pud_pfn(*pud) << PAGE_SHIFT) | (vaddr & ~PUD_PAGE_MASK); ++ ++ pmd = pmd_offset(pud, vaddr); ++ if (pmd_none(*pmd)) { ++ WARN_ON_ONCE(1); ++ return -1; ++ } ++ ++ if (pmd_large(*pmd)) ++ return (pmd_pfn(*pmd) << PAGE_SHIFT) | (vaddr & ~PMD_PAGE_MASK); ++ ++ pte = pte_offset_kernel(pmd, vaddr); ++ if (pte_none(*pte)) { ++ WARN_ON_ONCE(1); ++ return -1; ++ } ++ ++ return (pte_pfn(*pte) << PAGE_SHIFT) | (vaddr & ~PAGE_MASK); ++} ++ ++/* ++ * This is a relatively normal page table walk, except that it ++ * also tries to allocate page tables pages along the way. ++ * ++ * Returns a pointer to a PTE on success, or NULL on failure. ++ */ ++static pte_t *kaiser_pagetable_walk(unsigned long address) ++{ ++ pmd_t *pmd; ++ pud_t *pud; ++ pgd_t *pgd = native_get_shadow_pgd(pgd_offset_k(address)); ++ gfp_t gfp = (GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO); ++ ++ if (pgd_none(*pgd)) { ++ WARN_ONCE(1, "All shadow pgds should have been populated"); ++ return NULL; ++ } ++ BUILD_BUG_ON(pgd_large(*pgd) != 0); ++ ++ pud = pud_offset(pgd, address); ++ /* The shadow page tables do not use large mappings: */ ++ if (pud_large(*pud)) { ++ WARN_ON(1); ++ return NULL; ++ } ++ if (pud_none(*pud)) { ++ unsigned long new_pmd_page = __get_free_page(gfp); ++ if (!new_pmd_page) ++ return NULL; ++ spin_lock(&shadow_table_allocation_lock); ++ if (pud_none(*pud)) { ++ set_pud(pud, __pud(_KERNPG_TABLE | __pa(new_pmd_page))); ++ __inc_zone_page_state(virt_to_page((void *) ++ new_pmd_page), NR_KAISERTABLE); ++ } else ++ free_page(new_pmd_page); ++ spin_unlock(&shadow_table_allocation_lock); ++ } ++ ++ pmd = pmd_offset(pud, address); ++ /* The shadow page tables do not use large mappings: */ ++ if (pmd_large(*pmd)) { ++ WARN_ON(1); ++ return NULL; ++ } ++ if (pmd_none(*pmd)) { ++ unsigned long new_pte_page = __get_free_page(gfp); ++ if (!new_pte_page) ++ return NULL; ++ spin_lock(&shadow_table_allocation_lock); ++ if (pmd_none(*pmd)) { ++ set_pmd(pmd, __pmd(_KERNPG_TABLE | __pa(new_pte_page))); ++ __inc_zone_page_state(virt_to_page((void *) ++ new_pte_page), NR_KAISERTABLE); ++ } else ++ free_page(new_pte_page); ++ spin_unlock(&shadow_table_allocation_lock); ++ } ++ ++ return pte_offset_kernel(pmd, address); ++} ++ ++static int kaiser_add_user_map(const void *__start_addr, unsigned long size, ++ unsigned long flags) ++{ ++ int ret = 0; ++ pte_t *pte; ++ unsigned long start_addr = (unsigned long )__start_addr; ++ unsigned long address = start_addr & PAGE_MASK; ++ unsigned long end_addr = PAGE_ALIGN(start_addr + size); ++ unsigned long target_address; ++ ++ /* ++ * It is convenient for callers to pass in __PAGE_KERNEL etc, ++ * and there is no actual harm from setting _PAGE_GLOBAL, so ++ * long as CR4.PGE is not set. But it is nonetheless troubling ++ * to see Kaiser itself setting _PAGE_GLOBAL (now that "nokaiser" ++ * requires that not to be #defined to 0): so mask it off here. ++ */ ++ flags &= ~_PAGE_GLOBAL; ++ ++ for (; address < end_addr; address += PAGE_SIZE) { ++ target_address = get_pa_from_mapping(address); ++ if (target_address == -1) { ++ ret = -EIO; ++ break; ++ } ++ pte = kaiser_pagetable_walk(address); ++ if (!pte) { ++ ret = -ENOMEM; ++ break; ++ } ++ if (pte_none(*pte)) { ++ set_pte(pte, __pte(flags | target_address)); ++ } else { ++ pte_t tmp; ++ set_pte(&tmp, __pte(flags | target_address)); ++ WARN_ON_ONCE(!pte_same(*pte, tmp)); ++ } ++ } ++ return ret; ++} ++ ++static int kaiser_add_user_map_ptrs(const void *start, const void *end, unsigned long flags) ++{ ++ unsigned long size = end - start; ++ ++ return kaiser_add_user_map(start, size, flags); ++} ++ ++/* ++ * Ensure that the top level of the (shadow) page tables are ++ * entirely populated. This ensures that all processes that get ++ * forked have the same entries. This way, we do not have to ++ * ever go set up new entries in older processes. ++ * ++ * Note: we never free these, so there are no updates to them ++ * after this. ++ */ ++static void __init kaiser_init_all_pgds(void) ++{ ++ pgd_t *pgd; ++ int i = 0; ++ ++ pgd = native_get_shadow_pgd(pgd_offset_k((unsigned long )0)); ++ for (i = PTRS_PER_PGD / 2; i < PTRS_PER_PGD; i++) { ++ pgd_t new_pgd; ++ pud_t *pud = pud_alloc_one(&init_mm, ++ PAGE_OFFSET + i * PGDIR_SIZE); ++ if (!pud) { ++ WARN_ON(1); ++ break; ++ } ++ inc_zone_page_state(virt_to_page(pud), NR_KAISERTABLE); ++ new_pgd = __pgd(_KERNPG_TABLE |__pa(pud)); ++ /* ++ * Make sure not to stomp on some other pgd entry. ++ */ ++ if (!pgd_none(pgd[i])) { ++ WARN_ON(1); ++ continue; ++ } ++ set_pgd(pgd + i, new_pgd); ++ } ++} ++ ++#define kaiser_add_user_map_early(start, size, flags) do { \ ++ int __ret = kaiser_add_user_map(start, size, flags); \ ++ WARN_ON(__ret); \ ++} while (0) ++ ++#define kaiser_add_user_map_ptrs_early(start, end, flags) do { \ ++ int __ret = kaiser_add_user_map_ptrs(start, end, flags); \ ++ WARN_ON(__ret); \ ++} while (0) ++ ++void __init kaiser_check_boottime_disable(void) ++{ ++ bool enable = true; ++ char arg[5]; ++ int ret; ++ ++ if (boot_cpu_has(X86_FEATURE_XENPV)) ++ goto silent_disable; ++ ++ ret = cmdline_find_option(boot_command_line, "pti", arg, sizeof(arg)); ++ if (ret > 0) { ++ if (!strncmp(arg, "on", 2)) ++ goto enable; ++ ++ if (!strncmp(arg, "off", 3)) ++ goto disable; ++ ++ if (!strncmp(arg, "auto", 4)) ++ goto skip; ++ } ++ ++ if (cmdline_find_option_bool(boot_command_line, "nopti")) ++ goto disable; ++ ++skip: ++ if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) ++ goto disable; ++ ++enable: ++ if (enable) ++ setup_force_cpu_cap(X86_FEATURE_KAISER); ++ ++ return; ++ ++disable: ++ pr_info("disabled\n"); ++ ++silent_disable: ++ kaiser_enabled = 0; ++ setup_clear_cpu_cap(X86_FEATURE_KAISER); ++} ++ ++/* ++ * If anything in here fails, we will likely die on one of the ++ * first kernel->user transitions and init will die. But, we ++ * will have most of the kernel up by then and should be able to ++ * get a clean warning out of it. If we BUG_ON() here, we run ++ * the risk of being before we have good console output. ++ */ ++void __init kaiser_init(void) ++{ ++ int cpu; ++ ++ if (!kaiser_enabled) ++ return; ++ ++ kaiser_init_all_pgds(); ++ ++ for_each_possible_cpu(cpu) { ++ void *percpu_vaddr = __per_cpu_user_mapped_start + ++ per_cpu_offset(cpu); ++ unsigned long percpu_sz = __per_cpu_user_mapped_end - ++ __per_cpu_user_mapped_start; ++ kaiser_add_user_map_early(percpu_vaddr, percpu_sz, ++ __PAGE_KERNEL); ++ } ++ ++ /* ++ * Map the entry/exit text section, which is needed at ++ * switches from user to and from kernel. ++ */ ++ kaiser_add_user_map_ptrs_early(__entry_text_start, __entry_text_end, ++ __PAGE_KERNEL_RX); ++ ++#if defined(CONFIG_FUNCTION_GRAPH_TRACER) || defined(CONFIG_KASAN) ++ kaiser_add_user_map_ptrs_early(__irqentry_text_start, ++ __irqentry_text_end, ++ __PAGE_KERNEL_RX); ++#endif ++ kaiser_add_user_map_early((void *)idt_descr.address, ++ sizeof(gate_desc) * NR_VECTORS, ++ __PAGE_KERNEL_RO); ++#ifdef CONFIG_TRACING ++ kaiser_add_user_map_early(&trace_idt_descr, ++ sizeof(trace_idt_descr), ++ __PAGE_KERNEL); ++ kaiser_add_user_map_early(&trace_idt_table, ++ sizeof(gate_desc) * NR_VECTORS, ++ __PAGE_KERNEL); ++#endif ++ kaiser_add_user_map_early(&debug_idt_descr, sizeof(debug_idt_descr), ++ __PAGE_KERNEL); ++ kaiser_add_user_map_early(&debug_idt_table, ++ sizeof(gate_desc) * NR_VECTORS, ++ __PAGE_KERNEL); ++ ++ pr_info("enabled\n"); ++} ++ ++/* Add a mapping to the shadow mapping, and synchronize the mappings */ ++int kaiser_add_mapping(unsigned long addr, unsigned long size, unsigned long flags) ++{ ++ if (!kaiser_enabled) ++ return 0; ++ return kaiser_add_user_map((const void *)addr, size, flags); ++} ++ ++void kaiser_remove_mapping(unsigned long start, unsigned long size) ++{ ++ extern void unmap_pud_range_nofree(pgd_t *pgd, ++ unsigned long start, unsigned long end); ++ unsigned long end = start + size; ++ unsigned long addr, next; ++ pgd_t *pgd; ++ ++ if (!kaiser_enabled) ++ return; ++ pgd = native_get_shadow_pgd(pgd_offset_k(start)); ++ for (addr = start; addr < end; pgd++, addr = next) { ++ next = pgd_addr_end(addr, end); ++ unmap_pud_range_nofree(pgd, addr, next); ++ } ++} ++ ++/* ++ * Page table pages are page-aligned. The lower half of the top ++ * level is used for userspace and the top half for the kernel. ++ * This returns true for user pages that need to get copied into ++ * both the user and kernel copies of the page tables, and false ++ * for kernel pages that should only be in the kernel copy. ++ */ ++static inline bool is_userspace_pgd(pgd_t *pgdp) ++{ ++ return ((unsigned long)pgdp % PAGE_SIZE) < (PAGE_SIZE / 2); ++} ++ ++pgd_t kaiser_set_shadow_pgd(pgd_t *pgdp, pgd_t pgd) ++{ ++ if (!kaiser_enabled) ++ return pgd; ++ /* ++ * Do we need to also populate the shadow pgd? Check _PAGE_USER to ++ * skip cases like kexec and EFI which make temporary low mappings. ++ */ ++ if (pgd.pgd & _PAGE_USER) { ++ if (is_userspace_pgd(pgdp)) { ++ native_get_shadow_pgd(pgdp)->pgd = pgd.pgd; ++ /* ++ * Even if the entry is *mapping* userspace, ensure ++ * that userspace can not use it. This way, if we ++ * get out to userspace running on the kernel CR3, ++ * userspace will crash instead of running. ++ */ ++ if (__supported_pte_mask & _PAGE_NX) ++ pgd.pgd |= _PAGE_NX; ++ } ++ } else if (!pgd.pgd) { ++ /* ++ * pgd_clear() cannot check _PAGE_USER, and is even used to ++ * clear corrupted pgd entries: so just rely on cases like ++ * kexec and EFI never to be using pgd_clear(). ++ */ ++ if (!WARN_ON_ONCE((unsigned long)pgdp & PAGE_SIZE) && ++ is_userspace_pgd(pgdp)) ++ native_get_shadow_pgd(pgdp)->pgd = pgd.pgd; ++ } ++ return pgd; ++} ++ ++void kaiser_setup_pcid(void) ++{ ++ unsigned long user_cr3 = KAISER_SHADOW_PGD_OFFSET; ++ ++ if (this_cpu_has(X86_FEATURE_PCID)) ++ user_cr3 |= X86_CR3_PCID_USER_NOFLUSH; ++ /* ++ * These variables are used by the entry/exit ++ * code to change PCID and pgd and TLB flushing. ++ */ ++ this_cpu_write(x86_cr3_pcid_user, user_cr3); ++} ++ ++/* ++ * Make a note that this cpu will need to flush USER tlb on return to user. ++ * If cpu does not have PCID, then the NOFLUSH bit will never have been set. ++ */ ++void kaiser_flush_tlb_on_return_to_user(void) ++{ ++ if (this_cpu_has(X86_FEATURE_PCID)) ++ this_cpu_write(x86_cr3_pcid_user, ++ X86_CR3_PCID_USER_FLUSH | KAISER_SHADOW_PGD_OFFSET); ++} ++EXPORT_SYMBOL(kaiser_flush_tlb_on_return_to_user); +diff --git a/arch/x86/mm/kasan_init_64.c b/arch/x86/mm/kasan_init_64.c +index 4e5ac46adc9d..81ec7c02f968 100644 +--- a/arch/x86/mm/kasan_init_64.c ++++ b/arch/x86/mm/kasan_init_64.c +@@ -121,11 +121,16 @@ void __init kasan_init(void) + kasan_populate_zero_shadow(kasan_mem_to_shadow((void *)MODULES_END), + (void *)KASAN_SHADOW_END); + +- memset(kasan_zero_page, 0, PAGE_SIZE); +- + load_cr3(init_level4_pgt); + __flush_tlb_all(); +- init_task.kasan_depth = 0; + ++ /* ++ * kasan_zero_page has been used as early shadow memory, thus it may ++ * contain some garbage. Now we can clear it, since after the TLB flush ++ * no one should write to it. ++ */ ++ memset(kasan_zero_page, 0, PAGE_SIZE); ++ ++ init_task.kasan_depth = 0; + pr_info("KernelAddressSanitizer initialized\n"); + } +diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c +index b599a780a5a9..79377e2a7bcd 100644 +--- a/arch/x86/mm/pageattr.c ++++ b/arch/x86/mm/pageattr.c +@@ -52,6 +52,7 @@ static DEFINE_SPINLOCK(cpa_lock); + #define CPA_FLUSHTLB 1 + #define CPA_ARRAY 2 + #define CPA_PAGES_ARRAY 4 ++#define CPA_FREE_PAGETABLES 8 + + #ifdef CONFIG_PROC_FS + static unsigned long direct_pages_count[PG_LEVEL_NUM]; +@@ -723,10 +724,13 @@ static int split_large_page(struct cpa_data *cpa, pte_t *kpte, + return 0; + } + +-static bool try_to_free_pte_page(pte_t *pte) ++static bool try_to_free_pte_page(struct cpa_data *cpa, pte_t *pte) + { + int i; + ++ if (!(cpa->flags & CPA_FREE_PAGETABLES)) ++ return false; ++ + for (i = 0; i < PTRS_PER_PTE; i++) + if (!pte_none(pte[i])) + return false; +@@ -735,10 +739,13 @@ static bool try_to_free_pte_page(pte_t *pte) + return true; + } + +-static bool try_to_free_pmd_page(pmd_t *pmd) ++static bool try_to_free_pmd_page(struct cpa_data *cpa, pmd_t *pmd) + { + int i; + ++ if (!(cpa->flags & CPA_FREE_PAGETABLES)) ++ return false; ++ + for (i = 0; i < PTRS_PER_PMD; i++) + if (!pmd_none(pmd[i])) + return false; +@@ -759,7 +766,9 @@ static bool try_to_free_pud_page(pud_t *pud) + return true; + } + +-static bool unmap_pte_range(pmd_t *pmd, unsigned long start, unsigned long end) ++static bool unmap_pte_range(struct cpa_data *cpa, pmd_t *pmd, ++ unsigned long start, ++ unsigned long end) + { + pte_t *pte = pte_offset_kernel(pmd, start); + +@@ -770,22 +779,23 @@ static bool unmap_pte_range(pmd_t *pmd, unsigned long start, unsigned long end) + pte++; + } + +- if (try_to_free_pte_page((pte_t *)pmd_page_vaddr(*pmd))) { ++ if (try_to_free_pte_page(cpa, (pte_t *)pmd_page_vaddr(*pmd))) { + pmd_clear(pmd); + return true; + } + return false; + } + +-static void __unmap_pmd_range(pud_t *pud, pmd_t *pmd, ++static void __unmap_pmd_range(struct cpa_data *cpa, pud_t *pud, pmd_t *pmd, + unsigned long start, unsigned long end) + { +- if (unmap_pte_range(pmd, start, end)) +- if (try_to_free_pmd_page((pmd_t *)pud_page_vaddr(*pud))) ++ if (unmap_pte_range(cpa, pmd, start, end)) ++ if (try_to_free_pmd_page(cpa, (pmd_t *)pud_page_vaddr(*pud))) + pud_clear(pud); + } + +-static void unmap_pmd_range(pud_t *pud, unsigned long start, unsigned long end) ++static void unmap_pmd_range(struct cpa_data *cpa, pud_t *pud, ++ unsigned long start, unsigned long end) + { + pmd_t *pmd = pmd_offset(pud, start); + +@@ -796,7 +806,7 @@ static void unmap_pmd_range(pud_t *pud, unsigned long start, unsigned long end) + unsigned long next_page = (start + PMD_SIZE) & PMD_MASK; + unsigned long pre_end = min_t(unsigned long, end, next_page); + +- __unmap_pmd_range(pud, pmd, start, pre_end); ++ __unmap_pmd_range(cpa, pud, pmd, start, pre_end); + + start = pre_end; + pmd++; +@@ -809,7 +819,8 @@ static void unmap_pmd_range(pud_t *pud, unsigned long start, unsigned long end) + if (pmd_large(*pmd)) + pmd_clear(pmd); + else +- __unmap_pmd_range(pud, pmd, start, start + PMD_SIZE); ++ __unmap_pmd_range(cpa, pud, pmd, ++ start, start + PMD_SIZE); + + start += PMD_SIZE; + pmd++; +@@ -819,17 +830,19 @@ static void unmap_pmd_range(pud_t *pud, unsigned long start, unsigned long end) + * 4K leftovers? + */ + if (start < end) +- return __unmap_pmd_range(pud, pmd, start, end); ++ return __unmap_pmd_range(cpa, pud, pmd, start, end); + + /* + * Try again to free the PMD page if haven't succeeded above. + */ + if (!pud_none(*pud)) +- if (try_to_free_pmd_page((pmd_t *)pud_page_vaddr(*pud))) ++ if (try_to_free_pmd_page(cpa, (pmd_t *)pud_page_vaddr(*pud))) + pud_clear(pud); + } + +-static void unmap_pud_range(pgd_t *pgd, unsigned long start, unsigned long end) ++static void __unmap_pud_range(struct cpa_data *cpa, pgd_t *pgd, ++ unsigned long start, ++ unsigned long end) + { + pud_t *pud = pud_offset(pgd, start); + +@@ -840,7 +853,7 @@ static void unmap_pud_range(pgd_t *pgd, unsigned long start, unsigned long end) + unsigned long next_page = (start + PUD_SIZE) & PUD_MASK; + unsigned long pre_end = min_t(unsigned long, end, next_page); + +- unmap_pmd_range(pud, start, pre_end); ++ unmap_pmd_range(cpa, pud, start, pre_end); + + start = pre_end; + pud++; +@@ -854,7 +867,7 @@ static void unmap_pud_range(pgd_t *pgd, unsigned long start, unsigned long end) + if (pud_large(*pud)) + pud_clear(pud); + else +- unmap_pmd_range(pud, start, start + PUD_SIZE); ++ unmap_pmd_range(cpa, pud, start, start + PUD_SIZE); + + start += PUD_SIZE; + pud++; +@@ -864,7 +877,7 @@ static void unmap_pud_range(pgd_t *pgd, unsigned long start, unsigned long end) + * 2M leftovers? + */ + if (start < end) +- unmap_pmd_range(pud, start, end); ++ unmap_pmd_range(cpa, pud, start, end); + + /* + * No need to try to free the PUD page because we'll free it in +@@ -872,6 +885,24 @@ static void unmap_pud_range(pgd_t *pgd, unsigned long start, unsigned long end) + */ + } + ++static void unmap_pud_range(pgd_t *pgd, unsigned long start, unsigned long end) ++{ ++ struct cpa_data cpa = { ++ .flags = CPA_FREE_PAGETABLES, ++ }; ++ ++ __unmap_pud_range(&cpa, pgd, start, end); ++} ++ ++void unmap_pud_range_nofree(pgd_t *pgd, unsigned long start, unsigned long end) ++{ ++ struct cpa_data cpa = { ++ .flags = 0, ++ }; ++ ++ __unmap_pud_range(&cpa, pgd, start, end); ++} ++ + static void unmap_pgd_range(pgd_t *root, unsigned long addr, unsigned long end) + { + pgd_t *pgd_entry = root + pgd_index(addr); +diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c +index fb0a9dd1d6e4..dbc27a2b4ad5 100644 +--- a/arch/x86/mm/pgtable.c ++++ b/arch/x86/mm/pgtable.c +@@ -6,7 +6,7 @@ + #include <asm/fixmap.h> + #include <asm/mtrr.h> + +-#define PGALLOC_GFP GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO ++#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO) + + #ifdef CONFIG_HIGHPTE + #define PGALLOC_USER_GFP __GFP_HIGHMEM +@@ -340,14 +340,24 @@ static inline void _pgd_free(pgd_t *pgd) + kmem_cache_free(pgd_cache, pgd); + } + #else ++ ++/* ++ * Instead of one pgd, Kaiser acquires two pgds. Being order-1, it is ++ * both 8k in size and 8k-aligned. That lets us just flip bit 12 ++ * in a pointer to swap between the two 4k halves. ++ */ ++#define PGD_ALLOCATION_ORDER kaiser_enabled ++ + static inline pgd_t *_pgd_alloc(void) + { +- return (pgd_t *)__get_free_page(PGALLOC_GFP); ++ /* No __GFP_REPEAT: to avoid page allocation stalls in order-1 case */ ++ return (pgd_t *)__get_free_pages(PGALLOC_GFP & ~__GFP_REPEAT, ++ PGD_ALLOCATION_ORDER); + } + + static inline void _pgd_free(pgd_t *pgd) + { +- free_page((unsigned long)pgd); ++ free_pages((unsigned long)pgd, PGD_ALLOCATION_ORDER); + } + #endif /* CONFIG_X86_PAE */ + +diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c +index 7a4cdb632508..7cad01af6dcd 100644 +--- a/arch/x86/mm/tlb.c ++++ b/arch/x86/mm/tlb.c +@@ -6,13 +6,14 @@ + #include <linux/interrupt.h> + #include <linux/module.h> + #include <linux/cpu.h> ++#include <linux/debugfs.h> + + #include <asm/tlbflush.h> + #include <asm/mmu_context.h> + #include <asm/cache.h> + #include <asm/apic.h> + #include <asm/uv/uv.h> +-#include <linux/debugfs.h> ++#include <asm/kaiser.h> + + /* + * TLB flushing, formerly SMP-only +@@ -34,6 +35,36 @@ struct flush_tlb_info { + unsigned long flush_end; + }; + ++static void load_new_mm_cr3(pgd_t *pgdir) ++{ ++ unsigned long new_mm_cr3 = __pa(pgdir); ++ ++ if (kaiser_enabled) { ++ /* ++ * We reuse the same PCID for different tasks, so we must ++ * flush all the entries for the PCID out when we change tasks. ++ * Flush KERN below, flush USER when returning to userspace in ++ * kaiser's SWITCH_USER_CR3 (_SWITCH_TO_USER_CR3) macro. ++ * ++ * invpcid_flush_single_context(X86_CR3_PCID_ASID_USER) could ++ * do it here, but can only be used if X86_FEATURE_INVPCID is ++ * available - and many machines support pcid without invpcid. ++ * ++ * If X86_CR3_PCID_KERN_FLUSH actually added something, then it ++ * would be needed in the write_cr3() below - if PCIDs enabled. ++ */ ++ BUILD_BUG_ON(X86_CR3_PCID_KERN_FLUSH); ++ kaiser_flush_tlb_on_return_to_user(); ++ } ++ ++ /* ++ * Caution: many callers of this function expect ++ * that load_cr3() is serializing and orders TLB ++ * fills with respect to the mm_cpumask writes. ++ */ ++ write_cr3(new_mm_cr3); ++} ++ + /* + * We cannot call mmdrop() because we are in interrupt context, + * instead update mm->cpu_vm_mask. +@@ -45,7 +76,7 @@ void leave_mm(int cpu) + BUG(); + if (cpumask_test_cpu(cpu, mm_cpumask(active_mm))) { + cpumask_clear_cpu(cpu, mm_cpumask(active_mm)); +- load_cr3(swapper_pg_dir); ++ load_new_mm_cr3(swapper_pg_dir); + /* + * This gets called in the idle path where RCU + * functions differently. Tracing normally +@@ -105,7 +136,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, + * ordering guarantee we need. + * + */ +- load_cr3(next->pgd); ++ load_new_mm_cr3(next->pgd); + + trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL); + +@@ -152,7 +183,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, + * As above, load_cr3() is serializing and orders TLB + * fills with respect to the mm_cpumask write. + */ +- load_cr3(next->pgd); ++ load_new_mm_cr3(next->pgd); + trace_tlb_flush(TLB_FLUSH_ON_TASK_SWITCH, TLB_FLUSH_ALL); + load_mm_cr4(next); + load_mm_ldt(next); +diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h +index ef2e8c97e183..a461b6604fd9 100644 +--- a/include/asm-generic/vmlinux.lds.h ++++ b/include/asm-generic/vmlinux.lds.h +@@ -725,7 +725,14 @@ + */ + #define PERCPU_INPUT(cacheline) \ + VMLINUX_SYMBOL(__per_cpu_start) = .; \ ++ VMLINUX_SYMBOL(__per_cpu_user_mapped_start) = .; \ + *(.data..percpu..first) \ ++ . = ALIGN(cacheline); \ ++ *(.data..percpu..user_mapped) \ ++ *(.data..percpu..user_mapped..shared_aligned) \ ++ . = ALIGN(PAGE_SIZE); \ ++ *(.data..percpu..user_mapped..page_aligned) \ ++ VMLINUX_SYMBOL(__per_cpu_user_mapped_end) = .; \ + . = ALIGN(PAGE_SIZE); \ + *(.data..percpu..page_aligned) \ + . = ALIGN(cacheline); \ +diff --git a/include/linux/kaiser.h b/include/linux/kaiser.h +new file mode 100644 +index 000000000000..58c55b1589d0 +--- /dev/null ++++ b/include/linux/kaiser.h +@@ -0,0 +1,52 @@ ++#ifndef _LINUX_KAISER_H ++#define _LINUX_KAISER_H ++ ++#ifdef CONFIG_PAGE_TABLE_ISOLATION ++#include <asm/kaiser.h> ++ ++static inline int kaiser_map_thread_stack(void *stack) ++{ ++ /* ++ * Map that page of kernel stack on which we enter from user context. ++ */ ++ return kaiser_add_mapping((unsigned long)stack + ++ THREAD_SIZE - PAGE_SIZE, PAGE_SIZE, __PAGE_KERNEL); ++} ++ ++static inline void kaiser_unmap_thread_stack(void *stack) ++{ ++ /* ++ * Note: may be called even when kaiser_map_thread_stack() failed. ++ */ ++ kaiser_remove_mapping((unsigned long)stack + ++ THREAD_SIZE - PAGE_SIZE, PAGE_SIZE); ++} ++#else ++ ++/* ++ * These stubs are used whenever CONFIG_PAGE_TABLE_ISOLATION is off, which ++ * includes architectures that support KAISER, but have it disabled. ++ */ ++ ++static inline void kaiser_init(void) ++{ ++} ++static inline int kaiser_add_mapping(unsigned long addr, ++ unsigned long size, unsigned long flags) ++{ ++ return 0; ++} ++static inline void kaiser_remove_mapping(unsigned long start, ++ unsigned long size) ++{ ++} ++static inline int kaiser_map_thread_stack(void *stack) ++{ ++ return 0; ++} ++static inline void kaiser_unmap_thread_stack(void *stack) ++{ ++} ++ ++#endif /* !CONFIG_PAGE_TABLE_ISOLATION */ ++#endif /* _LINUX_KAISER_H */ +diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h +index ff88d6189411..b93b578cfa42 100644 +--- a/include/linux/mmzone.h ++++ b/include/linux/mmzone.h +@@ -131,8 +131,9 @@ enum zone_stat_item { + NR_SLAB_RECLAIMABLE, + NR_SLAB_UNRECLAIMABLE, + NR_PAGETABLE, /* used for pagetables */ +- NR_KERNEL_STACK, + /* Second 128 byte cacheline */ ++ NR_KERNEL_STACK, ++ NR_KAISERTABLE, + NR_UNSTABLE_NFS, /* NFS unstable pages */ + NR_BOUNCE, + NR_VMSCAN_WRITE, +diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h +index 8f16299ca068..8902f23bb770 100644 +--- a/include/linux/percpu-defs.h ++++ b/include/linux/percpu-defs.h +@@ -35,6 +35,12 @@ + + #endif + ++#ifdef CONFIG_PAGE_TABLE_ISOLATION ++#define USER_MAPPED_SECTION "..user_mapped" ++#else ++#define USER_MAPPED_SECTION "" ++#endif ++ + /* + * Base implementations of per-CPU variable declarations and definitions, where + * the section in which the variable is to be placed is provided by the +@@ -115,6 +121,12 @@ + #define DEFINE_PER_CPU(type, name) \ + DEFINE_PER_CPU_SECTION(type, name, "") + ++#define DECLARE_PER_CPU_USER_MAPPED(type, name) \ ++ DECLARE_PER_CPU_SECTION(type, name, USER_MAPPED_SECTION) ++ ++#define DEFINE_PER_CPU_USER_MAPPED(type, name) \ ++ DEFINE_PER_CPU_SECTION(type, name, USER_MAPPED_SECTION) ++ + /* + * Declaration/definition used for per-CPU variables that must come first in + * the set of variables. +@@ -144,6 +156,14 @@ + DEFINE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \ + ____cacheline_aligned_in_smp + ++#define DECLARE_PER_CPU_SHARED_ALIGNED_USER_MAPPED(type, name) \ ++ DECLARE_PER_CPU_SECTION(type, name, USER_MAPPED_SECTION PER_CPU_SHARED_ALIGNED_SECTION) \ ++ ____cacheline_aligned_in_smp ++ ++#define DEFINE_PER_CPU_SHARED_ALIGNED_USER_MAPPED(type, name) \ ++ DEFINE_PER_CPU_SECTION(type, name, USER_MAPPED_SECTION PER_CPU_SHARED_ALIGNED_SECTION) \ ++ ____cacheline_aligned_in_smp ++ + #define DECLARE_PER_CPU_ALIGNED(type, name) \ + DECLARE_PER_CPU_SECTION(type, name, PER_CPU_ALIGNED_SECTION) \ + ____cacheline_aligned +@@ -162,11 +182,21 @@ + #define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \ + DEFINE_PER_CPU_SECTION(type, name, "..page_aligned") \ + __aligned(PAGE_SIZE) ++/* ++ * Declaration/definition used for per-CPU variables that must be page aligned and need to be mapped in user mode. ++ */ ++#define DECLARE_PER_CPU_PAGE_ALIGNED_USER_MAPPED(type, name) \ ++ DECLARE_PER_CPU_SECTION(type, name, USER_MAPPED_SECTION"..page_aligned") \ ++ __aligned(PAGE_SIZE) ++ ++#define DEFINE_PER_CPU_PAGE_ALIGNED_USER_MAPPED(type, name) \ ++ DEFINE_PER_CPU_SECTION(type, name, USER_MAPPED_SECTION"..page_aligned") \ ++ __aligned(PAGE_SIZE) + + /* + * Declaration/definition used for per-CPU variables that must be read mostly. + */ +-#define DECLARE_PER_CPU_READ_MOSTLY(type, name) \ ++#define DECLARE_PER_CPU_READ_MOSTLY(type, name) \ + DECLARE_PER_CPU_SECTION(type, name, "..read_mostly") + + #define DEFINE_PER_CPU_READ_MOSTLY(type, name) \ +diff --git a/init/main.c b/init/main.c +index 9e64d7097f1a..49926d95442f 100644 +--- a/init/main.c ++++ b/init/main.c +@@ -81,6 +81,7 @@ + #include <linux/integrity.h> + #include <linux/proc_ns.h> + #include <linux/io.h> ++#include <linux/kaiser.h> + + #include <asm/io.h> + #include <asm/bugs.h> +@@ -492,6 +493,7 @@ static void __init mm_init(void) + pgtable_init(); + vmalloc_init(); + ioremap_huge_init(); ++ kaiser_init(); + } + + asmlinkage __visible void __init start_kernel(void) +diff --git a/kernel/fork.c b/kernel/fork.c +index 68cfda1c1800..ac00f14208b7 100644 +--- a/kernel/fork.c ++++ b/kernel/fork.c +@@ -58,6 +58,7 @@ + #include <linux/tsacct_kern.h> + #include <linux/cn_proc.h> + #include <linux/freezer.h> ++#include <linux/kaiser.h> + #include <linux/delayacct.h> + #include <linux/taskstats_kern.h> + #include <linux/random.h> +@@ -169,6 +170,7 @@ static struct thread_info *alloc_thread_info_node(struct task_struct *tsk, + + static inline void free_thread_info(struct thread_info *ti) + { ++ kaiser_unmap_thread_stack(ti); + free_kmem_pages((unsigned long)ti, THREAD_SIZE_ORDER); + } + # else +@@ -352,6 +354,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) + goto free_ti; + + tsk->stack = ti; ++ ++ err = kaiser_map_thread_stack(tsk->stack); ++ if (err) ++ goto free_ti; + #ifdef CONFIG_SECCOMP + /* + * We must handle setting up seccomp filters once we're under +diff --git a/mm/vmstat.c b/mm/vmstat.c +index c344e3609c53..324b7e90b4c5 100644 +--- a/mm/vmstat.c ++++ b/mm/vmstat.c +@@ -736,6 +736,7 @@ const char * const vmstat_text[] = { + "nr_slab_unreclaimable", + "nr_page_table_pages", + "nr_kernel_stack", ++ "nr_overhead", + "nr_unstable", + "nr_bounce", + "nr_vmscan_write", +diff --git a/security/Kconfig b/security/Kconfig +index e45237897b43..a3ebb6ee5bd5 100644 +--- a/security/Kconfig ++++ b/security/Kconfig +@@ -31,6 +31,16 @@ config SECURITY + + If you are unsure how to answer this question, answer N. + ++config PAGE_TABLE_ISOLATION ++ bool "Remove the kernel mapping in user mode" ++ default y ++ depends on X86_64 && SMP ++ help ++ This enforces a strict kernel and user space isolation, in order ++ to close hardware side channels on kernel address information. ++ ++ If you are unsure how to answer this question, answer Y. ++ + config SECURITYFS + bool "Enable the securityfs filesystem" + help
