Branch data Line data Source code
1 : : // SPDX-License-Identifier: GPL-2.0-only 2 : : /* ----------------------------------------------------------------------- * 3 : : * 4 : : * Copyright 2014 Intel Corporation; author: H. Peter Anvin 5 : : * 6 : : * ----------------------------------------------------------------------- */ 7 : : 8 : : /* 9 : : * The IRET instruction, when returning to a 16-bit segment, only 10 : : * restores the bottom 16 bits of the user space stack pointer. This 11 : : * causes some 16-bit software to break, but it also leaks kernel state 12 : : * to user space. 13 : : * 14 : : * This works around this by creating percpu "ministacks", each of which 15 : : * is mapped 2^16 times 64K apart. When we detect that the return SS is 16 : : * on the LDT, we copy the IRET frame to the ministack and use the 17 : : * relevant alias to return to userspace. The ministacks are mapped 18 : : * readonly, so if the IRET fault we promote #GP to #DF which is an IST 19 : : * vector and thus has its own stack; we then do the fixup in the #DF 20 : : * handler. 21 : : * 22 : : * This file sets up the ministacks and the related page tables. The 23 : : * actual ministack invocation is in entry_64.S. 24 : : */ 25 : : 26 : : #include <linux/init.h> 27 : : #include <linux/init_task.h> 28 : : #include <linux/kernel.h> 29 : : #include <linux/percpu.h> 30 : : #include <linux/gfp.h> 31 : : #include <linux/random.h> 32 : : #include <asm/pgtable.h> 33 : : #include <asm/pgalloc.h> 34 : : #include <asm/setup.h> 35 : : #include <asm/espfix.h> 36 : : 37 : : /* 38 : : * Note: we only need 6*8 = 48 bytes for the espfix stack, but round 39 : : * it up to a cache line to avoid unnecessary sharing. 40 : : */ 41 : : #define ESPFIX_STACK_SIZE (8*8UL) 42 : : #define ESPFIX_STACKS_PER_PAGE (PAGE_SIZE/ESPFIX_STACK_SIZE) 43 : : 44 : : /* There is address space for how many espfix pages? */ 45 : : #define ESPFIX_PAGE_SPACE (1UL << (P4D_SHIFT-PAGE_SHIFT-16)) 46 : : 47 : : #define ESPFIX_MAX_CPUS (ESPFIX_STACKS_PER_PAGE * ESPFIX_PAGE_SPACE) 48 : : #if CONFIG_NR_CPUS > ESPFIX_MAX_CPUS 49 : : # error "Need more virtual address space for the ESPFIX hack" 50 : : #endif 51 : : 52 : : #define PGALLOC_GFP (GFP_KERNEL | __GFP_ZERO) 53 : : 54 : : /* This contains the *bottom* address of the espfix stack */ 55 : : DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_stack); 56 : : DEFINE_PER_CPU_READ_MOSTLY(unsigned long, espfix_waddr); 57 : : 58 : : /* Initialization mutex - should this be a spinlock? */ 59 : : static DEFINE_MUTEX(espfix_init_mutex); 60 : : 61 : : /* Page allocation bitmap - each page serves ESPFIX_STACKS_PER_PAGE CPUs */ 62 : : #define ESPFIX_MAX_PAGES DIV_ROUND_UP(CONFIG_NR_CPUS, ESPFIX_STACKS_PER_PAGE) 63 : : static void *espfix_pages[ESPFIX_MAX_PAGES]; 64 : : 65 : : static __page_aligned_bss pud_t espfix_pud_page[PTRS_PER_PUD] 66 : : __aligned(PAGE_SIZE); 67 : : 68 : : static unsigned int page_random, slot_random; 69 : : 70 : : /* 71 : : * This returns the bottom address of the espfix stack for a specific CPU. 72 : : * The math allows for a non-power-of-two ESPFIX_STACK_SIZE, in which case 73 : : * we have to account for some amount of padding at the end of each page. 74 : : */ 75 : 13 : static inline unsigned long espfix_base_addr(unsigned int cpu) 76 : : { 77 : 13 : unsigned long page, slot; 78 : 13 : unsigned long addr; 79 : : 80 : 13 : page = (cpu / ESPFIX_STACKS_PER_PAGE) ^ page_random; 81 : 13 : slot = (cpu + slot_random) % ESPFIX_STACKS_PER_PAGE; 82 : 13 : addr = (page << PAGE_SHIFT) + (slot * ESPFIX_STACK_SIZE); 83 : 13 : addr = (addr & 0xffffUL) | ((addr & ~0xffffUL) << 16); 84 : 13 : addr += ESPFIX_BASE_ADDR; 85 : 13 : return addr; 86 : : } 87 : : 88 : : #define PTE_STRIDE (65536/PAGE_SIZE) 89 : : #define ESPFIX_PTE_CLONES (PTRS_PER_PTE/PTE_STRIDE) 90 : : #define ESPFIX_PMD_CLONES PTRS_PER_PMD 91 : : #define ESPFIX_PUD_CLONES (65536/(ESPFIX_PTE_CLONES*ESPFIX_PMD_CLONES)) 92 : : 93 : : #define PGTABLE_PROT ((_KERNPG_TABLE & ~_PAGE_RW) | _PAGE_NX) 94 : : 95 : 13 : static void init_espfix_random(void) 96 : : { 97 : 13 : unsigned long rand; 98 : : 99 : : /* 100 : : * This is run before the entropy pools are initialized, 101 : : * but this is hopefully better than nothing. 102 : : */ 103 [ + - ]: 13 : if (!arch_get_random_long(&rand)) { 104 : : /* The constant is an arbitrary large prime */ 105 : 13 : rand = rdtsc(); 106 : 13 : rand *= 0xc345c6b72fd16123UL; 107 : : } 108 : : 109 : 13 : slot_random = rand % ESPFIX_STACKS_PER_PAGE; 110 : 13 : page_random = (rand / ESPFIX_STACKS_PER_PAGE) 111 : 13 : & (ESPFIX_PAGE_SPACE - 1); 112 : 13 : } 113 : : 114 : 13 : void __init init_espfix_bsp(void) 115 : : { 116 : 13 : pgd_t *pgd; 117 : 13 : p4d_t *p4d; 118 : : 119 : : /* Install the espfix pud into the kernel page directory */ 120 : 13 : pgd = &init_top_pgt[pgd_index(ESPFIX_BASE_ADDR)]; 121 : 13 : p4d = p4d_alloc(&init_mm, pgd, ESPFIX_BASE_ADDR); 122 [ - + ]: 13 : p4d_populate(&init_mm, p4d, espfix_pud_page); 123 : : 124 : : /* Randomize the locations */ 125 : 13 : init_espfix_random(); 126 : : 127 : : /* The rest is the same as for any other processor */ 128 : 13 : init_espfix_ap(0); 129 : 13 : } 130 : : 131 : 13 : void init_espfix_ap(int cpu) 132 : : { 133 : 13 : unsigned int page; 134 : 13 : unsigned long addr; 135 : 13 : pud_t pud, *pud_p; 136 : 13 : pmd_t pmd, *pmd_p; 137 : 13 : pte_t pte, *pte_p; 138 : 13 : int n, node; 139 : 13 : void *stack_page; 140 : 13 : pteval_t ptemask; 141 : : 142 : : /* We only have to do this once... */ 143 [ + - ]: 13 : if (likely(per_cpu(espfix_stack, cpu))) 144 : : return; /* Already initialized */ 145 : : 146 : 13 : addr = espfix_base_addr(cpu); 147 : 13 : page = cpu/ESPFIX_STACKS_PER_PAGE; 148 : : 149 : : /* Did another CPU already set this up? */ 150 [ - + ]: 13 : stack_page = READ_ONCE(espfix_pages[page]); 151 [ - + ]: 13 : if (likely(stack_page)) 152 : 0 : goto done; 153 : : 154 : 13 : mutex_lock(&espfix_init_mutex); 155 : : 156 : : /* Did we race on the lock? */ 157 [ - + ]: 13 : stack_page = READ_ONCE(espfix_pages[page]); 158 [ - + ]: 13 : if (stack_page) 159 : 0 : goto unlock_done; 160 : : 161 [ + - ]: 13 : node = cpu_to_node(cpu); 162 : 13 : ptemask = __supported_pte_mask; 163 : : 164 [ + - ]: 13 : pud_p = &espfix_pud_page[pud_index(addr)]; 165 : 13 : pud = *pud_p; 166 [ + - + - ]: 26 : if (!pud_present(pud)) { 167 [ - + ]: 13 : struct page *page = alloc_pages_node(node, PGALLOC_GFP, 0); 168 : : 169 [ + - ]: 13 : pmd_p = (pmd_t *)page_address(page); 170 [ + - ]: 26 : pud = __pud(__pa(pmd_p) | (PGTABLE_PROT & ptemask)); 171 : 13 : paravirt_alloc_pmd(&init_mm, __pa(pmd_p) >> PAGE_SHIFT); 172 [ + + ]: 65 : for (n = 0; n < ESPFIX_PUD_CLONES; n++) 173 : 52 : set_pud(&pud_p[n], pud); 174 : : } 175 : : 176 [ + - ]: 13 : pmd_p = pmd_offset(&pud, addr); 177 : 13 : pmd = *pmd_p; 178 [ + - + - ]: 26 : if (!pmd_present(pmd)) { 179 [ - + ]: 13 : struct page *page = alloc_pages_node(node, PGALLOC_GFP, 0); 180 : : 181 [ + - ]: 13 : pte_p = (pte_t *)page_address(page); 182 [ + - ]: 26 : pmd = __pmd(__pa(pte_p) | (PGTABLE_PROT & ptemask)); 183 : 13 : paravirt_alloc_pte(&init_mm, __pa(pte_p) >> PAGE_SHIFT); 184 [ + + ]: 6669 : for (n = 0; n < ESPFIX_PMD_CLONES; n++) 185 : 6656 : set_pmd(&pmd_p[n], pmd); 186 : : } 187 : : 188 [ + - ]: 13 : pte_p = pte_offset_kernel(&pmd, addr); 189 [ - + ]: 13 : stack_page = page_address(alloc_pages_node(node, GFP_KERNEL, 0)); 190 : : /* 191 : : * __PAGE_KERNEL_* includes _PAGE_GLOBAL, which we want since 192 : : * this is mapped to userspace. 193 : : */ 194 [ + - ]: 26 : pte = __pte(__pa(stack_page) | ((__PAGE_KERNEL_RO | _PAGE_ENC) & ptemask)); 195 [ + + ]: 429 : for (n = 0; n < ESPFIX_PTE_CLONES; n++) 196 : 416 : set_pte(&pte_p[n*PTE_STRIDE], pte); 197 : : 198 : : /* Job is done for this CPU and any CPU which shares this page */ 199 : 13 : WRITE_ONCE(espfix_pages[page], stack_page); 200 : : 201 : 13 : unlock_done: 202 : 13 : mutex_unlock(&espfix_init_mutex); 203 : 13 : done: 204 : 13 : per_cpu(espfix_stack, cpu) = addr; 205 : 13 : per_cpu(espfix_waddr, cpu) = (unsigned long)stack_page 206 : 13 : + (addr & ~PAGE_MASK); 207 : : }