LCOV - code coverage report
Current view: top level - arch/x86/include/asm - mmu_context.h (source / functions) Hit Total Coverage
Test: combined.info Lines: 41 49 83.7 %
Date: 2022-04-01 14:17:54 Functions: 6 6 100.0 %
Branches: 14 25 56.0 %

           Branch data     Line data    Source code
       1                 :            : /* SPDX-License-Identifier: GPL-2.0 */
       2                 :            : #ifndef _ASM_X86_MMU_CONTEXT_H
       3                 :            : #define _ASM_X86_MMU_CONTEXT_H
       4                 :            : 
       5                 :            : #include <asm/desc.h>
       6                 :            : #include <linux/atomic.h>
       7                 :            : #include <linux/mm_types.h>
       8                 :            : #include <linux/pkeys.h>
       9                 :            : 
      10                 :            : #include <trace/events/tlb.h>
      11                 :            : 
      12                 :            : #include <asm/pgalloc.h>
      13                 :            : #include <asm/tlbflush.h>
      14                 :            : #include <asm/paravirt.h>
      15                 :            : #include <asm/debugreg.h>
      16                 :            : 
      17                 :            : extern atomic64_t last_mm_ctx_id;
      18                 :            : 
      19                 :            : #ifndef CONFIG_PARAVIRT_XXL
      20                 :       9900 : static inline void paravirt_activate_mm(struct mm_struct *prev,
      21                 :            :                                         struct mm_struct *next)
      22                 :            : {
      23                 :       9900 : }
      24                 :            : #endif  /* !CONFIG_PARAVIRT_XXL */
      25                 :            : 
      26                 :            : #ifdef CONFIG_PERF_EVENTS
      27                 :            : 
      28                 :            : DECLARE_STATIC_KEY_FALSE(rdpmc_never_available_key);
      29                 :            : DECLARE_STATIC_KEY_FALSE(rdpmc_always_available_key);
      30                 :            : 
      31                 :     475436 : static inline void load_mm_cr4_irqsoff(struct mm_struct *mm)
      32                 :            : {
      33   [ +  -  +  - ]:     950872 :         if (static_branch_unlikely(&rdpmc_always_available_key) ||
      34   [ +  -  +  -  :    1426308 :             (!static_branch_unlikely(&rdpmc_never_available_key) &&
                   -  + ]
      35                 :     475436 :              atomic_read(&mm->context.perf_rdpmc_allowed)))
      36                 :          0 :                 cr4_set_bits_irqsoff(X86_CR4_PCE);
      37                 :            :         else
      38                 :     475436 :                 cr4_clear_bits_irqsoff(X86_CR4_PCE);
      39                 :     475436 : }
      40                 :            : #else
      41                 :            : static inline void load_mm_cr4_irqsoff(struct mm_struct *mm) {}
      42                 :            : #endif
      43                 :            : 
      44                 :            : #ifdef CONFIG_MODIFY_LDT_SYSCALL
      45                 :            : /*
      46                 :            :  * ldt_structs can be allocated, used, and freed, but they are never
      47                 :            :  * modified while live.
      48                 :            :  */
      49                 :            : struct ldt_struct {
      50                 :            :         /*
      51                 :            :          * Xen requires page-aligned LDTs with special permissions.  This is
      52                 :            :          * needed to prevent us from installing evil descriptors such as
      53                 :            :          * call gates.  On native, we could merge the ldt_struct and LDT
      54                 :            :          * allocations, but it's not worth trying to optimize.
      55                 :            :          */
      56                 :            :         struct desc_struct      *entries;
      57                 :            :         unsigned int            nr_entries;
      58                 :            : 
      59                 :            :         /*
      60                 :            :          * If PTI is in use, then the entries array is not mapped while we're
      61                 :            :          * in user mode.  The whole array will be aliased at the addressed
      62                 :            :          * given by ldt_slot_va(slot).  We use two slots so that we can allocate
      63                 :            :          * and map, and enable a new LDT without invalidating the mapping
      64                 :            :          * of an older, still-in-use LDT.
      65                 :            :          *
      66                 :            :          * slot will be -1 if this LDT doesn't have an alias mapping.
      67                 :            :          */
      68                 :            :         int                     slot;
      69                 :            : };
      70                 :            : 
      71                 :            : /*
      72                 :            :  * Used for LDT copy/destruction.
      73                 :            :  */
      74                 :            : static inline void init_new_context_ldt(struct mm_struct *mm)
      75                 :            : {
      76                 :            :         mm->context.ldt = NULL;
      77                 :            :         init_rwsem(&mm->context.ldt_usr_sem);
      78                 :            : }
      79                 :            : int ldt_dup_context(struct mm_struct *oldmm, struct mm_struct *mm);
      80                 :            : void destroy_context_ldt(struct mm_struct *mm);
      81                 :            : void ldt_arch_exit_mmap(struct mm_struct *mm);
      82                 :            : #else   /* CONFIG_MODIFY_LDT_SYSCALL */
      83                 :            : static inline void init_new_context_ldt(struct mm_struct *mm) { }
      84                 :            : static inline int ldt_dup_context(struct mm_struct *oldmm,
      85                 :            :                                   struct mm_struct *mm)
      86                 :            : {
      87                 :            :         return 0;
      88                 :            : }
      89                 :            : static inline void destroy_context_ldt(struct mm_struct *mm) { }
      90                 :            : static inline void ldt_arch_exit_mmap(struct mm_struct *mm) { }
      91                 :            : #endif
      92                 :            : 
      93                 :            : #ifdef CONFIG_MODIFY_LDT_SYSCALL
      94                 :            : extern void load_mm_ldt(struct mm_struct *mm);
      95                 :            : extern void switch_ldt(struct mm_struct *prev, struct mm_struct *next);
      96                 :            : #else
      97                 :            : static inline void load_mm_ldt(struct mm_struct *mm)
      98                 :            : {
      99                 :            :         clear_LDT();
     100                 :            : }
     101                 :            : static inline void switch_ldt(struct mm_struct *prev, struct mm_struct *next)
     102                 :            : {
     103                 :            :         DEBUG_LOCKS_WARN_ON(preemptible());
     104                 :            : }
     105                 :            : #endif
     106                 :            : 
     107                 :            : extern void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk);
     108                 :            : 
     109                 :            : /*
     110                 :            :  * Init a new mm.  Used on mm copies, like at fork()
     111                 :            :  * and on mm's that are brand-new, like at execve().
     112                 :            :  */
     113                 :            : static inline int init_new_context(struct task_struct *tsk,
     114                 :            :                                    struct mm_struct *mm)
     115                 :            : {
     116                 :            :         mutex_init(&mm->context.lock);
     117                 :            : 
     118                 :            :         mm->context.ctx_id = atomic64_inc_return(&last_mm_ctx_id);
     119                 :            :         atomic64_set(&mm->context.tlb_gen, 0);
     120                 :            : 
     121                 :            : #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
     122                 :            :         if (cpu_feature_enabled(X86_FEATURE_OSPKE)) {
     123                 :            :                 /* pkey 0 is the default and allocated implicitly */
     124                 :            :                 mm->context.pkey_allocation_map = 0x1;
     125                 :            :                 /* -1 means unallocated or invalid */
     126                 :            :                 mm->context.execute_only_pkey = -1;
     127                 :            :         }
     128                 :            : #endif
     129                 :            :         init_new_context_ldt(mm);
     130                 :            :         return 0;
     131                 :            : }
     132                 :       7568 : static inline void destroy_context(struct mm_struct *mm)
     133                 :            : {
     134                 :       7568 :         destroy_context_ldt(mm);
     135                 :            : }
     136                 :            : 
     137                 :            : extern void switch_mm(struct mm_struct *prev, struct mm_struct *next,
     138                 :            :                       struct task_struct *tsk);
     139                 :            : 
     140                 :            : extern void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
     141                 :            :                                struct task_struct *tsk);
     142                 :            : #define switch_mm_irqs_off switch_mm_irqs_off
     143                 :            : 
     144                 :            : #define activate_mm(prev, next)                 \
     145                 :            : do {                                            \
     146                 :            :         paravirt_activate_mm((prev), (next));   \
     147                 :            :         switch_mm((prev), (next), NULL);        \
     148                 :            : } while (0);
     149                 :            : 
     150                 :            : #ifdef CONFIG_X86_32
     151                 :            : #define deactivate_mm(tsk, mm)                  \
     152                 :            : do {                                            \
     153                 :            :         lazy_load_gs(0);                        \
     154                 :            : } while (0)
     155                 :            : #else
     156                 :            : #define deactivate_mm(tsk, mm)                  \
     157                 :            : do {                                            \
     158                 :            :         load_gs_index(0);                       \
     159                 :            :         loadsegment(fs, 0);                     \
     160                 :            : } while (0)
     161                 :            : #endif
     162                 :            : 
     163                 :       3960 : static inline void arch_dup_pkeys(struct mm_struct *oldmm,
     164                 :            :                                   struct mm_struct *mm)
     165                 :            : {
     166                 :            : #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
     167      [ -  -  + ]:       3960 :         if (!cpu_feature_enabled(X86_FEATURE_OSPKE))
     168                 :            :                 return;
     169                 :            : 
     170                 :            :         /* Duplicate the oldmm pkey state in mm: */
     171                 :          0 :         mm->context.pkey_allocation_map = oldmm->context.pkey_allocation_map;
     172                 :          0 :         mm->context.execute_only_pkey   = oldmm->context.execute_only_pkey;
     173                 :            : #endif
     174                 :            : }
     175                 :            : 
     176                 :       3960 : static inline int arch_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
     177                 :            : {
     178                 :       3960 :         arch_dup_pkeys(oldmm, mm);
     179                 :       3960 :         paravirt_arch_dup_mmap(oldmm, mm);
     180                 :       3960 :         return ldt_dup_context(oldmm, mm);
     181                 :            : }
     182                 :            : 
     183                 :      17864 : static inline void arch_exit_mmap(struct mm_struct *mm)
     184                 :            : {
     185                 :      17864 :         paravirt_arch_exit_mmap(mm);
     186                 :      17864 :         ldt_arch_exit_mmap(mm);
     187                 :      17864 : }
     188                 :            : 
     189                 :            : #ifdef CONFIG_X86_64
     190                 :          0 : static inline bool is_64bit_mm(struct mm_struct *mm)
     191                 :            : {
     192                 :          0 :         return  !IS_ENABLED(CONFIG_IA32_EMULATION) ||
     193                 :          0 :                 !(mm->context.ia32_compat == TIF_IA32);
     194                 :            : }
     195                 :            : #else
     196                 :            : static inline bool is_64bit_mm(struct mm_struct *mm)
     197                 :            : {
     198                 :            :         return false;
     199                 :            : }
     200                 :            : #endif
     201                 :            : 
     202                 :     133268 : static inline void arch_unmap(struct mm_struct *mm, unsigned long start,
     203                 :            :                               unsigned long end)
     204                 :            : {
     205                 :     133268 : }
     206                 :            : 
     207                 :            : /*
     208                 :            :  * We only want to enforce protection keys on the current process
     209                 :            :  * because we effectively have no access to PKRU for other
     210                 :            :  * processes or any way to tell *which * PKRU in a threaded
     211                 :            :  * process we could use.
     212                 :            :  *
     213                 :            :  * So do not enforce things if the VMA is not from the current
     214                 :            :  * mm, or if we are in a kernel thread.
     215                 :            :  */
     216                 :    4783856 : static inline bool vma_is_foreign(struct vm_area_struct *vma)
     217                 :            : {
     218         [ +  - ]:    4783856 :         if (!current->mm)
     219                 :            :                 return true;
     220                 :            :         /*
     221                 :            :          * Should PKRU be enforced on the access to this VMA?  If
     222                 :            :          * the VMA is from another process, then PKRU has no
     223                 :            :          * relevance and should not be enforced.
     224                 :            :          */
     225         [ +  - ]:    4783856 :         if (current->mm != vma->vm_mm)
     226                 :            :                 return true;
     227                 :            : 
     228                 :            :         return false;
     229                 :            : }
     230                 :            : 
     231                 :    6692740 : static inline bool arch_vma_access_permitted(struct vm_area_struct *vma,
     232                 :            :                 bool write, bool execute, bool foreign)
     233                 :            : {
     234                 :            :         /* pkeys never affect instruction fetches */
     235         [ +  + ]:    6692740 :         if (execute)
     236                 :            :                 return true;
     237                 :            :         /* allow access if the VMA is not one from this process */
     238         [ +  + ]:    4820684 :         if (foreign || vma_is_foreign(vma))
     239                 :            :                 return true;
     240                 :    4783856 :         return __pkru_allows_pkey(vma_pkey(vma), write);
     241                 :            : }
     242                 :            : 
     243                 :            : /*
     244                 :            :  * This can be used from process context to figure out what the value of
     245                 :            :  * CR3 is without needing to do a (slow) __read_cr3().
     246                 :            :  *
     247                 :            :  * It's intended to be used for code like KVM that sneakily changes CR3
     248                 :            :  * and needs to restore it.  It needs to be used very carefully.
     249                 :            :  */
     250                 :            : static inline unsigned long __get_current_cr3_fast(void)
     251                 :            : {
     252                 :            :         unsigned long cr3 = build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd,
     253                 :            :                 this_cpu_read(cpu_tlbstate.loaded_mm_asid));
     254                 :            : 
     255                 :            :         /* For now, be very restrictive about when this can be called. */
     256                 :            :         VM_WARN_ON(in_nmi() || preemptible());
     257                 :            : 
     258                 :            :         VM_BUG_ON(cr3 != __read_cr3());
     259                 :            :         return cr3;
     260                 :            : }
     261                 :            : 
     262                 :            : typedef struct {
     263                 :            :         struct mm_struct *mm;
     264                 :            : } temp_mm_state_t;
     265                 :            : 
     266                 :            : /*
     267                 :            :  * Using a temporary mm allows to set temporary mappings that are not accessible
     268                 :            :  * by other CPUs. Such mappings are needed to perform sensitive memory writes
     269                 :            :  * that override the kernel memory protections (e.g., W^X), without exposing the
     270                 :            :  * temporary page-table mappings that are required for these write operations to
     271                 :            :  * other CPUs. Using a temporary mm also allows to avoid TLB shootdowns when the
     272                 :            :  * mapping is torn down.
     273                 :            :  *
     274                 :            :  * Context: The temporary mm needs to be used exclusively by a single core. To
     275                 :            :  *          harden security IRQs must be disabled while the temporary mm is
     276                 :            :  *          loaded, thereby preventing interrupt handler bugs from overriding
     277                 :            :  *          the kernel memory protection.
     278                 :            :  */
     279                 :     109494 : static inline temp_mm_state_t use_temporary_mm(struct mm_struct *mm)
     280                 :            : {
     281                 :     109494 :         temp_mm_state_t temp_state;
     282                 :            : 
     283                 :     109494 :         lockdep_assert_irqs_disabled();
     284                 :     109494 :         temp_state.mm = this_cpu_read(cpu_tlbstate.loaded_mm);
     285                 :     109494 :         switch_mm_irqs_off(NULL, mm, current);
     286                 :            : 
     287                 :            :         /*
     288                 :            :          * If breakpoints are enabled, disable them while the temporary mm is
     289                 :            :          * used. Userspace might set up watchpoints on addresses that are used
     290                 :            :          * in the temporary mm, which would lead to wrong signals being sent or
     291                 :            :          * crashes.
     292                 :            :          *
     293                 :            :          * Note that breakpoints are not disabled selectively, which also causes
     294                 :            :          * kernel breakpoints (e.g., perf's) to be disabled. This might be
     295                 :            :          * undesirable, but still seems reasonable as the code that runs in the
     296                 :            :          * temporary mm should be short.
     297                 :            :          */
     298         [ -  + ]:     109494 :         if (hw_breakpoint_active())
     299                 :          0 :                 hw_breakpoint_disable();
     300                 :            : 
     301                 :     109494 :         return temp_state;
     302                 :            : }
     303                 :            : 
     304                 :     109494 : static inline void unuse_temporary_mm(temp_mm_state_t prev_state)
     305                 :            : {
     306                 :     109494 :         lockdep_assert_irqs_disabled();
     307                 :     109494 :         switch_mm_irqs_off(NULL, prev_state.mm, current);
     308                 :            : 
     309                 :            :         /*
     310                 :            :          * Restore the breakpoints if they were disabled before the temporary mm
     311                 :            :          * was loaded.
     312                 :            :          */
     313         [ -  + ]:     109494 :         if (hw_breakpoint_active())
     314                 :          0 :                 hw_breakpoint_restore();
     315                 :     109494 : }
     316                 :            : 
     317                 :            : #endif /* _ASM_X86_MMU_CONTEXT_H */

Generated by: LCOV version 1.14