LCOV - code coverage report
Current view: top level - mm - mmap.c (source / functions) Hit Total Coverage
Test: combined.info Lines: 862 1298 66.4 %
Date: 2022-03-28 16:04:14 Functions: 60 82 73.2 %
Branches: 528 1103 47.9 %

           Branch data     Line data    Source code
       1                 :            : // SPDX-License-Identifier: GPL-2.0-only
       2                 :            : /*
       3                 :            :  * mm/mmap.c
       4                 :            :  *
       5                 :            :  * Written by obz.
       6                 :            :  *
       7                 :            :  * Address space accounting code        <alan@lxorguk.ukuu.org.uk>
       8                 :            :  */
       9                 :            : 
      10                 :            : #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
      11                 :            : 
      12                 :            : #include <linux/kernel.h>
      13                 :            : #include <linux/slab.h>
      14                 :            : #include <linux/backing-dev.h>
      15                 :            : #include <linux/mm.h>
      16                 :            : #include <linux/vmacache.h>
      17                 :            : #include <linux/shm.h>
      18                 :            : #include <linux/mman.h>
      19                 :            : #include <linux/pagemap.h>
      20                 :            : #include <linux/swap.h>
      21                 :            : #include <linux/syscalls.h>
      22                 :            : #include <linux/capability.h>
      23                 :            : #include <linux/init.h>
      24                 :            : #include <linux/file.h>
      25                 :            : #include <linux/fs.h>
      26                 :            : #include <linux/personality.h>
      27                 :            : #include <linux/security.h>
      28                 :            : #include <linux/hugetlb.h>
      29                 :            : #include <linux/shmem_fs.h>
      30                 :            : #include <linux/profile.h>
      31                 :            : #include <linux/export.h>
      32                 :            : #include <linux/mount.h>
      33                 :            : #include <linux/mempolicy.h>
      34                 :            : #include <linux/rmap.h>
      35                 :            : #include <linux/mmu_notifier.h>
      36                 :            : #include <linux/mmdebug.h>
      37                 :            : #include <linux/perf_event.h>
      38                 :            : #include <linux/audit.h>
      39                 :            : #include <linux/khugepaged.h>
      40                 :            : #include <linux/uprobes.h>
      41                 :            : #include <linux/rbtree_augmented.h>
      42                 :            : #include <linux/notifier.h>
      43                 :            : #include <linux/memory.h>
      44                 :            : #include <linux/printk.h>
      45                 :            : #include <linux/userfaultfd_k.h>
      46                 :            : #include <linux/moduleparam.h>
      47                 :            : #include <linux/pkeys.h>
      48                 :            : #include <linux/oom.h>
      49                 :            : #include <linux/sched/mm.h>
      50                 :            : 
      51                 :            : #include <linux/uaccess.h>
      52                 :            : #include <asm/cacheflush.h>
      53                 :            : #include <asm/tlb.h>
      54                 :            : #include <asm/mmu_context.h>
      55                 :            : 
      56                 :            : #include "internal.h"
      57                 :            : 
      58                 :            : #ifndef arch_mmap_check
      59                 :            : #define arch_mmap_check(addr, len, flags)       (0)
      60                 :            : #endif
      61                 :            : 
      62                 :            : #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
      63                 :            : const int mmap_rnd_bits_min = CONFIG_ARCH_MMAP_RND_BITS_MIN;
      64                 :            : const int mmap_rnd_bits_max = CONFIG_ARCH_MMAP_RND_BITS_MAX;
      65                 :            : int mmap_rnd_bits __read_mostly = CONFIG_ARCH_MMAP_RND_BITS;
      66                 :            : #endif
      67                 :            : #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
      68                 :            : const int mmap_rnd_compat_bits_min = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MIN;
      69                 :            : const int mmap_rnd_compat_bits_max = CONFIG_ARCH_MMAP_RND_COMPAT_BITS_MAX;
      70                 :            : int mmap_rnd_compat_bits __read_mostly = CONFIG_ARCH_MMAP_RND_COMPAT_BITS;
      71                 :            : #endif
      72                 :            : 
      73                 :            : static bool ignore_rlimit_data;
      74                 :            : core_param(ignore_rlimit_data, ignore_rlimit_data, bool, 0644);
      75                 :            : 
      76                 :            : static void unmap_region(struct mm_struct *mm,
      77                 :            :                 struct vm_area_struct *vma, struct vm_area_struct *prev,
      78                 :            :                 unsigned long start, unsigned long end);
      79                 :            : 
      80                 :            : /* description of effects of mapping type and prot in current implementation.
      81                 :            :  * this is due to the limited x86 page protection hardware.  The expected
      82                 :            :  * behavior is in parens:
      83                 :            :  *
      84                 :            :  * map_type     prot
      85                 :            :  *              PROT_NONE       PROT_READ       PROT_WRITE      PROT_EXEC
      86                 :            :  * MAP_SHARED   r: (no) no      r: (yes) yes    r: (no) yes     r: (no) yes
      87                 :            :  *              w: (no) no      w: (no) no      w: (yes) yes    w: (no) no
      88                 :            :  *              x: (no) no      x: (no) yes     x: (no) yes     x: (yes) yes
      89                 :            :  *
      90                 :            :  * MAP_PRIVATE  r: (no) no      r: (yes) yes    r: (no) yes     r: (no) yes
      91                 :            :  *              w: (no) no      w: (no) no      w: (copy) copy  w: (no) no
      92                 :            :  *              x: (no) no      x: (no) yes     x: (no) yes     x: (yes) yes
      93                 :            :  */
      94                 :            : pgprot_t protection_map[16] __ro_after_init = {
      95                 :            :         __P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111,
      96                 :            :         __S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111
      97                 :            : };
      98                 :            : 
      99                 :            : #ifndef CONFIG_ARCH_HAS_FILTER_PGPROT
     100                 :            : static inline pgprot_t arch_filter_pgprot(pgprot_t prot)
     101                 :            : {
     102                 :            :         return prot;
     103                 :            : }
     104                 :            : #endif
     105                 :            : 
     106                 :     829130 : pgprot_t vm_get_page_prot(unsigned long vm_flags)
     107                 :            : {
     108                 :     829130 :         pgprot_t ret = __pgprot(pgprot_val(protection_map[vm_flags &
     109                 :            :                                 (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]) |
     110                 :            :                         pgprot_val(arch_vm_get_page_prot(vm_flags)));
     111                 :            : 
     112         [ +  - ]:      10686 :         return arch_filter_pgprot(ret);
     113                 :            : }
     114                 :            : EXPORT_SYMBOL(vm_get_page_prot);
     115                 :            : 
     116                 :     439651 : static pgprot_t vm_pgprot_modify(pgprot_t oldprot, unsigned long vm_flags)
     117                 :            : {
     118                 :     439651 :         return pgprot_modify(oldprot, vm_get_page_prot(vm_flags));
     119                 :            : }
     120                 :            : 
     121                 :            : /* Update vma->vm_page_prot to reflect vma->vm_flags. */
     122                 :     439599 : void vma_set_page_prot(struct vm_area_struct *vma)
     123                 :            : {
     124                 :     439599 :         unsigned long vm_flags = vma->vm_flags;
     125                 :     439599 :         pgprot_t vm_page_prot;
     126                 :            : 
     127         [ +  + ]:     439599 :         vm_page_prot = vm_pgprot_modify(vma->vm_page_prot, vm_flags);
     128         [ -  + ]:     439599 :         if (vma_wants_writenotify(vma, vm_page_prot)) {
     129                 :          0 :                 vm_flags &= ~VM_SHARED;
     130         [ #  # ]:          0 :                 vm_page_prot = vm_pgprot_modify(vm_page_prot, vm_flags);
     131                 :            :         }
     132                 :            :         /* remove_protection_ptes reads vma->vm_page_prot without mmap_sem */
     133                 :     439599 :         WRITE_ONCE(vma->vm_page_prot, vm_page_prot);
     134                 :     439599 : }
     135                 :            : 
     136                 :            : /*
     137                 :            :  * Requires inode->i_mapping->i_mmap_rwsem
     138                 :            :  */
     139                 :    1034137 : static void __remove_shared_vm_struct(struct vm_area_struct *vma,
     140                 :            :                 struct file *file, struct address_space *mapping)
     141                 :            : {
     142         [ +  + ]:    1034137 :         if (vma->vm_flags & VM_DENYWRITE)
     143                 :     232960 :                 atomic_inc(&file_inode(file)->i_writecount);
     144         [ +  + ]:    1034137 :         if (vma->vm_flags & VM_SHARED)
     145                 :         65 :                 mapping_unmap_writable(mapping);
     146                 :            : 
     147                 :    1034137 :         flush_dcache_mmap_lock(mapping);
     148                 :    1034137 :         vma_interval_tree_remove(vma, &mapping->i_mmap);
     149                 :    1034137 :         flush_dcache_mmap_unlock(mapping);
     150                 :    1034137 : }
     151                 :            : 
     152                 :            : /*
     153                 :            :  * Unlink a file-based vm structure from its interval tree, to hide
     154                 :            :  * vma from rmap and vmtruncate before freeing its page tables.
     155                 :            :  */
     156                 :    1226332 : void unlink_file_vma(struct vm_area_struct *vma)
     157                 :            : {
     158                 :    1226332 :         struct file *file = vma->vm_file;
     159                 :            : 
     160         [ +  + ]:    1226332 :         if (file) {
     161                 :    1034137 :                 struct address_space *mapping = file->f_mapping;
     162                 :    1034137 :                 i_mmap_lock_write(mapping);
     163                 :    1034137 :                 __remove_shared_vm_struct(vma, file, mapping);
     164                 :    1034137 :                 i_mmap_unlock_write(mapping);
     165                 :            :         }
     166                 :    1226332 : }
     167                 :            : 
     168                 :            : /*
     169                 :            :  * Close a vm structure and free it, returning the next.
     170                 :            :  */
     171                 :    1226332 : static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
     172                 :            : {
     173                 :    1226332 :         struct vm_area_struct *next = vma->vm_next;
     174                 :            : 
     175                 :    1226332 :         might_sleep();
     176   [ +  +  +  + ]:    1226332 :         if (vma->vm_ops && vma->vm_ops->close)
     177                 :      42328 :                 vma->vm_ops->close(vma);
     178         [ +  + ]:    1226332 :         if (vma->vm_file)
     179                 :    1034137 :                 fput(vma->vm_file);
     180         [ -  + ]:    1226332 :         mpol_put(vma_policy(vma));
     181                 :    1226332 :         vm_area_free(vma);
     182                 :    1226332 :         return next;
     183                 :            : }
     184                 :            : 
     185                 :            : static int do_brk_flags(unsigned long addr, unsigned long request, unsigned long flags,
     186                 :            :                 struct list_head *uf);
     187                 :      65062 : SYSCALL_DEFINE1(brk, unsigned long, brk)
     188                 :            : {
     189                 :      32531 :         unsigned long retval;
     190                 :      32531 :         unsigned long newbrk, oldbrk, origbrk;
     191                 :      32531 :         struct mm_struct *mm = current->mm;
     192                 :      32531 :         struct vm_area_struct *next;
     193                 :      32531 :         unsigned long min_brk;
     194                 :      32531 :         bool populate;
     195                 :      32531 :         bool downgraded = false;
     196                 :      32531 :         LIST_HEAD(uf);
     197                 :            : 
     198         [ +  - ]:      32531 :         if (down_write_killable(&mm->mmap_sem))
     199                 :            :                 return -EINTR;
     200                 :            : 
     201                 :      32531 :         origbrk = mm->brk;
     202                 :            : 
     203                 :            : #ifdef CONFIG_COMPAT_BRK
     204                 :            :         /*
     205                 :            :          * CONFIG_COMPAT_BRK can still be overridden by setting
     206                 :            :          * randomize_va_space to 2, which will still cause mm->start_brk
     207                 :            :          * to be arbitrarily shifted
     208                 :            :          */
     209                 :            :         if (current->brk_randomized)
     210                 :            :                 min_brk = mm->start_brk;
     211                 :            :         else
     212                 :            :                 min_brk = mm->end_data;
     213                 :            : #else
     214                 :      32531 :         min_brk = mm->start_brk;
     215                 :            : #endif
     216         [ +  + ]:      32531 :         if (brk < min_brk)
     217                 :      21346 :                 goto out;
     218                 :            : 
     219                 :            :         /*
     220                 :            :          * Check against rlimit here. If this check is done later after the test
     221                 :            :          * of oldbrk with newbrk then it can escape the test and let the data
     222                 :            :          * segment grow beyond its set limit the in case where the limit is
     223                 :            :          * not page aligned -Ram Gupta
     224                 :            :          */
     225         [ -  + ]:      11185 :         if (check_data_rlimit(rlimit(RLIMIT_DATA), brk, mm->start_brk,
     226                 :            :                               mm->end_data, mm->start_data))
     227                 :          0 :                 goto out;
     228                 :            : 
     229                 :      11185 :         newbrk = PAGE_ALIGN(brk);
     230                 :      11185 :         oldbrk = PAGE_ALIGN(mm->brk);
     231         [ -  + ]:      11185 :         if (oldbrk == newbrk) {
     232                 :          0 :                 mm->brk = brk;
     233                 :          0 :                 goto success;
     234                 :            :         }
     235                 :            : 
     236                 :            :         /*
     237                 :            :          * Always allow shrinking brk.
     238                 :            :          * __do_munmap() may downgrade mmap_sem to read.
     239                 :            :          */
     240         [ +  + ]:      11185 :         if (brk <= mm->brk) {
     241                 :        275 :                 int ret;
     242                 :            : 
     243                 :            :                 /*
     244                 :            :                  * mm->brk must to be protected by write mmap_sem so update it
     245                 :            :                  * before downgrading mmap_sem. When __do_munmap() fails,
     246                 :            :                  * mm->brk will be restored from origbrk.
     247                 :            :                  */
     248                 :        275 :                 mm->brk = brk;
     249                 :        275 :                 ret = __do_munmap(mm, newbrk, oldbrk-newbrk, &uf, true);
     250         [ -  + ]:        275 :                 if (ret < 0) {
     251                 :          0 :                         mm->brk = origbrk;
     252                 :          0 :                         goto out;
     253         [ +  - ]:        275 :                 } else if (ret == 1) {
     254                 :        275 :                         downgraded = true;
     255                 :            :                 }
     256                 :        275 :                 goto success;
     257                 :            :         }
     258                 :            : 
     259                 :            :         /* Check against existing mmap mappings. */
     260                 :      10910 :         next = find_vma(mm, oldbrk);
     261   [ +  -  -  +  :      10910 :         if (next && newbrk + PAGE_SIZE > vm_start_gap(next))
                   -  + ]
     262                 :          0 :                 goto out;
     263                 :            : 
     264                 :            :         /* Ok, looks good - let it rip. */
     265         [ -  + ]:      10910 :         if (do_brk_flags(oldbrk, newbrk-oldbrk, 0, &uf) < 0)
     266                 :          0 :                 goto out;
     267                 :      10910 :         mm->brk = brk;
     268                 :            : 
     269                 :      11185 : success:
     270   [ +  +  +  - ]:      11185 :         populate = newbrk > oldbrk && (mm->def_flags & VM_LOCKED) != 0;
     271         [ +  + ]:      11185 :         if (downgraded)
     272                 :        275 :                 up_read(&mm->mmap_sem);
     273                 :            :         else
     274                 :      10910 :                 up_write(&mm->mmap_sem);
     275         [ -  + ]:      11185 :         userfaultfd_unmap_complete(mm, &uf);
     276         [ -  + ]:      11185 :         if (populate)
     277                 :          0 :                 mm_populate(oldbrk, newbrk - oldbrk);
     278                 :      11185 :         return brk;
     279                 :            : 
     280                 :      21346 : out:
     281                 :      21346 :         retval = origbrk;
     282                 :      21346 :         up_write(&mm->mmap_sem);
     283                 :      21346 :         return retval;
     284                 :            : }
     285                 :            : 
     286                 :    5496891 : static inline unsigned long vma_compute_gap(struct vm_area_struct *vma)
     287                 :            : {
     288                 :    5496891 :         unsigned long gap, prev_end;
     289                 :            : 
     290                 :            :         /*
     291                 :            :          * Note: in the rare case of a VM_GROWSDOWN above a VM_GROWSUP, we
     292                 :            :          * allow two stack_guard_gaps between them here, and when choosing
     293                 :            :          * an unmapped area; whereas when expanding we only require one.
     294                 :            :          * That's a little inconsistent, but keeps the code here simpler.
     295                 :            :          */
     296                 :    5496891 :         gap = vm_start_gap(vma);
     297         [ +  + ]:    5496891 :         if (vma->vm_prev) {
     298         [ +  + ]:    5411195 :                 prev_end = vm_end_gap(vma->vm_prev);
     299         [ +  + ]:    5411195 :                 if (gap > prev_end)
     300                 :    1399583 :                         gap -= prev_end;
     301                 :            :                 else
     302                 :            :                         gap = 0;
     303                 :            :         }
     304                 :    5496891 :         return gap;
     305                 :            : }
     306                 :            : 
     307                 :            : #ifdef CONFIG_DEBUG_VM_RB
     308                 :            : static unsigned long vma_compute_subtree_gap(struct vm_area_struct *vma)
     309                 :            : {
     310                 :            :         unsigned long max = vma_compute_gap(vma), subtree_gap;
     311                 :            :         if (vma->vm_rb.rb_left) {
     312                 :            :                 subtree_gap = rb_entry(vma->vm_rb.rb_left,
     313                 :            :                                 struct vm_area_struct, vm_rb)->rb_subtree_gap;
     314                 :            :                 if (subtree_gap > max)
     315                 :            :                         max = subtree_gap;
     316                 :            :         }
     317                 :            :         if (vma->vm_rb.rb_right) {
     318                 :            :                 subtree_gap = rb_entry(vma->vm_rb.rb_right,
     319                 :            :                                 struct vm_area_struct, vm_rb)->rb_subtree_gap;
     320                 :            :                 if (subtree_gap > max)
     321                 :            :                         max = subtree_gap;
     322                 :            :         }
     323                 :            :         return max;
     324                 :            : }
     325                 :            : 
     326                 :            : static int browse_rb(struct mm_struct *mm)
     327                 :            : {
     328                 :            :         struct rb_root *root = &mm->mm_rb;
     329                 :            :         int i = 0, j, bug = 0;
     330                 :            :         struct rb_node *nd, *pn = NULL;
     331                 :            :         unsigned long prev = 0, pend = 0;
     332                 :            : 
     333                 :            :         for (nd = rb_first(root); nd; nd = rb_next(nd)) {
     334                 :            :                 struct vm_area_struct *vma;
     335                 :            :                 vma = rb_entry(nd, struct vm_area_struct, vm_rb);
     336                 :            :                 if (vma->vm_start < prev) {
     337                 :            :                         pr_emerg("vm_start %lx < prev %lx\n",
     338                 :            :                                   vma->vm_start, prev);
     339                 :            :                         bug = 1;
     340                 :            :                 }
     341                 :            :                 if (vma->vm_start < pend) {
     342                 :            :                         pr_emerg("vm_start %lx < pend %lx\n",
     343                 :            :                                   vma->vm_start, pend);
     344                 :            :                         bug = 1;
     345                 :            :                 }
     346                 :            :                 if (vma->vm_start > vma->vm_end) {
     347                 :            :                         pr_emerg("vm_start %lx > vm_end %lx\n",
     348                 :            :                                   vma->vm_start, vma->vm_end);
     349                 :            :                         bug = 1;
     350                 :            :                 }
     351                 :            :                 spin_lock(&mm->page_table_lock);
     352                 :            :                 if (vma->rb_subtree_gap != vma_compute_subtree_gap(vma)) {
     353                 :            :                         pr_emerg("free gap %lx, correct %lx\n",
     354                 :            :                                vma->rb_subtree_gap,
     355                 :            :                                vma_compute_subtree_gap(vma));
     356                 :            :                         bug = 1;
     357                 :            :                 }
     358                 :            :                 spin_unlock(&mm->page_table_lock);
     359                 :            :                 i++;
     360                 :            :                 pn = nd;
     361                 :            :                 prev = vma->vm_start;
     362                 :            :                 pend = vma->vm_end;
     363                 :            :         }
     364                 :            :         j = 0;
     365                 :            :         for (nd = pn; nd; nd = rb_prev(nd))
     366                 :            :                 j++;
     367                 :            :         if (i != j) {
     368                 :            :                 pr_emerg("backwards %d, forwards %d\n", j, i);
     369                 :            :                 bug = 1;
     370                 :            :         }
     371                 :            :         return bug ? -1 : i;
     372                 :            : }
     373                 :            : 
     374                 :            : static void validate_mm_rb(struct rb_root *root, struct vm_area_struct *ignore)
     375                 :            : {
     376                 :            :         struct rb_node *nd;
     377                 :            : 
     378                 :            :         for (nd = rb_first(root); nd; nd = rb_next(nd)) {
     379                 :            :                 struct vm_area_struct *vma;
     380                 :            :                 vma = rb_entry(nd, struct vm_area_struct, vm_rb);
     381                 :            :                 VM_BUG_ON_VMA(vma != ignore &&
     382                 :            :                         vma->rb_subtree_gap != vma_compute_subtree_gap(vma),
     383                 :            :                         vma);
     384                 :            :         }
     385                 :            : }
     386                 :            : 
     387                 :            : static void validate_mm(struct mm_struct *mm)
     388                 :            : {
     389                 :            :         int bug = 0;
     390                 :            :         int i = 0;
     391                 :            :         unsigned long highest_address = 0;
     392                 :            :         struct vm_area_struct *vma = mm->mmap;
     393                 :            : 
     394                 :            :         while (vma) {
     395                 :            :                 struct anon_vma *anon_vma = vma->anon_vma;
     396                 :            :                 struct anon_vma_chain *avc;
     397                 :            : 
     398                 :            :                 if (anon_vma) {
     399                 :            :                         anon_vma_lock_read(anon_vma);
     400                 :            :                         list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
     401                 :            :                                 anon_vma_interval_tree_verify(avc);
     402                 :            :                         anon_vma_unlock_read(anon_vma);
     403                 :            :                 }
     404                 :            : 
     405                 :            :                 highest_address = vm_end_gap(vma);
     406                 :            :                 vma = vma->vm_next;
     407                 :            :                 i++;
     408                 :            :         }
     409                 :            :         if (i != mm->map_count) {
     410                 :            :                 pr_emerg("map_count %d vm_next %d\n", mm->map_count, i);
     411                 :            :                 bug = 1;
     412                 :            :         }
     413                 :            :         if (highest_address != mm->highest_vm_end) {
     414                 :            :                 pr_emerg("mm->highest_vm_end %lx, found %lx\n",
     415                 :            :                           mm->highest_vm_end, highest_address);
     416                 :            :                 bug = 1;
     417                 :            :         }
     418                 :            :         i = browse_rb(mm);
     419                 :            :         if (i != mm->map_count) {
     420                 :            :                 if (i != -1)
     421                 :            :                         pr_emerg("map_count %d rb %d\n", mm->map_count, i);
     422                 :            :                 bug = 1;
     423                 :            :         }
     424                 :            :         VM_BUG_ON_MM(bug, mm);
     425                 :            : }
     426                 :            : #else
     427                 :            : #define validate_mm_rb(root, ignore) do { } while (0)
     428                 :            : #define validate_mm(mm) do { } while (0)
     429                 :            : #endif
     430                 :            : 
     431   [ +  +  +  +  :   16602564 : RB_DECLARE_CALLBACKS_MAX(static, vma_gap_callbacks,
          +  +  +  +  +  
                      + ]
     432                 :            :                          struct vm_area_struct, vm_rb,
     433                 :            :                          unsigned long, rb_subtree_gap, vma_compute_gap)
     434                 :            : 
     435                 :            : /*
     436                 :            :  * Update augmented rbtree rb_subtree_gap values after vma->vm_start or
     437                 :            :  * vma->vm_prev->vm_end values changed, without modifying the vma's position
     438                 :            :  * in the rbtree.
     439                 :            :  */
     440                 :    2072841 : static void vma_gap_update(struct vm_area_struct *vma)
     441                 :            : {
     442                 :            :         /*
     443                 :            :          * As it turns out, RB_DECLARE_CALLBACKS_MAX() already created
     444                 :            :          * a callback function that does exactly what we want.
     445                 :            :          */
     446                 :    2072841 :         vma_gap_callbacks_propagate(&vma->vm_rb, NULL);
     447                 :     805130 : }
     448                 :            : 
     449                 :    1257025 : static inline void vma_rb_insert(struct vm_area_struct *vma,
     450                 :            :                                  struct rb_root *root)
     451                 :            : {
     452                 :            :         /* All rb_subtree_gap values must be consistent prior to insertion */
     453                 :    1257025 :         validate_mm_rb(root, NULL);
     454                 :            : 
     455                 :    1257025 :         rb_insert_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
     456                 :            : }
     457                 :            : 
     458                 :     158086 : static void __vma_rb_erase(struct vm_area_struct *vma, struct rb_root *root)
     459                 :            : {
     460                 :            :         /*
     461                 :            :          * Note rb_erase_augmented is a fairly large inline function,
     462                 :            :          * so make sure we instantiate it only once with our desired
     463                 :            :          * augmented rbtree callbacks.
     464                 :            :          */
     465         [ +  + ]:     158086 :         rb_erase_augmented(&vma->vm_rb, root, &vma_gap_callbacks);
     466                 :     158086 : }
     467                 :            : 
     468                 :          0 : static __always_inline void vma_rb_erase_ignore(struct vm_area_struct *vma,
     469                 :            :                                                 struct rb_root *root,
     470                 :            :                                                 struct vm_area_struct *ignore)
     471                 :            : {
     472                 :            :         /*
     473                 :            :          * All rb_subtree_gap values must be consistent prior to erase,
     474                 :            :          * with the possible exception of the "next" vma being erased if
     475                 :            :          * next->vm_start was reduced.
     476                 :            :          */
     477                 :          0 :         validate_mm_rb(root, ignore);
     478                 :            : 
     479                 :          0 :         __vma_rb_erase(vma, root);
     480                 :            : }
     481                 :            : 
     482                 :     158086 : static __always_inline void vma_rb_erase(struct vm_area_struct *vma,
     483                 :            :                                          struct rb_root *root)
     484                 :            : {
     485                 :            :         /*
     486                 :            :          * All rb_subtree_gap values must be consistent prior to erase,
     487                 :            :          * with the possible exception of the vma being erased.
     488                 :            :          */
     489                 :     158086 :         validate_mm_rb(root, vma);
     490                 :            : 
     491                 :     158086 :         __vma_rb_erase(vma, root);
     492                 :            : }
     493                 :            : 
     494                 :            : /*
     495                 :            :  * vma has some anon_vma assigned, and is already inserted on that
     496                 :            :  * anon_vma's interval trees.
     497                 :            :  *
     498                 :            :  * Before updating the vma's vm_start / vm_end / vm_pgoff fields, the
     499                 :            :  * vma must be removed from the anon_vma's interval trees using
     500                 :            :  * anon_vma_interval_tree_pre_update_vma().
     501                 :            :  *
     502                 :            :  * After the update, the vma will be reinserted using
     503                 :            :  * anon_vma_interval_tree_post_update_vma().
     504                 :            :  *
     505                 :            :  * The entire update must be protected by exclusive mmap_sem and by
     506                 :            :  * the root anon_vma's mutex.
     507                 :            :  */
     508                 :            : static inline void
     509                 :     104147 : anon_vma_interval_tree_pre_update_vma(struct vm_area_struct *vma)
     510                 :            : {
     511                 :     104147 :         struct anon_vma_chain *avc;
     512                 :            : 
     513         [ +  + ]:     207511 :         list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
     514                 :     103364 :                 anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root);
     515                 :     104147 : }
     516                 :            : 
     517                 :            : static inline void
     518                 :     104147 : anon_vma_interval_tree_post_update_vma(struct vm_area_struct *vma)
     519                 :            : {
     520                 :     104147 :         struct anon_vma_chain *avc;
     521                 :            : 
     522         [ +  + ]:     207511 :         list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
     523                 :     103364 :                 anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
     524                 :     104147 : }
     525                 :            : 
     526                 :     779306 : static int find_vma_links(struct mm_struct *mm, unsigned long addr,
     527                 :            :                 unsigned long end, struct vm_area_struct **pprev,
     528                 :            :                 struct rb_node ***rb_link, struct rb_node **rb_parent)
     529                 :            : {
     530                 :     779306 :         struct rb_node **__rb_link, *__rb_parent, *rb_prev;
     531                 :            : 
     532                 :     779306 :         __rb_link = &mm->mm_rb.rb_node;
     533                 :     779306 :         rb_prev = __rb_parent = NULL;
     534                 :            : 
     535   [ -  -  +  +  :    5184940 :         while (*__rb_link) {
          +  +  +  +  +  
                      + ]
     536                 :    4527158 :                 struct vm_area_struct *vma_tmp;
     537                 :            : 
     538                 :    4527158 :                 __rb_parent = *__rb_link;
     539                 :    4527158 :                 vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb);
     540                 :            : 
     541   [ -  -  +  +  :    4527158 :                 if (vma_tmp->vm_end > addr) {
          +  +  +  +  +  
                      + ]
     542                 :            :                         /* Fail if an existing vma overlaps the area */
     543   [ -  -  +  -  :    2873266 :                         if (vma_tmp->vm_start < end)
          +  -  +  +  +  
                      - ]
     544                 :            :                                 return -ENOMEM;
     545                 :    2751742 :                         __rb_link = &__rb_parent->rb_left;
     546                 :            :                 } else {
     547                 :    1653892 :                         rb_prev = __rb_parent;
     548                 :    1653892 :                         __rb_link = &__rb_parent->rb_right;
     549                 :            :                 }
     550                 :            :         }
     551                 :            : 
     552                 :     657782 :         *pprev = NULL;
     553   [ -  -  +  +  :     657782 :         if (rb_prev)
          +  -  +  +  +  
                      - ]
     554                 :     636410 :                 *pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
     555                 :            :         *rb_link = __rb_link;
     556                 :            :         *rb_parent = __rb_parent;
     557                 :            :         return 0;
     558                 :            : }
     559                 :            : 
     560                 :          0 : static unsigned long count_vma_pages_range(struct mm_struct *mm,
     561                 :            :                 unsigned long addr, unsigned long end)
     562                 :            : {
     563                 :          0 :         unsigned long nr_pages = 0;
     564                 :          0 :         struct vm_area_struct *vma;
     565                 :            : 
     566                 :            :         /* Find first overlaping mapping */
     567                 :          0 :         vma = find_vma_intersection(mm, addr, end);
     568         [ #  # ]:          0 :         if (!vma)
     569                 :            :                 return 0;
     570                 :            : 
     571                 :          0 :         nr_pages = (min(end, vma->vm_end) -
     572                 :          0 :                 max(addr, vma->vm_start)) >> PAGE_SHIFT;
     573                 :            : 
     574                 :            :         /* Iterate over the rest of the overlaps */
     575         [ #  # ]:          0 :         for (vma = vma->vm_next; vma; vma = vma->vm_next) {
     576                 :          0 :                 unsigned long overlap_len;
     577                 :            : 
     578         [ #  # ]:          0 :                 if (vma->vm_start > end)
     579                 :            :                         break;
     580                 :            : 
     581                 :          0 :                 overlap_len = min(end, vma->vm_end) - vma->vm_start;
     582                 :          0 :                 nr_pages += overlap_len >> PAGE_SHIFT;
     583                 :            :         }
     584                 :            : 
     585                 :            :         return nr_pages;
     586                 :            : }
     587                 :            : 
     588                 :    1257025 : void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
     589                 :            :                 struct rb_node **rb_link, struct rb_node *rb_parent)
     590                 :            : {
     591                 :            :         /* Update tracking information for the gap following the new vma. */
     592         [ +  + ]:    1257025 :         if (vma->vm_next)
     593                 :     606635 :                 vma_gap_update(vma->vm_next);
     594                 :            :         else
     595                 :     650390 :                 mm->highest_vm_end = vm_end_gap(vma);
     596                 :            : 
     597                 :            :         /*
     598                 :            :          * vma->vm_prev wasn't known when we followed the rbtree to find the
     599                 :            :          * correct insertion point for that vma. As a result, we could not
     600                 :            :          * update the vma vm_rb parents rb_subtree_gap values on the way down.
     601                 :            :          * So, we first insert the vma with a zero rb_subtree_gap value
     602                 :            :          * (to be consistent with what we did on the way down), and then
     603                 :            :          * immediately update the gap to the correct value. Finally we
     604                 :            :          * rebalance the rbtree after all augmented values have been set.
     605                 :            :          */
     606                 :    1257025 :         rb_link_node(&vma->vm_rb, rb_parent, rb_link);
     607                 :    1257025 :         vma->rb_subtree_gap = 0;
     608                 :    1257025 :         vma_gap_update(vma);
     609                 :    1257025 :         vma_rb_insert(vma, &mm->mm_rb);
     610                 :    1257025 : }
     611                 :            : 
     612                 :     627119 : static void __vma_link_file(struct vm_area_struct *vma)
     613                 :            : {
     614                 :     627119 :         struct file *file;
     615                 :            : 
     616                 :     627119 :         file = vma->vm_file;
     617         [ +  + ]:     627119 :         if (file) {
     618                 :     540306 :                 struct address_space *mapping = file->f_mapping;
     619                 :            : 
     620         [ +  + ]:     540306 :                 if (vma->vm_flags & VM_DENYWRITE)
     621                 :     128180 :                         atomic_dec(&file_inode(file)->i_writecount);
     622         [ +  + ]:     540306 :                 if (vma->vm_flags & VM_SHARED)
     623                 :         52 :                         atomic_inc(&mapping->i_mmap_writable);
     624                 :            : 
     625                 :     540306 :                 flush_dcache_mmap_lock(mapping);
     626                 :     540306 :                 vma_interval_tree_insert(vma, &mapping->i_mmap);
     627                 :     540306 :                 flush_dcache_mmap_unlock(mapping);
     628                 :            :         }
     629                 :     627119 : }
     630                 :            : 
     631                 :            : static void
     632                 :     628007 : __vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
     633                 :            :         struct vm_area_struct *prev, struct rb_node **rb_link,
     634                 :            :         struct rb_node *rb_parent)
     635                 :            : {
     636                 :     628007 :         __vma_link_list(mm, vma, prev);
     637                 :     628007 :         __vma_link_rb(mm, vma, rb_link, rb_parent);
     638                 :            : }
     639                 :            : 
     640                 :     389479 : static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
     641                 :            :                         struct vm_area_struct *prev, struct rb_node **rb_link,
     642                 :            :                         struct rb_node *rb_parent)
     643                 :            : {
     644                 :     389479 :         struct address_space *mapping = NULL;
     645                 :            : 
     646         [ +  + ]:     389479 :         if (vma->vm_file) {
     647                 :     302666 :                 mapping = vma->vm_file->f_mapping;
     648                 :     302666 :                 i_mmap_lock_write(mapping);
     649                 :            :         }
     650                 :            : 
     651                 :     389479 :         __vma_link(mm, vma, prev, rb_link, rb_parent);
     652                 :     389479 :         __vma_link_file(vma);
     653                 :            : 
     654         [ +  + ]:     389479 :         if (mapping)
     655                 :     302666 :                 i_mmap_unlock_write(mapping);
     656                 :            : 
     657                 :     389479 :         mm->map_count++;
     658                 :     389479 :         validate_mm(mm);
     659                 :     389479 : }
     660                 :            : 
     661                 :            : /*
     662                 :            :  * Helper for vma_adjust() in the split_vma insert case: insert a vma into the
     663                 :            :  * mm's list and rbtree.  It has already been inserted into the interval tree.
     664                 :            :  */
     665                 :     238528 : static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
     666                 :            : {
     667                 :     238528 :         struct vm_area_struct *prev;
     668                 :     238528 :         struct rb_node **rb_link, *rb_parent;
     669                 :            : 
     670         [ -  + ]:     477056 :         if (find_vma_links(mm, vma->vm_start, vma->vm_end,
     671                 :            :                            &prev, &rb_link, &rb_parent))
     672                 :          0 :                 BUG();
     673                 :     238528 :         __vma_link(mm, vma, prev, rb_link, rb_parent);
     674                 :     238528 :         mm->map_count++;
     675                 :     238528 : }
     676                 :            : 
     677                 :          0 : static __always_inline void __vma_unlink_common(struct mm_struct *mm,
     678                 :            :                                                 struct vm_area_struct *vma,
     679                 :            :                                                 struct vm_area_struct *ignore)
     680                 :            : {
     681                 :          0 :         vma_rb_erase_ignore(vma, &mm->mm_rb, ignore);
     682                 :          0 :         __vma_unlink_list(mm, vma);
     683                 :            :         /* Kill the cache */
     684                 :          0 :         vmacache_invalidate(mm);
     685                 :            : }
     686                 :            : 
     687                 :            : /*
     688                 :            :  * We cannot adjust vm_start, vm_end, vm_pgoff fields of a vma that
     689                 :            :  * is already present in an i_mmap tree without adjusting the tree.
     690                 :            :  * The following helper function should be used when such adjustments
     691                 :            :  * are necessary.  The "insert" vma (if any) is to be inserted
     692                 :            :  * before we drop the necessary locks.
     693                 :            :  */
     694                 :     290458 : int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
     695                 :            :         unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert,
     696                 :            :         struct vm_area_struct *expand)
     697                 :            : {
     698                 :     290458 :         struct mm_struct *mm = vma->vm_mm;
     699                 :     290458 :         struct vm_area_struct *next = vma->vm_next, *orig_vma = vma;
     700                 :     290458 :         struct address_space *mapping = NULL;
     701                 :     290458 :         struct rb_root_cached *root = NULL;
     702                 :     290458 :         struct anon_vma *anon_vma = NULL;
     703                 :     290458 :         struct file *file = vma->vm_file;
     704                 :     290458 :         bool start_changed = false, end_changed = false;
     705                 :     290458 :         long adjust_next = 0;
     706                 :     290458 :         int remove_next = 0;
     707                 :            : 
     708         [ +  + ]:     290458 :         if (next && !insert) {
     709                 :      30558 :                 struct vm_area_struct *exporter = NULL, *importer = NULL;
     710                 :            : 
     711         [ -  + ]:      30558 :                 if (end >= next->vm_end) {
     712                 :            :                         /*
     713                 :            :                          * vma expands, overlapping all the next, and
     714                 :            :                          * perhaps the one after too (mprotect case 6).
     715                 :            :                          * The only other cases that gets here are
     716                 :            :                          * case 1, case 7 and case 8.
     717                 :            :                          */
     718         [ #  # ]:          0 :                         if (next == expand) {
     719                 :            :                                 /*
     720                 :            :                                  * The only case where we don't expand "vma"
     721                 :            :                                  * and we expand "next" instead is case 8.
     722                 :            :                                  */
     723                 :            :                                 VM_WARN_ON(end != next->vm_end);
     724                 :            :                                 /*
     725                 :            :                                  * remove_next == 3 means we're
     726                 :            :                                  * removing "vma" and that to do so we
     727                 :            :                                  * swapped "vma" and "next".
     728                 :            :                                  */
     729                 :            :                                 remove_next = 3;
     730                 :            :                                 VM_WARN_ON(file != next->vm_file);
     731                 :            :                                 swap(vma, next);
     732                 :            :                         } else {
     733                 :          0 :                                 VM_WARN_ON(expand != vma);
     734                 :            :                                 /*
     735                 :            :                                  * case 1, 6, 7, remove_next == 2 is case 6,
     736                 :            :                                  * remove_next == 1 is case 1 or 7.
     737                 :            :                                  */
     738         [ #  # ]:          0 :                                 remove_next = 1 + (end > next->vm_end);
     739                 :            :                                 VM_WARN_ON(remove_next == 2 &&
     740                 :            :                                            end != next->vm_next->vm_end);
     741                 :            :                                 /* trim end to next, for case 6 first pass */
     742                 :            :                                 end = next->vm_end;
     743                 :            :                         }
     744                 :            : 
     745                 :          0 :                         exporter = next;
     746                 :          0 :                         importer = vma;
     747                 :            : 
     748                 :            :                         /*
     749                 :            :                          * If next doesn't have anon_vma, import from vma after
     750                 :            :                          * next, if the vma overlaps with it.
     751                 :            :                          */
     752   [ #  #  #  # ]:          0 :                         if (remove_next == 2 && !next->anon_vma)
     753                 :          0 :                                 exporter = next->vm_next;
     754                 :            : 
     755         [ +  + ]:      30558 :                 } else if (end > next->vm_start) {
     756                 :            :                         /*
     757                 :            :                          * vma expands, overlapping part of the next:
     758                 :            :                          * mprotect case 5 shifting the boundary up.
     759                 :            :                          */
     760                 :        757 :                         adjust_next = (end - next->vm_start) >> PAGE_SHIFT;
     761                 :        757 :                         exporter = next;
     762                 :        757 :                         importer = vma;
     763                 :        757 :                         VM_WARN_ON(expand != importer);
     764         [ +  + ]:      29801 :                 } else if (end < vma->vm_end) {
     765                 :            :                         /*
     766                 :            :                          * vma shrinks, and !insert tells it's not
     767                 :            :                          * split_vma inserting another: so it must be
     768                 :            :                          * mprotect case 4 shifting the boundary down.
     769                 :            :                          */
     770                 :         26 :                         adjust_next = -((vma->vm_end - end) >> PAGE_SHIFT);
     771                 :         26 :                         exporter = vma;
     772                 :         26 :                         importer = next;
     773                 :      30558 :                         VM_WARN_ON(expand != importer);
     774                 :            :                 }
     775                 :            : 
     776                 :            :                 /*
     777                 :            :                  * Easily overlooked: when mprotect shifts the boundary,
     778                 :            :                  * make sure the expanding vma has anon_vma set if the
     779                 :            :                  * shrinking vma had, to cover any anon pages imported.
     780                 :            :                  */
     781   [ +  +  +  -  :      30558 :                 if (exporter && exporter->anon_vma && !importer->anon_vma) {
                   -  - ]
     782                 :          0 :                         int error;
     783                 :            : 
     784                 :          0 :                         importer->anon_vma = exporter->anon_vma;
     785                 :          0 :                         error = anon_vma_clone(importer, exporter);
     786         [ #  # ]:          0 :                         if (error)
     787                 :            :                                 return error;
     788                 :            :                 }
     789                 :            :         }
     790                 :     290458 : again:
     791         [ +  + ]:     290458 :         vma_adjust_trans_huge(orig_vma, start, end, adjust_next);
     792                 :            : 
     793         [ +  + ]:     290458 :         if (file) {
     794                 :     255619 :                 mapping = file->f_mapping;
     795                 :     255619 :                 root = &mapping->i_mmap;
     796                 :     255619 :                 uprobe_munmap(vma, vma->vm_start, vma->vm_end);
     797                 :            : 
     798         [ -  + ]:     255619 :                 if (adjust_next)
     799                 :          0 :                         uprobe_munmap(next, next->vm_start, next->vm_end);
     800                 :            : 
     801                 :     255619 :                 i_mmap_lock_write(mapping);
     802         [ +  + ]:     255619 :                 if (insert) {
     803                 :            :                         /*
     804                 :            :                          * Put into interval tree now, so instantiated pages
     805                 :            :                          * are visible to arm/parisc __flush_dcache_page
     806                 :            :                          * throughout; but we cannot insert into address
     807                 :            :                          * space until vma start or end is updated.
     808                 :            :                          */
     809                 :     237640 :                         __vma_link_file(insert);
     810                 :            :                 }
     811                 :            :         }
     812                 :            : 
     813                 :     290458 :         anon_vma = vma->anon_vma;
     814         [ +  + ]:     290458 :         if (!anon_vma && adjust_next)
     815                 :         26 :                 anon_vma = next->anon_vma;
     816         [ +  + ]:     290458 :         if (anon_vma) {
     817                 :      92678 :                 VM_WARN_ON(adjust_next && next->anon_vma &&
     818                 :            :                            anon_vma != next->anon_vma);
     819                 :      92678 :                 anon_vma_lock_write(anon_vma);
     820                 :      92678 :                 anon_vma_interval_tree_pre_update_vma(vma);
     821         [ +  + ]:      92678 :                 if (adjust_next)
     822                 :        783 :                         anon_vma_interval_tree_pre_update_vma(next);
     823                 :            :         }
     824                 :            : 
     825         [ +  + ]:     290458 :         if (root) {
     826                 :     255619 :                 flush_dcache_mmap_lock(mapping);
     827                 :     255619 :                 vma_interval_tree_remove(vma, root);
     828         [ -  + ]:     255619 :                 if (adjust_next)
     829                 :          0 :                         vma_interval_tree_remove(next, root);
     830                 :            :         }
     831                 :            : 
     832         [ +  + ]:     290458 :         if (start != vma->vm_start) {
     833                 :     143989 :                 vma->vm_start = start;
     834                 :     143989 :                 start_changed = true;
     835                 :            :         }
     836         [ +  + ]:     290458 :         if (end != vma->vm_end) {
     837                 :     146469 :                 vma->vm_end = end;
     838                 :     146469 :                 end_changed = true;
     839                 :            :         }
     840                 :     290458 :         vma->vm_pgoff = pgoff;
     841         [ +  + ]:     290458 :         if (adjust_next) {
     842                 :        783 :                 next->vm_start += adjust_next << PAGE_SHIFT;
     843                 :        783 :                 next->vm_pgoff += adjust_next;
     844                 :            :         }
     845                 :            : 
     846         [ +  + ]:     290458 :         if (root) {
     847         [ -  + ]:     255619 :                 if (adjust_next)
     848                 :          0 :                         vma_interval_tree_insert(next, root);
     849                 :     255619 :                 vma_interval_tree_insert(vma, root);
     850                 :     255619 :                 flush_dcache_mmap_unlock(mapping);
     851                 :            :         }
     852                 :            : 
     853         [ -  + ]:     290458 :         if (remove_next) {
     854                 :            :                 /*
     855                 :            :                  * vma_merge has merged next into vma, and needs
     856                 :            :                  * us to remove next before dropping the locks.
     857                 :            :                  */
     858         [ #  # ]:          0 :                 if (remove_next != 3)
     859                 :          0 :                         __vma_unlink_common(mm, next, next);
     860                 :            :                 else
     861                 :            :                         /*
     862                 :            :                          * vma is not before next if they've been
     863                 :            :                          * swapped.
     864                 :            :                          *
     865                 :            :                          * pre-swap() next->vm_start was reduced so
     866                 :            :                          * tell validate_mm_rb to ignore pre-swap()
     867                 :            :                          * "next" (which is stored in post-swap()
     868                 :            :                          * "vma").
     869                 :            :                          */
     870                 :          0 :                         __vma_unlink_common(mm, next, vma);
     871         [ #  # ]:          0 :                 if (file)
     872                 :          0 :                         __remove_shared_vm_struct(next, file, mapping);
     873         [ +  + ]:     290458 :         } else if (insert) {
     874                 :            :                 /*
     875                 :            :                  * split_vma has split insert from vma, and needs
     876                 :            :                  * us to insert it before dropping the locks
     877                 :            :                  * (it may either follow vma or precede it).
     878                 :            :                  */
     879                 :     238528 :                 __insert_vm_struct(mm, insert);
     880                 :            :         } else {
     881         [ +  + ]:      51930 :                 if (start_changed)
     882                 :      40222 :                         vma_gap_update(vma);
     883         [ +  + ]:      51930 :                 if (end_changed) {
     884         [ +  + ]:      11708 :                         if (!next)
     885                 :      10686 :                                 mm->highest_vm_end = vm_end_gap(vma);
     886         [ +  + ]:       1022 :                         else if (!adjust_next)
     887                 :        239 :                                 vma_gap_update(next);
     888                 :            :                 }
     889                 :            :         }
     890                 :            : 
     891         [ +  + ]:     290458 :         if (anon_vma) {
     892                 :      92678 :                 anon_vma_interval_tree_post_update_vma(vma);
     893         [ +  + ]:      92678 :                 if (adjust_next)
     894                 :        783 :                         anon_vma_interval_tree_post_update_vma(next);
     895                 :      92678 :                 anon_vma_unlock_write(anon_vma);
     896                 :            :         }
     897         [ +  + ]:     290458 :         if (mapping)
     898                 :     255619 :                 i_mmap_unlock_write(mapping);
     899                 :            : 
     900         [ +  + ]:     290458 :         if (root) {
     901                 :     255619 :                 uprobe_mmap(vma);
     902                 :            : 
     903         [ -  + ]:     255619 :                 if (adjust_next)
     904                 :          0 :                         uprobe_mmap(next);
     905                 :            :         }
     906                 :            : 
     907         [ -  + ]:     290458 :         if (remove_next) {
     908         [ #  # ]:          0 :                 if (file) {
     909                 :          0 :                         uprobe_munmap(next, next->vm_start, next->vm_end);
     910                 :          0 :                         fput(file);
     911                 :            :                 }
     912         [ #  # ]:          0 :                 if (next->anon_vma)
     913                 :          0 :                         anon_vma_merge(vma, next);
     914                 :          0 :                 mm->map_count--;
     915         [ #  # ]:          0 :                 mpol_put(vma_policy(next));
     916                 :          0 :                 vm_area_free(next);
     917                 :            :                 /*
     918                 :            :                  * In mprotect's case 6 (see comments on vma_merge),
     919                 :            :                  * we must remove another next too. It would clutter
     920                 :            :                  * up the code too much to do both in one go.
     921                 :            :                  */
     922         [ #  # ]:          0 :                 if (remove_next != 3) {
     923                 :            :                         /*
     924                 :            :                          * If "next" was removed and vma->vm_end was
     925                 :            :                          * expanded (up) over it, in turn
     926                 :            :                          * "next->vm_prev->vm_end" changed and the
     927                 :            :                          * "vma->vm_next" gap must be updated.
     928                 :            :                          */
     929                 :          0 :                         next = vma->vm_next;
     930                 :            :                 } else {
     931                 :            :                         /*
     932                 :            :                          * For the scope of the comment "next" and
     933                 :            :                          * "vma" considered pre-swap(): if "vma" was
     934                 :            :                          * removed, next->vm_start was expanded (down)
     935                 :            :                          * over it and the "next" gap must be updated.
     936                 :            :                          * Because of the swap() the post-swap() "vma"
     937                 :            :                          * actually points to pre-swap() "next"
     938                 :            :                          * (post-swap() "next" as opposed is now a
     939                 :            :                          * dangling pointer).
     940                 :            :                          */
     941                 :            :                         next = vma;
     942                 :            :                 }
     943         [ #  # ]:          0 :                 if (remove_next == 2) {
     944                 :          0 :                         remove_next = 1;
     945                 :          0 :                         end = next->vm_end;
     946                 :          0 :                         goto again;
     947                 :            :                 }
     948         [ #  # ]:          0 :                 else if (next)
     949                 :          0 :                         vma_gap_update(next);
     950                 :            :                 else {
     951                 :            :                         /*
     952                 :            :                          * If remove_next == 2 we obviously can't
     953                 :            :                          * reach this path.
     954                 :            :                          *
     955                 :            :                          * If remove_next == 3 we can't reach this
     956                 :            :                          * path because pre-swap() next is always not
     957                 :            :                          * NULL. pre-swap() "next" is not being
     958                 :            :                          * removed and its next->vm_end is not altered
     959                 :            :                          * (and furthermore "end" already matches
     960                 :            :                          * next->vm_end in remove_next == 3).
     961                 :            :                          *
     962                 :            :                          * We reach this only in the remove_next == 1
     963                 :            :                          * case if the "next" vma that was removed was
     964                 :            :                          * the highest vma of the mm. However in such
     965                 :            :                          * case next->vm_end == "end" and the extended
     966                 :            :                          * "vma" has vma->vm_end == next->vm_end so
     967                 :            :                          * mm->highest_vm_end doesn't need any update
     968                 :            :                          * in remove_next == 1 case.
     969                 :            :                          */
     970                 :     290458 :                         VM_WARN_ON(mm->highest_vm_end != vm_end_gap(vma));
     971                 :            :                 }
     972                 :            :         }
     973         [ +  + ]:     290458 :         if (insert && file)
     974                 :     237640 :                 uprobe_mmap(insert);
     975                 :            : 
     976                 :            :         validate_mm(mm);
     977                 :            : 
     978                 :            :         return 0;
     979                 :            : }
     980                 :            : 
     981                 :            : /*
     982                 :            :  * If the vma has a ->close operation then the driver probably needs to release
     983                 :            :  * per-vma resources, so we don't attempt to merge those.
     984                 :            :  */
     985                 :     528929 : static inline int is_mergeable_vma(struct vm_area_struct *vma,
     986                 :            :                                 struct file *file, unsigned long vm_flags,
     987                 :            :                                 struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
     988                 :            : {
     989                 :            :         /*
     990                 :            :          * VM_SOFTDIRTY should not prevent from VMA merging, if we
     991                 :            :          * match the flags but dirty bit -- the caller should mark
     992                 :            :          * merged VMA as dirty. If dirty bit won't be excluded from
     993                 :            :          * comparison, we increase pressure on the memory system forcing
     994                 :            :          * the kernel to generate new VMAs when old one could be
     995                 :            :          * extended instead.
     996                 :            :          */
     997                 :     528929 :         if ((vma->vm_flags ^ vm_flags) & ~VM_SOFTDIRTY)
     998                 :            :                 return 0;
     999   [ +  +  +  + ]:     138273 :         if (vma->vm_file != file)
    1000                 :            :                 return 0;
    1001   [ -  +  -  -  :      30581 :         if (vma->vm_ops && vma->vm_ops->close)
             +  +  +  - ]
    1002                 :            :                 return 0;
    1003   [ +  -  +  - ]:      30581 :         if (!is_mergeable_vm_userfaultfd_ctx(vma, vm_userfaultfd_ctx))
    1004                 :            :                 return 0;
    1005                 :      30581 :         return 1;
    1006                 :            : }
    1007                 :            : 
    1008                 :      30581 : static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,
    1009                 :            :                                         struct anon_vma *anon_vma2,
    1010                 :            :                                         struct vm_area_struct *vma)
    1011                 :            : {
    1012                 :            :         /*
    1013                 :            :          * The list_is_singular() test is to avoid merging VMA cloned from
    1014                 :            :          * parents. This can improve scalability caused by anon_vma lock.
    1015                 :            :          */
    1016   [ +  -  +  - ]:      30581 :         if ((!anon_vma1 || !anon_vma2) && (!vma ||
    1017   [ +  +  +  + ]:      30581 :                 list_is_singular(&vma->anon_vma_chain)))
    1018                 :            :                 return 1;
    1019                 :      18039 :         return anon_vma1 == anon_vma2;
    1020                 :            : }
    1021                 :            : 
    1022                 :            : /*
    1023                 :            :  * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
    1024                 :            :  * in front of (at a lower virtual address and file offset than) the vma.
    1025                 :            :  *
    1026                 :            :  * We cannot merge two vmas if they have differently assigned (non-NULL)
    1027                 :            :  * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
    1028                 :            :  *
    1029                 :            :  * We don't check here for the merged mmap wrapping around the end of pagecache
    1030                 :            :  * indices (16TB on ia32) because do_mmap_pgoff() does not permit mmap's which
    1031                 :            :  * wrap, nor mmaps which cover the final page at index -1UL.
    1032                 :            :  */
    1033                 :            : static int
    1034                 :     276564 : can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
    1035                 :            :                      struct anon_vma *anon_vma, struct file *file,
    1036                 :            :                      pgoff_t vm_pgoff,
    1037                 :            :                      struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
    1038                 :            : {
    1039   [ +  +  +  - ]:     294568 :         if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) &&
    1040         [ +  - ]:      29562 :             is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
    1041         [ +  - ]:      29562 :                 if (vma->vm_pgoff == vm_pgoff)
    1042                 :      29562 :                         return 1;
    1043                 :            :         }
    1044                 :            :         return 0;
    1045                 :            : }
    1046                 :            : 
    1047                 :            : /*
    1048                 :            :  * Return true if we can merge this (vm_flags,anon_vma,file,vm_pgoff)
    1049                 :            :  * beyond (at a higher virtual address and file offset than) the vma.
    1050                 :            :  *
    1051                 :            :  * We cannot merge two vmas if they have differently assigned (non-NULL)
    1052                 :            :  * anon_vmas, nor if same anon_vma is assigned but offsets incompatible.
    1053                 :            :  */
    1054                 :            : static int
    1055                 :     252365 : can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
    1056                 :            :                     struct anon_vma *anon_vma, struct file *file,
    1057                 :            :                     pgoff_t vm_pgoff,
    1058                 :            :                     struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
    1059                 :            : {
    1060   [ +  +  +  + ]:     252400 :         if (is_mergeable_vma(vma, file, vm_flags, vm_userfaultfd_ctx) &&
    1061         [ +  - ]:       1019 :             is_mergeable_anon_vma(anon_vma, vma->anon_vma, vma)) {
    1062                 :        996 :                 pgoff_t vm_pglen;
    1063         [ +  - ]:        996 :                 vm_pglen = vma_pages(vma);
    1064         [ +  - ]:        996 :                 if (vma->vm_pgoff + vm_pglen == vm_pgoff)
    1065                 :        996 :                         return 1;
    1066                 :            :         }
    1067                 :            :         return 0;
    1068                 :            : }
    1069                 :            : 
    1070                 :            : /*
    1071                 :            :  * Given a mapping request (addr,end,vm_flags,file,pgoff), figure out
    1072                 :            :  * whether that can be merged with its predecessor or its successor.
    1073                 :            :  * Or both (it neatly fills a hole).
    1074                 :            :  *
    1075                 :            :  * In most cases - when called for mmap, brk or mremap - [addr,end) is
    1076                 :            :  * certain not to be mapped by the time vma_merge is called; but when
    1077                 :            :  * called for mprotect, it is certain to be already mapped (either at
    1078                 :            :  * an offset within prev, or at the start of next), and the flags of
    1079                 :            :  * this area are about to be changed to vm_flags - and the no-change
    1080                 :            :  * case has already been eliminated.
    1081                 :            :  *
    1082                 :            :  * The following mprotect cases have to be considered, where AAAA is
    1083                 :            :  * the area passed down from mprotect_fixup, never extending beyond one
    1084                 :            :  * vma, PPPPPP is the prev vma specified, and NNNNNN the next vma after:
    1085                 :            :  *
    1086                 :            :  *     AAAA             AAAA                   AAAA
    1087                 :            :  *    PPPPPPNNNNNN    PPPPPPNNNNNN       PPPPPPNNNNNN
    1088                 :            :  *    cannot merge    might become       might become
    1089                 :            :  *                    PPNNNNNNNNNN       PPPPPPPPPPNN
    1090                 :            :  *    mmap, brk or    case 4 below       case 5 below
    1091                 :            :  *    mremap move:
    1092                 :            :  *                        AAAA               AAAA
    1093                 :            :  *                    PPPP    NNNN       PPPPNNNNXXXX
    1094                 :            :  *                    might become       might become
    1095                 :            :  *                    PPPPPPPPPPPP 1 or  PPPPPPPPPPPP 6 or
    1096                 :            :  *                    PPPPPPPPNNNN 2 or  PPPPPPPPXXXX 7 or
    1097                 :            :  *                    PPPPNNNNNNNN 3     PPPPXXXXXXXX 8
    1098                 :            :  *
    1099                 :            :  * It is important for case 8 that the vma NNNN overlapping the
    1100                 :            :  * region AAAA is never going to extended over XXXX. Instead XXXX must
    1101                 :            :  * be extended in region AAAA and NNNN must be removed. This way in
    1102                 :            :  * all cases where vma_merge succeeds, the moment vma_adjust drops the
    1103                 :            :  * rmap_locks, the properties of the merged vma will be already
    1104                 :            :  * correct for the whole merged range. Some of those properties like
    1105                 :            :  * vm_page_prot/vm_flags may be accessed by rmap_walks and they must
    1106                 :            :  * be correct for the whole merged range immediately after the
    1107                 :            :  * rmap_locks are released. Otherwise if XXXX would be removed and
    1108                 :            :  * NNNN would be extended over the XXXX range, remove_migration_ptes
    1109                 :            :  * or other rmap walkers (if working on addresses beyond the "end"
    1110                 :            :  * parameter) may establish ptes with the wrong permissions of NNNN
    1111                 :            :  * instead of the right permissions of XXXX.
    1112                 :            :  */
    1113                 :     464822 : struct vm_area_struct *vma_merge(struct mm_struct *mm,
    1114                 :            :                         struct vm_area_struct *prev, unsigned long addr,
    1115                 :            :                         unsigned long end, unsigned long vm_flags,
    1116                 :            :                         struct anon_vma *anon_vma, struct file *file,
    1117                 :            :                         pgoff_t pgoff, struct mempolicy *policy,
    1118                 :            :                         struct vm_userfaultfd_ctx vm_userfaultfd_ctx)
    1119                 :            : {
    1120                 :     464822 :         pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
    1121                 :     464822 :         struct vm_area_struct *area, *next;
    1122                 :     464822 :         int err;
    1123                 :            : 
    1124                 :            :         /*
    1125                 :            :          * We later require that vma->vm_flags == vm_flags,
    1126                 :            :          * so this tests vma->vm_flags & VM_SPECIAL, too.
    1127                 :            :          */
    1128         [ +  - ]:     464822 :         if (vm_flags & VM_SPECIAL)
    1129                 :            :                 return NULL;
    1130                 :            : 
    1131         [ +  + ]:     464822 :         if (prev)
    1132                 :     454136 :                 next = prev->vm_next;
    1133                 :            :         else
    1134                 :      10686 :                 next = mm->mmap;
    1135                 :     464822 :         area = next;
    1136   [ +  -  -  + ]:     464822 :         if (area && area->vm_end == end)             /* cases 6, 7, 8 */
    1137                 :          0 :                 next = next->vm_next;
    1138                 :            : 
    1139                 :            :         /* verify some invariant that must be enforced by the caller */
    1140                 :     464822 :         VM_WARN_ON(prev && addr <= prev->vm_start);
    1141                 :     464822 :         VM_WARN_ON(area && end > area->vm_end);
    1142                 :     464822 :         VM_WARN_ON(addr >= end);
    1143                 :            : 
    1144                 :            :         /*
    1145                 :            :          * Can it merge with the predecessor?
    1146                 :            :          */
    1147   [ +  +  +  +  :     464822 :         if (prev && prev->vm_end == addr &&
                   -  - ]
    1148   [ -  +  +  + ]:     504730 :                         mpol_equal(vma_policy(prev), policy) &&
    1149                 :     252365 :                         can_vma_merge_after(prev, vm_flags,
    1150                 :            :                                             anon_vma, file, pgoff,
    1151                 :            :                                             vm_userfaultfd_ctx)) {
    1152                 :            :                 /*
    1153                 :            :                  * OK, it can.  Can we now merge in the successor as well?
    1154                 :            :                  */
    1155   [ +  -  +  +  :        996 :                 if (next && end == next->vm_start &&
                   -  - ]
    1156   [ -  +  -  + ]:         24 :                                 mpol_equal(policy, vma_policy(next)) &&
    1157                 :         12 :                                 can_vma_merge_before(next, vm_flags,
    1158                 :            :                                                      anon_vma, file,
    1159                 :            :                                                      pgoff+pglen,
    1160         [ #  # ]:          0 :                                                      vm_userfaultfd_ctx) &&
    1161         [ #  # ]:          0 :                                 is_mergeable_anon_vma(prev->anon_vma,
    1162                 :            :                                                       next->anon_vma, NULL)) {
    1163                 :            :                                                         /* cases 1, 6 */
    1164                 :          0 :                         err = __vma_adjust(prev, prev->vm_start,
    1165                 :            :                                          next->vm_end, prev->vm_pgoff, NULL,
    1166                 :            :                                          prev);
    1167                 :            :                 } else                                  /* cases 2, 5, 7 */
    1168                 :        996 :                         err = __vma_adjust(prev, prev->vm_start,
    1169                 :            :                                          end, prev->vm_pgoff, NULL, prev);
    1170         [ +  - ]:        996 :                 if (err)
    1171                 :            :                         return NULL;
    1172                 :        996 :                 khugepaged_enter_vma_merge(prev, vm_flags);
    1173                 :        996 :                 return prev;
    1174                 :            :         }
    1175                 :            : 
    1176                 :            :         /*
    1177                 :            :          * Can this new request be merged in front of next?
    1178                 :            :          */
    1179   [ +  -  +  +  :     463826 :         if (next && end == next->vm_start &&
                   -  - ]
    1180   [ -  +  +  + ]:     553104 :                         mpol_equal(policy, vma_policy(next)) &&
    1181                 :     276552 :                         can_vma_merge_before(next, vm_flags,
    1182                 :            :                                              anon_vma, file, pgoff+pglen,
    1183                 :            :                                              vm_userfaultfd_ctx)) {
    1184   [ +  -  +  + ]:      29562 :                 if (prev && addr < prev->vm_end)  /* case 4 */
    1185                 :         26 :                         err = __vma_adjust(prev, prev->vm_start,
    1186                 :            :                                          addr, prev->vm_pgoff, NULL, next);
    1187                 :            :                 else {                                  /* cases 3, 8 */
    1188                 :      29536 :                         err = __vma_adjust(area, addr, next->vm_end,
    1189                 :      29536 :                                          next->vm_pgoff - pglen, NULL, next);
    1190                 :            :                         /*
    1191                 :            :                          * In case 3 area is already equal to next and
    1192                 :            :                          * this is a noop, but in case 8 "area" has
    1193                 :            :                          * been removed and next was expanded over it.
    1194                 :            :                          */
    1195                 :      29536 :                         area = next;
    1196                 :            :                 }
    1197         [ +  - ]:      29562 :                 if (err)
    1198                 :            :                         return NULL;
    1199                 :      29562 :                 khugepaged_enter_vma_merge(area, vm_flags);
    1200                 :      29562 :                 return area;
    1201                 :            :         }
    1202                 :            : 
    1203                 :            :         return NULL;
    1204                 :            : }
    1205                 :            : 
    1206                 :            : /*
    1207                 :            :  * Rough compatbility check to quickly see if it's even worth looking
    1208                 :            :  * at sharing an anon_vma.
    1209                 :            :  *
    1210                 :            :  * They need to have the same vm_file, and the flags can only differ
    1211                 :            :  * in things that mprotect may change.
    1212                 :            :  *
    1213                 :            :  * NOTE! The fact that we share an anon_vma doesn't _have_ to mean that
    1214                 :            :  * we can merge the two vma's. For example, we refuse to merge a vma if
    1215                 :            :  * there is a vm_ops->close() function, because that indicates that the
    1216                 :            :  * driver is doing some kind of reference counting. But that doesn't
    1217                 :            :  * really matter for the anon_vma sharing case.
    1218                 :            :  */
    1219                 :     224504 : static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
    1220                 :            : {
    1221         [ #  # ]:          0 :         return a->vm_end == b->vm_start &&
    1222         [ -  + ]:     144117 :                 mpol_equal(vma_policy(a), vma_policy(b)) &&
    1223         [ +  + ]:     144117 :                 a->vm_file == b->vm_file &&
    1224   [ +  +  +  + ]:     295922 :                 !((a->vm_flags ^ b->vm_flags) & ~(VM_READ|VM_WRITE|VM_EXEC|VM_SOFTDIRTY)) &&
    1225         [ -  + ]:         87 :                 b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
    1226                 :            : }
    1227                 :            : 
    1228                 :            : /*
    1229                 :            :  * Do some basic sanity checking to see if we can re-use the anon_vma
    1230                 :            :  * from 'old'. The 'a'/'b' vma's are in VM order - one of them will be
    1231                 :            :  * the same as 'old', the other will be the new one that is trying
    1232                 :            :  * to share the anon_vma.
    1233                 :            :  *
    1234                 :            :  * NOTE! This runs with mm_sem held for reading, so it is possible that
    1235                 :            :  * the anon_vma of 'old' is concurrently in the process of being set up
    1236                 :            :  * by another page fault trying to merge _that_. But that's ok: if it
    1237                 :            :  * is being set up, that automatically means that it will be a singleton
    1238                 :            :  * acceptable for merging, so we can do all of this optimistically. But
    1239                 :            :  * we do that READ_ONCE() to make sure that we never re-load the pointer.
    1240                 :            :  *
    1241                 :            :  * IOW: that the "list_is_singular()" test on the anon_vma_chain only
    1242                 :            :  * matters for the 'stable anon_vma' case (ie the thing we want to avoid
    1243                 :            :  * is to return an anon_vma that is "complex" due to having gone through
    1244                 :            :  * a fork).
    1245                 :            :  *
    1246                 :            :  * We also make sure that the two vma's are compatible (adjacent,
    1247                 :            :  * and with the same memory policies). That's all stable, even with just
    1248                 :            :  * a read lock on the mm_sem.
    1249                 :            :  */
    1250                 :     224504 : static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b)
    1251                 :            : {
    1252         [ +  + ]:     224504 :         if (anon_vma_compatible(a, b)) {
    1253         [ +  + ]:         87 :                 struct anon_vma *anon_vma = READ_ONCE(old->anon_vma);
    1254                 :            : 
    1255   [ +  +  +  - ]:         87 :                 if (anon_vma && list_is_singular(&old->anon_vma_chain))
    1256                 :          0 :                         return anon_vma;
    1257                 :            :         }
    1258                 :            :         return NULL;
    1259                 :            : }
    1260                 :            : 
    1261                 :            : /*
    1262                 :            :  * find_mergeable_anon_vma is used by anon_vma_prepare, to check
    1263                 :            :  * neighbouring vmas for a suitable anon_vma, before it goes off
    1264                 :            :  * to allocate a new anon_vma.  It checks because a repetitive
    1265                 :            :  * sequence of mprotects and faults may otherwise lead to distinct
    1266                 :            :  * anon_vmas being allocated, preventing vma merge in subsequent
    1267                 :            :  * mprotect.
    1268                 :            :  */
    1269                 :     122938 : struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
    1270                 :            : {
    1271                 :     122938 :         struct anon_vma *anon_vma = NULL;
    1272                 :            : 
    1273                 :            :         /* Try next first. */
    1274         [ +  + ]:     122938 :         if (vma->vm_next) {
    1275                 :     112252 :                 anon_vma = reusable_anon_vma(vma->vm_next, vma, vma->vm_next);
    1276         [ +  - ]:     112252 :                 if (anon_vma)
    1277                 :            :                         return anon_vma;
    1278                 :            :         }
    1279                 :            : 
    1280                 :            :         /* Try prev next. */
    1281         [ +  + ]:     122938 :         if (vma->vm_prev)
    1282                 :     112252 :                 anon_vma = reusable_anon_vma(vma->vm_prev, vma->vm_prev, vma);
    1283                 :            : 
    1284                 :            :         /*
    1285                 :            :          * We might reach here with anon_vma == NULL if we can't find
    1286                 :            :          * any reusable anon_vma.
    1287                 :            :          * There's no absolute need to look only at touching neighbours:
    1288                 :            :          * we could search further afield for "compatible" anon_vmas.
    1289                 :            :          * But it would probably just be a waste of time searching,
    1290                 :            :          * or lead to too many vmas hanging off the same anon_vma.
    1291                 :            :          * We're trying to allow mprotect remerging later on,
    1292                 :            :          * not trying to minimize memory used for anon_vmas.
    1293                 :            :          */
    1294                 :            :         return anon_vma;
    1295                 :            : }
    1296                 :            : 
    1297                 :            : /*
    1298                 :            :  * If a hint addr is less than mmap_min_addr change hint to be as
    1299                 :            :  * low as possible but still greater than mmap_min_addr
    1300                 :            :  */
    1301                 :     165699 : static inline unsigned long round_hint_to_min(unsigned long hint)
    1302                 :            : {
    1303                 :     165699 :         hint &= PAGE_MASK;
    1304                 :     165711 :         if (((void *)hint != NULL) &&
    1305         [ -  + ]:         12 :             (hint < mmap_min_addr))
    1306                 :          0 :                 return PAGE_ALIGN(mmap_min_addr);
    1307                 :            :         return hint;
    1308                 :            : }
    1309                 :            : 
    1310                 :     387196 : static inline int mlock_future_check(struct mm_struct *mm,
    1311                 :            :                                      unsigned long flags,
    1312                 :            :                                      unsigned long len)
    1313                 :            : {
    1314                 :     387196 :         unsigned long locked, lock_limit;
    1315                 :            : 
    1316                 :            :         /*  mlock MCL_FUTURE? */
    1317         [ -  + ]:     387196 :         if (flags & VM_LOCKED) {
    1318                 :          0 :                 locked = len >> PAGE_SHIFT;
    1319                 :          0 :                 locked += mm->locked_vm;
    1320         [ #  # ]:          0 :                 lock_limit = rlimit(RLIMIT_MEMLOCK);
    1321                 :          0 :                 lock_limit >>= PAGE_SHIFT;
    1322   [ #  #  #  # ]:          0 :                 if (locked > lock_limit && !capable(CAP_IPC_LOCK))
    1323                 :          0 :                         return -EAGAIN;
    1324                 :            :         }
    1325                 :            :         return 0;
    1326                 :            : }
    1327                 :            : 
    1328                 :     320645 : static inline u64 file_mmap_size_max(struct file *file, struct inode *inode)
    1329                 :            : {
    1330                 :     320645 :         if (S_ISREG(inode->i_mode))
    1331                 :            :                 return MAX_LFS_FILESIZE;
    1332                 :            : 
    1333         [ #  # ]:          0 :         if (S_ISBLK(inode->i_mode))
    1334                 :            :                 return MAX_LFS_FILESIZE;
    1335                 :            : 
    1336         [ #  # ]:          0 :         if (S_ISSOCK(inode->i_mode))
    1337                 :            :                 return MAX_LFS_FILESIZE;
    1338                 :            : 
    1339                 :            :         /* Special "we do even unsigned file positions" case */
    1340         [ #  # ]:          0 :         if (file->f_mode & FMODE_UNSIGNED_OFFSET)
    1341                 :          0 :                 return 0;
    1342                 :            : 
    1343                 :            :         /* Yes, random drivers might want more. But I'm tired of buggy drivers */
    1344                 :            :         return ULONG_MAX;
    1345                 :            : }
    1346                 :            : 
    1347                 :     320645 : static inline bool file_mmap_ok(struct file *file, struct inode *inode,
    1348                 :            :                                 unsigned long pgoff, unsigned long len)
    1349                 :            : {
    1350                 :     320645 :         u64 maxsize = file_mmap_size_max(file, inode);
    1351                 :            : 
    1352         [ +  - ]:     320645 :         if (maxsize && len > maxsize)
    1353                 :            :                 return false;
    1354                 :     320645 :         maxsize -= len;
    1355         [ +  - ]:     320645 :         if (pgoff > maxsize >> PAGE_SHIFT)
    1356                 :            :                 return false;
    1357                 :            :         return true;
    1358                 :            : }
    1359                 :            : 
    1360                 :            : /*
    1361                 :            :  * The caller must hold down_write(&current->mm->mmap_sem).
    1362                 :            :  */
    1363                 :     361973 : unsigned long do_mmap(struct file *file, unsigned long addr,
    1364                 :            :                         unsigned long len, unsigned long prot,
    1365                 :            :                         unsigned long flags, vm_flags_t vm_flags,
    1366                 :            :                         unsigned long pgoff, unsigned long *populate,
    1367                 :            :                         struct list_head *uf)
    1368                 :            : {
    1369         [ +  - ]:     361973 :         struct mm_struct *mm = current->mm;
    1370                 :     361973 :         int pkey = 0;
    1371                 :            : 
    1372                 :     361973 :         *populate = 0;
    1373                 :            : 
    1374         [ +  - ]:     361973 :         if (!len)
    1375                 :            :                 return -EINVAL;
    1376                 :            : 
    1377                 :            :         /*
    1378                 :            :          * Does the application expect PROT_READ to imply PROT_EXEC?
    1379                 :            :          *
    1380                 :            :          * (the exception is when the underlying filesystem is noexec
    1381                 :            :          *  mounted, in which case we dont add PROT_EXEC.)
    1382                 :            :          */
    1383   [ +  +  -  + ]:     361973 :         if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
    1384   [ #  #  #  # ]:          0 :                 if (!(file && path_noexec(&file->f_path)))
    1385                 :          0 :                         prot |= PROT_EXEC;
    1386                 :            : 
    1387                 :            :         /* force arch specific MAP_FIXED handling in get_unmapped_area */
    1388         [ +  + ]:     361973 :         if (flags & MAP_FIXED_NOREPLACE)
    1389                 :      32058 :                 flags |= MAP_FIXED;
    1390                 :            : 
    1391         [ +  + ]:     361973 :         if (!(flags & MAP_FIXED))
    1392         [ +  + ]:     165699 :                 addr = round_hint_to_min(addr);
    1393                 :            : 
    1394                 :            :         /* Careful about overflows.. */
    1395                 :     361973 :         len = PAGE_ALIGN(len);
    1396         [ +  - ]:     361973 :         if (!len)
    1397                 :            :                 return -ENOMEM;
    1398                 :            : 
    1399                 :            :         /* offset overflow? */
    1400         [ +  - ]:     361973 :         if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
    1401                 :            :                 return -EOVERFLOW;
    1402                 :            : 
    1403                 :            :         /* Too many mappings? */
    1404         [ +  - ]:     361973 :         if (mm->map_count > sysctl_max_map_count)
    1405                 :            :                 return -ENOMEM;
    1406                 :            : 
    1407                 :            :         /* Obtain the address to map to. we verify (or select) it and ensure
    1408                 :            :          * that it represents a valid section of the address space.
    1409                 :            :          */
    1410                 :     361973 :         addr = get_unmapped_area(file, addr, len, pgoff, flags);
    1411         [ +  - ]:     361973 :         if (IS_ERR_VALUE(addr))
    1412                 :            :                 return addr;
    1413                 :            : 
    1414         [ +  + ]:     361973 :         if (flags & MAP_FIXED_NOREPLACE) {
    1415                 :      32058 :                 struct vm_area_struct *vma = find_vma(mm, addr);
    1416                 :            : 
    1417   [ +  -  +  - ]:      32058 :                 if (vma && vma->vm_start < addr + len)
    1418                 :            :                         return -EEXIST;
    1419                 :            :         }
    1420                 :            : 
    1421         [ -  + ]:     361973 :         if (prot == PROT_EXEC) {
    1422                 :          0 :                 pkey = execute_only_pkey(mm);
    1423                 :          0 :                 if (pkey < 0)
    1424                 :            :                         pkey = 0;
    1425                 :            :         }
    1426                 :            : 
    1427                 :            :         /* Do simple checking here so the lower-level routines won't have
    1428                 :            :          * to. we assume access permissions have been handled by the open
    1429                 :            :          * of the memory object, so we don't do any here.
    1430                 :            :          */
    1431         [ -  + ]:     361973 :         vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) |
    1432                 :     361973 :                         mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
    1433                 :            : 
    1434         [ -  + ]:     361973 :         if (flags & MAP_LOCKED)
    1435         [ #  # ]:          0 :                 if (!can_do_mlock())
    1436                 :            :                         return -EPERM;
    1437                 :            : 
    1438         [ +  - ]:     361973 :         if (mlock_future_check(mm, vm_flags, len))
    1439                 :            :                 return -EAGAIN;
    1440                 :            : 
    1441         [ +  + ]:     361973 :         if (file) {
    1442         [ -  + ]:     320645 :                 struct inode *inode = file_inode(file);
    1443                 :     320645 :                 unsigned long flags_mask;
    1444                 :            : 
    1445         [ -  + ]:     320645 :                 if (!file_mmap_ok(file, inode, pgoff, len))
    1446                 :            :                         return -EOVERFLOW;
    1447                 :            : 
    1448                 :     320645 :                 flags_mask = LEGACY_MAP_MASK | file->f_op->mmap_supported_flags;
    1449                 :            : 
    1450   [ +  -  +  - ]:     320645 :                 switch (flags & MAP_TYPE) {
    1451                 :       6643 :                 case MAP_SHARED:
    1452                 :            :                         /*
    1453                 :            :                          * Force use of MAP_SHARED_VALIDATE with non-legacy
    1454                 :            :                          * flags. E.g. MAP_SYNC is dangerous to use with
    1455                 :            :                          * MAP_SHARED as you don't know which consistency model
    1456                 :            :                          * you will get. We silently ignore unsupported flags
    1457                 :            :                          * with MAP_SHARED to preserve backward compatibility.
    1458                 :            :                          */
    1459                 :       6643 :                         flags &= LEGACY_MAP_MASK;
    1460                 :            :                         /* fall through */
    1461                 :       6643 :                 case MAP_SHARED_VALIDATE:
    1462         [ +  - ]:       6643 :                         if (flags & ~flags_mask)
    1463                 :            :                                 return -EOPNOTSUPP;
    1464         [ +  + ]:       6643 :                         if (prot & PROT_WRITE) {
    1465         [ +  - ]:         52 :                                 if (!(file->f_mode & FMODE_WRITE))
    1466                 :            :                                         return -EACCES;
    1467         [ +  - ]:         52 :                                 if (IS_SWAPFILE(file->f_mapping->host))
    1468                 :            :                                         return -ETXTBSY;
    1469                 :            :                         }
    1470                 :            : 
    1471                 :            :                         /*
    1472                 :            :                          * Make sure we don't allow writing to an append-only
    1473                 :            :                          * file..
    1474                 :            :                          */
    1475   [ -  +  -  - ]:       6643 :                         if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
    1476                 :            :                                 return -EACCES;
    1477                 :            : 
    1478                 :            :                         /*
    1479                 :            :                          * Make sure there are no mandatory locks on the file.
    1480                 :            :                          */
    1481         [ +  - ]:       6643 :                         if (locks_verify_locked(file))
    1482                 :            :                                 return -EAGAIN;
    1483                 :            : 
    1484                 :       6643 :                         vm_flags |= VM_SHARED | VM_MAYSHARE;
    1485         [ +  + ]:       6643 :                         if (!(file->f_mode & FMODE_WRITE))
    1486                 :       6591 :                                 vm_flags &= ~(VM_MAYWRITE | VM_SHARED);
    1487                 :            : 
    1488                 :            :                         /* fall through */
    1489                 :            :                 case MAP_PRIVATE:
    1490         [ +  - ]:     320645 :                         if (!(file->f_mode & FMODE_READ))
    1491                 :            :                                 return -EACCES;
    1492         [ -  + ]:     320645 :                         if (path_noexec(&file->f_path)) {
    1493         [ #  # ]:          0 :                                 if (vm_flags & VM_EXEC)
    1494                 :            :                                         return -EPERM;
    1495                 :          0 :                                 vm_flags &= ~VM_MAYEXEC;
    1496                 :            :                         }
    1497                 :            : 
    1498         [ +  - ]:     320645 :                         if (!file->f_op->mmap)
    1499                 :            :                                 return -ENODEV;
    1500         [ +  - ]:     320645 :                         if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
    1501                 :            :                                 return -EINVAL;
    1502                 :            :                         break;
    1503                 :            : 
    1504                 :            :                 default:
    1505                 :            :                         return -EINVAL;
    1506                 :            :                 }
    1507                 :            :         } else {
    1508      [ -  +  - ]:      41328 :                 switch (flags & MAP_TYPE) {
    1509                 :          0 :                 case MAP_SHARED:
    1510         [ #  # ]:          0 :                         if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
    1511                 :            :                                 return -EINVAL;
    1512                 :            :                         /*
    1513                 :            :                          * Ignore pgoff.
    1514                 :            :                          */
    1515                 :          0 :                         pgoff = 0;
    1516                 :          0 :                         vm_flags |= VM_SHARED | VM_MAYSHARE;
    1517                 :          0 :                         break;
    1518                 :      41328 :                 case MAP_PRIVATE:
    1519                 :            :                         /*
    1520                 :            :                          * Set pgoff according to addr for anon_vma.
    1521                 :            :                          */
    1522                 :      41328 :                         pgoff = addr >> PAGE_SHIFT;
    1523                 :      41328 :                         break;
    1524                 :            :                 default:
    1525                 :            :                         return -EINVAL;
    1526                 :            :                 }
    1527                 :            :         }
    1528                 :            : 
    1529                 :            :         /*
    1530                 :            :          * Set 'VM_NORESERVE' if we should not account for the
    1531                 :            :          * memory use of this mapping.
    1532                 :            :          */
    1533         [ +  + ]:     361973 :         if (flags & MAP_NORESERVE) {
    1534                 :            :                 /* We honor MAP_NORESERVE if allowed to overcommit */
    1535         [ +  - ]:         52 :                 if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
    1536                 :         52 :                         vm_flags |= VM_NORESERVE;
    1537                 :            : 
    1538                 :            :                 /* hugetlb applies strict overcommit unless MAP_NORESERVE */
    1539   [ -  +  -  - ]:         52 :                 if (file && is_file_hugepages(file))
    1540                 :          0 :                         vm_flags |= VM_NORESERVE;
    1541                 :            :         }
    1542                 :            : 
    1543                 :     361973 :         addr = mmap_region(file, addr, len, vm_flags, pgoff, uf);
    1544         [ +  - ]:     361973 :         if (!IS_ERR_VALUE(addr) &&
    1545         [ +  - ]:     361973 :             ((vm_flags & VM_LOCKED) ||
    1546         [ -  + ]:     361973 :              (flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE))
    1547                 :          0 :                 *populate = len;
    1548                 :            :         return addr;
    1549                 :            : }
    1550                 :            : 
    1551                 :     276537 : unsigned long ksys_mmap_pgoff(unsigned long addr, unsigned long len,
    1552                 :            :                               unsigned long prot, unsigned long flags,
    1553                 :            :                               unsigned long fd, unsigned long pgoff)
    1554                 :            : {
    1555                 :     276537 :         struct file *file = NULL;
    1556                 :     276537 :         unsigned long retval;
    1557                 :            : 
    1558         [ +  + ]:     276537 :         if (!(flags & MAP_ANONYMOUS)) {
    1559                 :     235209 :                 audit_mmap_fd(fd, flags);
    1560                 :     235209 :                 file = fget(fd);
    1561         [ +  - ]:     235209 :                 if (!file)
    1562                 :            :                         return -EBADF;
    1563   [ +  -  -  + ]:     470418 :                 if (is_file_hugepages(file))
    1564                 :          0 :                         len = ALIGN(len, huge_page_size(hstate_file(file)));
    1565                 :     235209 :                 retval = -EINVAL;
    1566   [ -  +  -  - ]:     235209 :                 if (unlikely(flags & MAP_HUGETLB && !is_file_hugepages(file)))
    1567                 :          0 :                         goto out_fput;
    1568         [ -  + ]:      41328 :         } else if (flags & MAP_HUGETLB) {
    1569                 :          0 :                 struct user_struct *user = NULL;
    1570                 :          0 :                 struct hstate *hs;
    1571                 :            : 
    1572         [ #  # ]:          0 :                 hs = hstate_sizelog((flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
    1573         [ #  # ]:          0 :                 if (!hs)
    1574                 :          0 :                         return -EINVAL;
    1575                 :            : 
    1576                 :          0 :                 len = ALIGN(len, huge_page_size(hs));
    1577                 :            :                 /*
    1578                 :            :                  * VM_NORESERVE is used because the reservations will be
    1579                 :            :                  * taken when vm_ops->mmap() is called
    1580                 :            :                  * A dummy user value is used because we are not locking
    1581                 :            :                  * memory so no accounting is necessary
    1582                 :            :                  */
    1583                 :          0 :                 file = hugetlb_file_setup(HUGETLB_ANON_FILE, len,
    1584                 :            :                                 VM_NORESERVE,
    1585                 :            :                                 &user, HUGETLB_ANONHUGE_INODE,
    1586                 :            :                                 (flags >> MAP_HUGE_SHIFT) & MAP_HUGE_MASK);
    1587         [ #  # ]:          0 :                 if (IS_ERR(file))
    1588                 :            :                         return PTR_ERR(file);
    1589                 :            :         }
    1590                 :            : 
    1591                 :     276537 :         flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
    1592                 :            : 
    1593                 :     276537 :         retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff);
    1594                 :     276537 : out_fput:
    1595         [ +  + ]:     276537 :         if (file)
    1596                 :     235209 :                 fput(file);
    1597                 :            :         return retval;
    1598                 :            : }
    1599                 :            : 
    1600                 :          0 : SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
    1601                 :            :                 unsigned long, prot, unsigned long, flags,
    1602                 :            :                 unsigned long, fd, unsigned long, pgoff)
    1603                 :            : {
    1604                 :          0 :         return ksys_mmap_pgoff(addr, len, prot, flags, fd, pgoff);
    1605                 :            : }
    1606                 :            : 
    1607                 :            : #ifdef __ARCH_WANT_SYS_OLD_MMAP
    1608                 :            : struct mmap_arg_struct {
    1609                 :            :         unsigned long addr;
    1610                 :            :         unsigned long len;
    1611                 :            :         unsigned long prot;
    1612                 :            :         unsigned long flags;
    1613                 :            :         unsigned long fd;
    1614                 :            :         unsigned long offset;
    1615                 :            : };
    1616                 :            : 
    1617                 :            : SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
    1618                 :            : {
    1619                 :            :         struct mmap_arg_struct a;
    1620                 :            : 
    1621                 :            :         if (copy_from_user(&a, arg, sizeof(a)))
    1622                 :            :                 return -EFAULT;
    1623                 :            :         if (offset_in_page(a.offset))
    1624                 :            :                 return -EINVAL;
    1625                 :            : 
    1626                 :            :         return ksys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
    1627                 :            :                                a.offset >> PAGE_SHIFT);
    1628                 :            : }
    1629                 :            : #endif /* __ARCH_WANT_SYS_OLD_MMAP */
    1630                 :            : 
    1631                 :            : /*
    1632                 :            :  * Some shared mappings will want the pages marked read-only
    1633                 :            :  * to track write events. If so, we'll downgrade vm_page_prot
    1634                 :            :  * to the private version (using protection_map[] without the
    1635                 :            :  * VM_SHARED bit).
    1636                 :            :  */
    1637                 :     517225 : int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot)
    1638                 :            : {
    1639                 :     517225 :         vm_flags_t vm_flags = vma->vm_flags;
    1640                 :     517225 :         const struct vm_operations_struct *vm_ops = vma->vm_ops;
    1641                 :            : 
    1642                 :            :         /* If it was private or non-writable, the write bit is already clear */
    1643         [ +  + ]:     517225 :         if ((vm_flags & (VM_WRITE|VM_SHARED)) != ((VM_WRITE|VM_SHARED)))
    1644                 :            :                 return 0;
    1645                 :            : 
    1646                 :            :         /* The backer wishes to know when pages are first written to? */
    1647   [ +  -  +  -  :         52 :         if (vm_ops && (vm_ops->page_mkwrite || vm_ops->pfn_mkwrite))
                   +  - ]
    1648                 :            :                 return 1;
    1649                 :            : 
    1650                 :            :         /* The open routine did something to the protections that pgprot_modify
    1651                 :            :          * won't preserve? */
    1652   [ +  -  +  - ]:        104 :         if (pgprot_val(vm_page_prot) !=
    1653                 :            :             pgprot_val(vm_pgprot_modify(vm_page_prot, vm_flags)))
    1654                 :            :                 return 0;
    1655                 :            : 
    1656                 :            :         /* Do we need to track softdirty? */
    1657                 :         52 :         if (IS_ENABLED(CONFIG_MEM_SOFT_DIRTY) && !(vm_flags & VM_SOFTDIRTY))
    1658                 :            :                 return 1;
    1659                 :            : 
    1660                 :            :         /* Specialty mapping? */
    1661         [ +  - ]:         52 :         if (vm_flags & VM_PFNMAP)
    1662                 :            :                 return 0;
    1663                 :            : 
    1664                 :            :         /* Can the mapping track the dirty pages? */
    1665   [ +  -  +  -  :        104 :         return vma->vm_file && vma->vm_file->f_mapping &&
                   +  - ]
    1666                 :         52 :                 mapping_cap_account_dirty(vma->vm_file->f_mapping);
    1667                 :            : }
    1668                 :            : 
    1669                 :            : /*
    1670                 :            :  * We account for memory if it's a private writeable mapping,
    1671                 :            :  * not hugepages and VM_NORESERVE wasn't set.
    1672                 :            :  */
    1673                 :     361973 : static inline int accountable_mapping(struct file *file, vm_flags_t vm_flags)
    1674                 :            : {
    1675                 :            :         /*
    1676                 :            :          * hugetlb has its own accounting separate from the core VM
    1677                 :            :          * VM_HUGETLB may not be set yet so we cannot check for that flag.
    1678                 :            :          */
    1679   [ +  +  -  + ]:     682618 :         if (file && is_file_hugepages(file))
    1680                 :          0 :                 return 0;
    1681                 :            : 
    1682                 :     361973 :         return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE;
    1683                 :            : }
    1684                 :            : 
    1685                 :     361973 : unsigned long mmap_region(struct file *file, unsigned long addr,
    1686                 :            :                 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff,
    1687                 :            :                 struct list_head *uf)
    1688                 :            : {
    1689                 :     361973 :         struct mm_struct *mm = current->mm;
    1690                 :     361973 :         struct vm_area_struct *vma, *prev;
    1691                 :     361973 :         int error;
    1692                 :     361973 :         struct rb_node **rb_link, *rb_parent;
    1693                 :     361973 :         unsigned long charged = 0;
    1694                 :            : 
    1695                 :            :         /* Check against address space limit. */
    1696         [ -  + ]:     361973 :         if (!may_expand_vm(mm, vm_flags, len >> PAGE_SHIFT)) {
    1697                 :          0 :                 unsigned long nr_pages;
    1698                 :            : 
    1699                 :            :                 /*
    1700                 :            :                  * MAP_FIXED may remove pages of mappings that intersects with
    1701                 :            :                  * requested mapping. Account for the pages it would unmap.
    1702                 :            :                  */
    1703                 :          0 :                 nr_pages = count_vma_pages_range(mm, addr, addr + len);
    1704                 :            : 
    1705         [ #  # ]:          0 :                 if (!may_expand_vm(mm, vm_flags,
    1706                 :            :                                         (len >> PAGE_SHIFT) - nr_pages))
    1707                 :            :                         return -ENOMEM;
    1708                 :            :         }
    1709                 :            : 
    1710                 :            :         /* Clear old maps */
    1711         [ +  + ]:     966994 :         while (find_vma_links(mm, addr, addr + len, &prev, &rb_link,
    1712                 :            :                               &rb_parent)) {
    1713         [ +  - ]:     121524 :                 if (do_munmap(mm, addr, len, uf))
    1714                 :            :                         return -ENOMEM;
    1715                 :            :         }
    1716                 :            : 
    1717                 :            :         /*
    1718                 :            :          * Private writable mapping: check memory availability
    1719                 :            :          */
    1720         [ +  + ]:     361973 :         if (accountable_mapping(file, vm_flags)) {
    1721                 :      99282 :                 charged = len >> PAGE_SHIFT;
    1722         [ +  - ]:      99282 :                 if (security_vm_enough_memory_mm(mm, charged))
    1723                 :            :                         return -ENOMEM;
    1724                 :      99282 :                 vm_flags |= VM_ACCOUNT;
    1725                 :            :         }
    1726                 :            : 
    1727                 :            :         /*
    1728                 :            :          * Can we just expand an old mapping?
    1729                 :            :          */
    1730                 :     361973 :         vma = vma_merge(mm, prev, addr, addr + len, vm_flags,
    1731                 :            :                         NULL, file, pgoff, NULL, NULL_VM_UFFD_CTX);
    1732         [ +  + ]:     361973 :         if (vma)
    1733                 :      29548 :                 goto out;
    1734                 :            : 
    1735                 :            :         /*
    1736                 :            :          * Determine the object being mapped and call the appropriate
    1737                 :            :          * specific mapper. the address has already been validated, but
    1738                 :            :          * not unmapped, but the maps are removed from the list.
    1739                 :            :          */
    1740                 :     332425 :         vma = vm_area_alloc(mm);
    1741         [ -  + ]:     332425 :         if (!vma) {
    1742                 :          0 :                 error = -ENOMEM;
    1743                 :          0 :                 goto unacct_error;
    1744                 :            :         }
    1745                 :            : 
    1746                 :     332425 :         vma->vm_start = addr;
    1747                 :     332425 :         vma->vm_end = addr + len;
    1748                 :     332425 :         vma->vm_flags = vm_flags;
    1749         [ +  + ]:     332425 :         vma->vm_page_prot = vm_get_page_prot(vm_flags);
    1750                 :     332425 :         vma->vm_pgoff = pgoff;
    1751                 :            : 
    1752         [ +  + ]:     332425 :         if (file) {
    1753         [ +  + ]:     302666 :                 if (vm_flags & VM_DENYWRITE) {
    1754                 :      85436 :                         error = deny_write_access(file);
    1755                 :          0 :                         if (error)
    1756                 :          0 :                                 goto free_vma;
    1757                 :            :                 }
    1758         [ +  + ]:     302666 :                 if (vm_flags & VM_SHARED) {
    1759                 :         52 :                         error = mapping_map_writable(file->f_mapping);
    1760                 :          0 :                         if (error)
    1761                 :          0 :                                 goto allow_write_and_free_vma;
    1762                 :            :                 }
    1763                 :            : 
    1764                 :            :                 /* ->mmap() can change vma->vm_file, but must guarantee that
    1765                 :            :                  * vma_link() below can deny write-access if VM_DENYWRITE is set
    1766                 :            :                  * and map writably if VM_SHARED is set. This usually means the
    1767                 :            :                  * new file must not have been exposed to user-space, yet.
    1768                 :            :                  */
    1769                 :     302666 :                 vma->vm_file = get_file(file);
    1770                 :     302666 :                 error = call_mmap(file, vma);
    1771         [ -  + ]:     302666 :                 if (error)
    1772                 :          0 :                         goto unmap_and_free_vma;
    1773                 :            : 
    1774                 :            :                 /* Can addr have changed??
    1775                 :            :                  *
    1776                 :            :                  * Answer: Yes, several device drivers can do it in their
    1777                 :            :                  *         f_op->mmap method. -DaveM
    1778                 :            :                  * Bug: If addr is changed, prev, rb_link, rb_parent should
    1779                 :            :                  *      be updated for vma_link()
    1780                 :            :                  */
    1781         [ -  + ]:     302666 :                 WARN_ON_ONCE(addr != vma->vm_start);
    1782                 :            : 
    1783                 :     302666 :                 addr = vma->vm_start;
    1784                 :     302666 :                 vm_flags = vma->vm_flags;
    1785         [ -  + ]:      29759 :         } else if (vm_flags & VM_SHARED) {
    1786                 :          0 :                 error = shmem_zero_setup(vma);
    1787         [ #  # ]:          0 :                 if (error)
    1788                 :          0 :                         goto free_vma;
    1789                 :            :         } else {
    1790                 :      29759 :                 vma_set_anonymous(vma);
    1791                 :            :         }
    1792                 :            : 
    1793                 :     332425 :         vma_link(mm, vma, prev, rb_link, rb_parent);
    1794                 :            :         /* Once vma denies write, undo our temporary denial count */
    1795         [ +  + ]:     332425 :         if (file) {
    1796         [ +  + ]:     302666 :                 if (vm_flags & VM_SHARED)
    1797                 :         52 :                         mapping_unmap_writable(file->f_mapping);
    1798         [ +  + ]:     302666 :                 if (vm_flags & VM_DENYWRITE)
    1799                 :      85436 :                         allow_write_access(file);
    1800                 :            :         }
    1801                 :     332425 :         file = vma->vm_file;
    1802                 :     361973 : out:
    1803                 :     361973 :         perf_event_mmap(vma);
    1804                 :            : 
    1805                 :     361973 :         vm_stat_account(mm, vm_flags, len >> PAGE_SHIFT);
    1806         [ -  + ]:     361973 :         if (vm_flags & VM_LOCKED) {
    1807   [ #  #  #  # ]:          0 :                 if ((vm_flags & VM_SPECIAL) || vma_is_dax(vma) ||
    1808   [ #  #  #  # ]:          0 :                                         is_vm_hugetlb_page(vma) ||
    1809                 :          0 :                                         vma == get_gate_vma(current->mm))
    1810                 :          0 :                         vma->vm_flags &= VM_LOCKED_CLEAR_MASK;
    1811                 :            :                 else
    1812                 :          0 :                         mm->locked_vm += (len >> PAGE_SHIFT);
    1813                 :            :         }
    1814                 :            : 
    1815         [ +  + ]:     361973 :         if (file)
    1816                 :     320645 :                 uprobe_mmap(vma);
    1817                 :            : 
    1818                 :            :         /*
    1819                 :            :          * New (or expanded) vma always get soft dirty status.
    1820                 :            :          * Otherwise user-space soft-dirty page tracker won't
    1821                 :            :          * be able to distinguish situation when vma area unmapped,
    1822                 :            :          * then new mapped in-place (which must be aimed as
    1823                 :            :          * a completely new data area).
    1824                 :            :          */
    1825                 :     361973 :         vma->vm_flags |= VM_SOFTDIRTY;
    1826                 :            : 
    1827                 :     361973 :         vma_set_page_prot(vma);
    1828                 :            : 
    1829                 :     361973 :         return addr;
    1830                 :            : 
    1831                 :            : unmap_and_free_vma:
    1832                 :          0 :         vma->vm_file = NULL;
    1833                 :          0 :         fput(file);
    1834                 :            : 
    1835                 :            :         /* Undo any partial mapping done by a device driver. */
    1836                 :          0 :         unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end);
    1837                 :          0 :         charged = 0;
    1838         [ #  # ]:          0 :         if (vm_flags & VM_SHARED)
    1839                 :          0 :                 mapping_unmap_writable(file->f_mapping);
    1840                 :          0 : allow_write_and_free_vma:
    1841         [ #  # ]:          0 :         if (vm_flags & VM_DENYWRITE)
    1842                 :          0 :                 allow_write_access(file);
    1843                 :          0 : free_vma:
    1844                 :          0 :         vm_area_free(vma);
    1845                 :          0 : unacct_error:
    1846         [ #  # ]:          0 :         if (charged)
    1847                 :          0 :                 vm_unacct_memory(charged);
    1848                 :          0 :         return error;
    1849                 :            : }
    1850                 :            : 
    1851                 :          0 : unsigned long unmapped_area(struct vm_unmapped_area_info *info)
    1852                 :            : {
    1853                 :            :         /*
    1854                 :            :          * We implement the search by looking for an rbtree node that
    1855                 :            :          * immediately follows a suitable gap. That is,
    1856                 :            :          * - gap_start = vma->vm_prev->vm_end <= info->high_limit - length;
    1857                 :            :          * - gap_end   = vma->vm_start        >= info->low_limit  + length;
    1858                 :            :          * - gap_end - gap_start >= length
    1859                 :            :          */
    1860                 :            : 
    1861         [ #  # ]:          0 :         struct mm_struct *mm = current->mm;
    1862                 :          0 :         struct vm_area_struct *vma;
    1863                 :          0 :         unsigned long length, low_limit, high_limit, gap_start, gap_end;
    1864                 :            : 
    1865                 :            :         /* Adjust search length to account for worst case alignment overhead */
    1866                 :          0 :         length = info->length + info->align_mask;
    1867         [ #  # ]:          0 :         if (length < info->length)
    1868                 :            :                 return -ENOMEM;
    1869                 :            : 
    1870                 :            :         /* Adjust search limits by the desired length */
    1871         [ #  # ]:          0 :         if (info->high_limit < length)
    1872                 :            :                 return -ENOMEM;
    1873                 :          0 :         high_limit = info->high_limit - length;
    1874                 :            : 
    1875         [ #  # ]:          0 :         if (info->low_limit > high_limit)
    1876                 :            :                 return -ENOMEM;
    1877                 :          0 :         low_limit = info->low_limit + length;
    1878                 :            : 
    1879                 :            :         /* Check if rbtree root looks promising */
    1880         [ #  # ]:          0 :         if (RB_EMPTY_ROOT(&mm->mm_rb))
    1881                 :          0 :                 goto check_highest;
    1882                 :          0 :         vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
    1883         [ #  # ]:          0 :         if (vma->rb_subtree_gap < length)
    1884                 :          0 :                 goto check_highest;
    1885                 :            : 
    1886                 :          0 :         while (true) {
    1887                 :            :                 /* Visit left subtree if it looks promising */
    1888         [ #  # ]:          0 :                 gap_end = vm_start_gap(vma);
    1889   [ #  #  #  # ]:          0 :                 if (gap_end >= low_limit && vma->vm_rb.rb_left) {
    1890                 :          0 :                         struct vm_area_struct *left =
    1891                 :          0 :                                 rb_entry(vma->vm_rb.rb_left,
    1892                 :            :                                          struct vm_area_struct, vm_rb);
    1893         [ #  # ]:          0 :                         if (left->rb_subtree_gap >= length) {
    1894                 :          0 :                                 vma = left;
    1895                 :          0 :                                 continue;
    1896                 :            :                         }
    1897                 :            :                 }
    1898                 :            : 
    1899         [ #  # ]:          0 :                 gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0;
    1900                 :          0 : check_current:
    1901                 :            :                 /* Check if current node has a suitable gap */
    1902         [ #  # ]:          0 :                 if (gap_start > high_limit)
    1903                 :            :                         return -ENOMEM;
    1904                 :          0 :                 if (gap_end >= low_limit &&
    1905   [ #  #  #  # ]:          0 :                     gap_end > gap_start && gap_end - gap_start >= length)
    1906                 :          0 :                         goto found;
    1907                 :            : 
    1908                 :            :                 /* Visit right subtree if it looks promising */
    1909         [ #  # ]:          0 :                 if (vma->vm_rb.rb_right) {
    1910                 :          0 :                         struct vm_area_struct *right =
    1911                 :          0 :                                 rb_entry(vma->vm_rb.rb_right,
    1912                 :            :                                          struct vm_area_struct, vm_rb);
    1913         [ #  # ]:          0 :                         if (right->rb_subtree_gap >= length) {
    1914                 :          0 :                                 vma = right;
    1915                 :          0 :                                 continue;
    1916                 :            :                         }
    1917                 :            :                 }
    1918                 :            : 
    1919                 :            :                 /* Go back up the rbtree to find next candidate node */
    1920                 :          0 :                 while (true) {
    1921                 :          0 :                         struct rb_node *prev = &vma->vm_rb;
    1922         [ #  # ]:          0 :                         if (!rb_parent(prev))
    1923                 :          0 :                                 goto check_highest;
    1924                 :          0 :                         vma = rb_entry(rb_parent(prev),
    1925                 :            :                                        struct vm_area_struct, vm_rb);
    1926         [ #  # ]:          0 :                         if (prev == vma->vm_rb.rb_left) {
    1927         [ #  # ]:          0 :                                 gap_start = vm_end_gap(vma->vm_prev);
    1928         [ #  # ]:          0 :                                 gap_end = vm_start_gap(vma);
    1929                 :          0 :                                 goto check_current;
    1930                 :            :                         }
    1931                 :            :                 }
    1932                 :            :         }
    1933                 :            : 
    1934                 :          0 : check_highest:
    1935                 :            :         /* Check highest gap, which does not precede any rbtree node */
    1936                 :          0 :         gap_start = mm->highest_vm_end;
    1937                 :          0 :         gap_end = ULONG_MAX;  /* Only for VM_BUG_ON below */
    1938         [ #  # ]:          0 :         if (gap_start > high_limit)
    1939                 :            :                 return -ENOMEM;
    1940                 :            : 
    1941                 :          0 : found:
    1942                 :            :         /* We found a suitable gap. Clip it with the original low_limit. */
    1943                 :          0 :         if (gap_start < info->low_limit)
    1944                 :            :                 gap_start = info->low_limit;
    1945                 :            : 
    1946                 :            :         /* Adjust gap address to the desired alignment */
    1947                 :          0 :         gap_start += (info->align_offset - gap_start) & info->align_mask;
    1948                 :            : 
    1949                 :          0 :         VM_BUG_ON(gap_start + info->length > info->high_limit);
    1950                 :          0 :         VM_BUG_ON(gap_start + info->length > gap_end);
    1951                 :          0 :         return gap_start;
    1952                 :            : }
    1953                 :            : 
    1954                 :     165687 : unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info)
    1955                 :            : {
    1956         [ +  - ]:     165687 :         struct mm_struct *mm = current->mm;
    1957                 :     165687 :         struct vm_area_struct *vma;
    1958                 :     165687 :         unsigned long length, low_limit, high_limit, gap_start, gap_end;
    1959                 :            : 
    1960                 :            :         /* Adjust search length to account for worst case alignment overhead */
    1961                 :     165687 :         length = info->length + info->align_mask;
    1962         [ +  - ]:     165687 :         if (length < info->length)
    1963                 :            :                 return -ENOMEM;
    1964                 :            : 
    1965                 :            :         /*
    1966                 :            :          * Adjust search limits by the desired length.
    1967                 :            :          * See implementation comment at top of unmapped_area().
    1968                 :            :          */
    1969                 :     165687 :         gap_end = info->high_limit;
    1970         [ +  - ]:     165687 :         if (gap_end < length)
    1971                 :            :                 return -ENOMEM;
    1972                 :     165687 :         high_limit = gap_end - length;
    1973                 :            : 
    1974         [ +  - ]:     165687 :         if (info->low_limit > high_limit)
    1975                 :            :                 return -ENOMEM;
    1976                 :     165687 :         low_limit = info->low_limit + length;
    1977                 :            : 
    1978                 :            :         /* Check highest gap, which does not precede any rbtree node */
    1979                 :     165687 :         gap_start = mm->highest_vm_end;
    1980         [ -  + ]:     165687 :         if (gap_start <= high_limit)
    1981                 :          0 :                 goto found_highest;
    1982                 :            : 
    1983                 :            :         /* Check if rbtree root looks promising */
    1984         [ +  - ]:     165687 :         if (RB_EMPTY_ROOT(&mm->mm_rb))
    1985                 :            :                 return -ENOMEM;
    1986                 :     165687 :         vma = rb_entry(mm->mm_rb.rb_node, struct vm_area_struct, vm_rb);
    1987         [ +  - ]:     165687 :         if (vma->rb_subtree_gap < length)
    1988                 :            :                 return -ENOMEM;
    1989                 :            : 
    1990                 :    1299995 :         while (true) {
    1991                 :            :                 /* Visit right subtree if it looks promising */
    1992         [ +  - ]:    1299995 :                 gap_start = vma->vm_prev ? vm_end_gap(vma->vm_prev) : 0;
    1993   [ +  +  +  + ]:    1299995 :                 if (gap_start <= high_limit && vma->vm_rb.rb_right) {
    1994                 :     926289 :                         struct vm_area_struct *right =
    1995                 :     926289 :                                 rb_entry(vma->vm_rb.rb_right,
    1996                 :            :                                          struct vm_area_struct, vm_rb);
    1997         [ +  + ]:     926289 :                         if (right->rb_subtree_gap >= length) {
    1998                 :     589422 :                                 vma = right;
    1999                 :     589422 :                                 continue;
    2000                 :            :                         }
    2001                 :            :                 }
    2002                 :            : 
    2003                 :     710573 : check_current:
    2004                 :            :                 /* Check if current node has a suitable gap */
    2005         [ +  + ]:    1015582 :                 gap_end = vm_start_gap(vma);
    2006         [ +  - ]:    1015582 :                 if (gap_end < low_limit)
    2007                 :            :                         return -ENOMEM;
    2008                 :    1015582 :                 if (gap_start <= high_limit &&
    2009   [ +  +  +  + ]:    1015582 :                     gap_end > gap_start && gap_end - gap_start >= length)
    2010                 :     165687 :                         goto found;
    2011                 :            : 
    2012                 :            :                 /* Visit left subtree if it looks promising */
    2013         [ +  + ]:     849895 :                 if (vma->vm_rb.rb_left) {
    2014                 :     848101 :                         struct vm_area_struct *left =
    2015                 :     848101 :                                 rb_entry(vma->vm_rb.rb_left,
    2016                 :            :                                          struct vm_area_struct, vm_rb);
    2017         [ +  + ]:     848101 :                         if (left->rb_subtree_gap >= length) {
    2018                 :     544886 :                                 vma = left;
    2019                 :     544886 :                                 continue;
    2020                 :            :                         }
    2021                 :            :                 }
    2022                 :            : 
    2023                 :            :                 /* Go back up the rbtree to find next candidate node */
    2024                 :     306803 :                 while (true) {
    2025                 :     306803 :                         struct rb_node *prev = &vma->vm_rb;
    2026         [ +  - ]:     306803 :                         if (!rb_parent(prev))
    2027                 :            :                                 return -ENOMEM;
    2028                 :     306803 :                         vma = rb_entry(rb_parent(prev),
    2029                 :            :                                        struct vm_area_struct, vm_rb);
    2030         [ +  + ]:     306803 :                         if (prev == vma->vm_rb.rb_right) {
    2031                 :     305009 :                                 gap_start = vma->vm_prev ?
    2032         [ +  - ]:     305009 :                                         vm_end_gap(vma->vm_prev) : 0;
    2033                 :     305009 :                                 goto check_current;
    2034                 :            :                         }
    2035                 :            :                 }
    2036                 :            :         }
    2037                 :            : 
    2038                 :            : found:
    2039                 :            :         /* We found a suitable gap. Clip it with the original high_limit. */
    2040                 :     165687 :         if (gap_end > info->high_limit)
    2041                 :            :                 gap_end = info->high_limit;
    2042                 :            : 
    2043                 :     165687 : found_highest:
    2044                 :            :         /* Compute highest gap address at the desired alignment */
    2045                 :     165687 :         gap_end -= info->length;
    2046                 :     165687 :         gap_end -= (gap_end - info->align_offset) & info->align_mask;
    2047                 :            : 
    2048                 :     165687 :         VM_BUG_ON(gap_end < info->low_limit);
    2049                 :     165687 :         VM_BUG_ON(gap_end < gap_start);
    2050                 :     165687 :         return gap_end;
    2051                 :            : }
    2052                 :            : 
    2053                 :            : 
    2054                 :            : #ifndef arch_get_mmap_end
    2055                 :            : #define arch_get_mmap_end(addr) (TASK_SIZE)
    2056                 :            : #endif
    2057                 :            : 
    2058                 :            : #ifndef arch_get_mmap_base
    2059                 :            : #define arch_get_mmap_base(addr, base) (base)
    2060                 :            : #endif
    2061                 :            : 
    2062                 :            : /* Get an address range which is currently unmapped.
    2063                 :            :  * For shmat() with addr=0.
    2064                 :            :  *
    2065                 :            :  * Ugly calling convention alert:
    2066                 :            :  * Return value with the low bits set means error value,
    2067                 :            :  * ie
    2068                 :            :  *      if (ret & ~PAGE_MASK)
    2069                 :            :  *              error = ret;
    2070                 :            :  *
    2071                 :            :  * This function "knows" that -ENOMEM has the bits set.
    2072                 :            :  */
    2073                 :            : #ifndef HAVE_ARCH_UNMAPPED_AREA
    2074                 :            : unsigned long
    2075                 :            : arch_get_unmapped_area(struct file *filp, unsigned long addr,
    2076                 :            :                 unsigned long len, unsigned long pgoff, unsigned long flags)
    2077                 :            : {
    2078                 :            :         struct mm_struct *mm = current->mm;
    2079                 :            :         struct vm_area_struct *vma, *prev;
    2080                 :            :         struct vm_unmapped_area_info info;
    2081                 :            :         const unsigned long mmap_end = arch_get_mmap_end(addr);
    2082                 :            : 
    2083                 :            :         if (len > mmap_end - mmap_min_addr)
    2084                 :            :                 return -ENOMEM;
    2085                 :            : 
    2086                 :            :         if (flags & MAP_FIXED)
    2087                 :            :                 return addr;
    2088                 :            : 
    2089                 :            :         if (addr) {
    2090                 :            :                 addr = PAGE_ALIGN(addr);
    2091                 :            :                 vma = find_vma_prev(mm, addr, &prev);
    2092                 :            :                 if (mmap_end - len >= addr && addr >= mmap_min_addr &&
    2093                 :            :                     (!vma || addr + len <= vm_start_gap(vma)) &&
    2094                 :            :                     (!prev || addr >= vm_end_gap(prev)))
    2095                 :            :                         return addr;
    2096                 :            :         }
    2097                 :            : 
    2098                 :            :         info.flags = 0;
    2099                 :            :         info.length = len;
    2100                 :            :         info.low_limit = mm->mmap_base;
    2101                 :            :         info.high_limit = mmap_end;
    2102                 :            :         info.align_mask = 0;
    2103                 :            :         return vm_unmapped_area(&info);
    2104                 :            : }
    2105                 :            : #endif
    2106                 :            : 
    2107                 :            : /*
    2108                 :            :  * This mmap-allocator allocates new areas top-down from below the
    2109                 :            :  * stack's low limit (the base):
    2110                 :            :  */
    2111                 :            : #ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
    2112                 :            : unsigned long
    2113                 :            : arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr,
    2114                 :            :                           unsigned long len, unsigned long pgoff,
    2115                 :            :                           unsigned long flags)
    2116                 :            : {
    2117                 :            :         struct vm_area_struct *vma, *prev;
    2118                 :            :         struct mm_struct *mm = current->mm;
    2119                 :            :         struct vm_unmapped_area_info info;
    2120                 :            :         const unsigned long mmap_end = arch_get_mmap_end(addr);
    2121                 :            : 
    2122                 :            :         /* requested length too big for entire address space */
    2123                 :            :         if (len > mmap_end - mmap_min_addr)
    2124                 :            :                 return -ENOMEM;
    2125                 :            : 
    2126                 :            :         if (flags & MAP_FIXED)
    2127                 :            :                 return addr;
    2128                 :            : 
    2129                 :            :         /* requesting a specific address */
    2130                 :            :         if (addr) {
    2131                 :            :                 addr = PAGE_ALIGN(addr);
    2132                 :            :                 vma = find_vma_prev(mm, addr, &prev);
    2133                 :            :                 if (mmap_end - len >= addr && addr >= mmap_min_addr &&
    2134                 :            :                                 (!vma || addr + len <= vm_start_gap(vma)) &&
    2135                 :            :                                 (!prev || addr >= vm_end_gap(prev)))
    2136                 :            :                         return addr;
    2137                 :            :         }
    2138                 :            : 
    2139                 :            :         info.flags = VM_UNMAPPED_AREA_TOPDOWN;
    2140                 :            :         info.length = len;
    2141                 :            :         info.low_limit = max(PAGE_SIZE, mmap_min_addr);
    2142                 :            :         info.high_limit = arch_get_mmap_base(addr, mm->mmap_base);
    2143                 :            :         info.align_mask = 0;
    2144                 :            :         addr = vm_unmapped_area(&info);
    2145                 :            : 
    2146                 :            :         /*
    2147                 :            :          * A failed mmap() very likely causes application failure,
    2148                 :            :          * so fall back to the bottom-up function here. This scenario
    2149                 :            :          * can happen with large stack limits and large mmap()
    2150                 :            :          * allocations.
    2151                 :            :          */
    2152                 :            :         if (offset_in_page(addr)) {
    2153                 :            :                 VM_BUG_ON(addr != -ENOMEM);
    2154                 :            :                 info.flags = 0;
    2155                 :            :                 info.low_limit = TASK_UNMAPPED_BASE;
    2156                 :            :                 info.high_limit = mmap_end;
    2157                 :            :                 addr = vm_unmapped_area(&info);
    2158                 :            :         }
    2159                 :            : 
    2160                 :            :         return addr;
    2161                 :            : }
    2162                 :            : #endif
    2163                 :            : 
    2164                 :            : unsigned long
    2165                 :     397882 : get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
    2166                 :            :                 unsigned long pgoff, unsigned long flags)
    2167                 :            : {
    2168                 :     397882 :         unsigned long (*get_area)(struct file *, unsigned long,
    2169                 :            :                                   unsigned long, unsigned long, unsigned long);
    2170                 :            : 
    2171                 :     397882 :         unsigned long error = arch_mmap_check(addr, len, flags);
    2172                 :     397882 :         if (error)
    2173                 :            :                 return error;
    2174                 :            : 
    2175                 :            :         /* Careful about overflows.. */
    2176   [ -  +  -  -  :     795764 :         if (len > TASK_SIZE)
                   +  - ]
    2177                 :            :                 return -ENOMEM;
    2178                 :            : 
    2179         [ +  + ]:     397882 :         get_area = current->mm->get_unmapped_area;
    2180         [ +  + ]:     397882 :         if (file) {
    2181         [ +  + ]:     320645 :                 if (file->f_op->get_unmapped_area)
    2182                 :         39 :                         get_area = file->f_op->get_unmapped_area;
    2183         [ -  + ]:      77237 :         } else if (flags & MAP_SHARED) {
    2184                 :            :                 /*
    2185                 :            :                  * mmap_region() will call shmem_zero_setup() to create a file,
    2186                 :            :                  * so use shmem's get_unmapped_area in case it can be huge.
    2187                 :            :                  * do_mmap_pgoff() will clear pgoff, so match alignment.
    2188                 :            :                  */
    2189                 :          0 :                 pgoff = 0;
    2190                 :          0 :                 get_area = shmem_get_unmapped_area;
    2191                 :            :         }
    2192                 :            : 
    2193                 :     397882 :         addr = get_area(file, addr, len, pgoff, flags);
    2194         [ +  - ]:     397882 :         if (IS_ERR_VALUE(addr))
    2195                 :            :                 return addr;
    2196                 :            : 
    2197   [ -  +  -  -  :     795764 :         if (addr > TASK_SIZE - len)
                   +  - ]
    2198                 :            :                 return -ENOMEM;
    2199         [ +  - ]:     397882 :         if (offset_in_page(addr))
    2200                 :            :                 return -EINVAL;
    2201                 :            : 
    2202                 :     397882 :         error = security_mmap_addr(addr);
    2203         [ +  - ]:     397882 :         return error ? error : addr;
    2204                 :            : }
    2205                 :            : 
    2206                 :            : EXPORT_SYMBOL(get_unmapped_area);
    2207                 :            : 
    2208                 :            : /* Look up the first VMA which satisfies  addr < vm_end,  NULL if none. */
    2209                 :    1867209 : struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
    2210                 :            : {
    2211                 :    1867209 :         struct rb_node *rb_node;
    2212                 :    1867209 :         struct vm_area_struct *vma;
    2213                 :            : 
    2214                 :            :         /* Check the cache first. */
    2215                 :    1867209 :         vma = vmacache_find(mm, addr);
    2216         [ +  + ]:    1867209 :         if (likely(vma))
    2217                 :            :                 return vma;
    2218                 :            : 
    2219                 :    1233490 :         rb_node = mm->mm_rb.rb_node;
    2220                 :            : 
    2221         [ +  + ]:    6418081 :         while (rb_node) {
    2222                 :    6331662 :                 struct vm_area_struct *tmp;
    2223                 :            : 
    2224                 :    6331662 :                 tmp = rb_entry(rb_node, struct vm_area_struct, vm_rb);
    2225                 :            : 
    2226         [ +  + ]:    6331662 :                 if (tmp->vm_end > addr) {
    2227                 :    4009075 :                         vma = tmp;
    2228         [ +  + ]:    4009075 :                         if (tmp->vm_start <= addr)
    2229                 :            :                                 break;
    2230                 :    2862004 :                         rb_node = rb_node->rb_left;
    2231                 :            :                 } else
    2232                 :    2322587 :                         rb_node = rb_node->rb_right;
    2233                 :            :         }
    2234                 :            : 
    2235         [ +  + ]:    1233490 :         if (vma)
    2236                 :    1222804 :                 vmacache_update(addr, vma);
    2237                 :            :         return vma;
    2238                 :            : }
    2239                 :            : 
    2240                 :            : EXPORT_SYMBOL(find_vma);
    2241                 :            : 
    2242                 :            : /*
    2243                 :            :  * Same as find_vma, but also return a pointer to the previous VMA in *pprev.
    2244                 :            :  */
    2245                 :            : struct vm_area_struct *
    2246                 :          0 : find_vma_prev(struct mm_struct *mm, unsigned long addr,
    2247                 :            :                         struct vm_area_struct **pprev)
    2248                 :            : {
    2249                 :          0 :         struct vm_area_struct *vma;
    2250                 :            : 
    2251                 :          0 :         vma = find_vma(mm, addr);
    2252         [ #  # ]:          0 :         if (vma) {
    2253                 :          0 :                 *pprev = vma->vm_prev;
    2254                 :            :         } else {
    2255                 :          0 :                 struct rb_node *rb_node = rb_last(&mm->mm_rb);
    2256                 :            : 
    2257         [ #  # ]:          0 :                 *pprev = rb_node ? rb_entry(rb_node, struct vm_area_struct, vm_rb) : NULL;
    2258                 :            :         }
    2259                 :          0 :         return vma;
    2260                 :            : }
    2261                 :            : 
    2262                 :            : /*
    2263                 :            :  * Verify that the stack growth is acceptable and
    2264                 :            :  * update accounting. This is shared with both the
    2265                 :            :  * grow-up and grow-down cases.
    2266                 :            :  */
    2267                 :            : static int acct_stack_growth(struct vm_area_struct *vma,
    2268                 :            :                              unsigned long size, unsigned long grow)
    2269                 :            : {
    2270                 :            :         struct mm_struct *mm = vma->vm_mm;
    2271                 :            :         unsigned long new_start;
    2272                 :            : 
    2273                 :            :         /* address space limit tests */
    2274                 :            :         if (!may_expand_vm(mm, vma->vm_flags, grow))
    2275                 :            :                 return -ENOMEM;
    2276                 :            : 
    2277                 :            :         /* Stack limit test */
    2278                 :            :         if (size > rlimit(RLIMIT_STACK))
    2279                 :            :                 return -ENOMEM;
    2280                 :            : 
    2281                 :            :         /* mlock limit tests */
    2282                 :            :         if (vma->vm_flags & VM_LOCKED) {
    2283                 :            :                 unsigned long locked;
    2284                 :            :                 unsigned long limit;
    2285                 :            :                 locked = mm->locked_vm + grow;
    2286                 :            :                 limit = rlimit(RLIMIT_MEMLOCK);
    2287                 :            :                 limit >>= PAGE_SHIFT;
    2288                 :            :                 if (locked > limit && !capable(CAP_IPC_LOCK))
    2289                 :            :                         return -ENOMEM;
    2290                 :            :         }
    2291                 :            : 
    2292                 :            :         /* Check to ensure the stack will not grow into a hugetlb-only region */
    2293                 :            :         new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start :
    2294                 :            :                         vma->vm_end - size;
    2295                 :            :         if (is_hugepage_only_range(vma->vm_mm, new_start, size))
    2296                 :            :                 return -EFAULT;
    2297                 :            : 
    2298                 :            :         /*
    2299                 :            :          * Overcommit..  This must be the final test, as it will
    2300                 :            :          * update security statistics.
    2301                 :            :          */
    2302                 :            :         if (security_vm_enough_memory_mm(mm, grow))
    2303                 :            :                 return -ENOMEM;
    2304                 :            : 
    2305                 :            :         return 0;
    2306                 :            : }
    2307                 :            : 
    2308                 :            : #if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64)
    2309                 :            : /*
    2310                 :            :  * PA-RISC uses this for its stack; IA64 for its Register Backing Store.
    2311                 :            :  * vma is the last one with address > vma->vm_end.  Have to extend vma.
    2312                 :            :  */
    2313                 :            : int expand_upwards(struct vm_area_struct *vma, unsigned long address)
    2314                 :            : {
    2315                 :            :         struct mm_struct *mm = vma->vm_mm;
    2316                 :            :         struct vm_area_struct *next;
    2317                 :            :         unsigned long gap_addr;
    2318                 :            :         int error = 0;
    2319                 :            : 
    2320                 :            :         if (!(vma->vm_flags & VM_GROWSUP))
    2321                 :            :                 return -EFAULT;
    2322                 :            : 
    2323                 :            :         /* Guard against exceeding limits of the address space. */
    2324                 :            :         address &= PAGE_MASK;
    2325                 :            :         if (address >= (TASK_SIZE & PAGE_MASK))
    2326                 :            :                 return -ENOMEM;
    2327                 :            :         address += PAGE_SIZE;
    2328                 :            : 
    2329                 :            :         /* Enforce stack_guard_gap */
    2330                 :            :         gap_addr = address + stack_guard_gap;
    2331                 :            : 
    2332                 :            :         /* Guard against overflow */
    2333                 :            :         if (gap_addr < address || gap_addr > TASK_SIZE)
    2334                 :            :                 gap_addr = TASK_SIZE;
    2335                 :            : 
    2336                 :            :         next = vma->vm_next;
    2337                 :            :         if (next && next->vm_start < gap_addr &&
    2338                 :            :                         (next->vm_flags & (VM_WRITE|VM_READ|VM_EXEC))) {
    2339                 :            :                 if (!(next->vm_flags & VM_GROWSUP))
    2340                 :            :                         return -ENOMEM;
    2341                 :            :                 /* Check that both stack segments have the same anon_vma? */
    2342                 :            :         }
    2343                 :            : 
    2344                 :            :         /* We must make sure the anon_vma is allocated. */
    2345                 :            :         if (unlikely(anon_vma_prepare(vma)))
    2346                 :            :                 return -ENOMEM;
    2347                 :            : 
    2348                 :            :         /*
    2349                 :            :          * vma->vm_start/vm_end cannot change under us because the caller
    2350                 :            :          * is required to hold the mmap_sem in read mode.  We need the
    2351                 :            :          * anon_vma lock to serialize against concurrent expand_stacks.
    2352                 :            :          */
    2353                 :            :         anon_vma_lock_write(vma->anon_vma);
    2354                 :            : 
    2355                 :            :         /* Somebody else might have raced and expanded it already */
    2356                 :            :         if (address > vma->vm_end) {
    2357                 :            :                 unsigned long size, grow;
    2358                 :            : 
    2359                 :            :                 size = address - vma->vm_start;
    2360                 :            :                 grow = (address - vma->vm_end) >> PAGE_SHIFT;
    2361                 :            : 
    2362                 :            :                 error = -ENOMEM;
    2363                 :            :                 if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) {
    2364                 :            :                         error = acct_stack_growth(vma, size, grow);
    2365                 :            :                         if (!error) {
    2366                 :            :                                 /*
    2367                 :            :                                  * vma_gap_update() doesn't support concurrent
    2368                 :            :                                  * updates, but we only hold a shared mmap_sem
    2369                 :            :                                  * lock here, so we need to protect against
    2370                 :            :                                  * concurrent vma expansions.
    2371                 :            :                                  * anon_vma_lock_write() doesn't help here, as
    2372                 :            :                                  * we don't guarantee that all growable vmas
    2373                 :            :                                  * in a mm share the same root anon vma.
    2374                 :            :                                  * So, we reuse mm->page_table_lock to guard
    2375                 :            :                                  * against concurrent vma expansions.
    2376                 :            :                                  */
    2377                 :            :                                 spin_lock(&mm->page_table_lock);
    2378                 :            :                                 if (vma->vm_flags & VM_LOCKED)
    2379                 :            :                                         mm->locked_vm += grow;
    2380                 :            :                                 vm_stat_account(mm, vma->vm_flags, grow);
    2381                 :            :                                 anon_vma_interval_tree_pre_update_vma(vma);
    2382                 :            :                                 vma->vm_end = address;
    2383                 :            :                                 anon_vma_interval_tree_post_update_vma(vma);
    2384                 :            :                                 if (vma->vm_next)
    2385                 :            :                                         vma_gap_update(vma->vm_next);
    2386                 :            :                                 else
    2387                 :            :                                         mm->highest_vm_end = vm_end_gap(vma);
    2388                 :            :                                 spin_unlock(&mm->page_table_lock);
    2389                 :            : 
    2390                 :            :                                 perf_event_mmap(vma);
    2391                 :            :                         }
    2392                 :            :                 }
    2393                 :            :         }
    2394                 :            :         anon_vma_unlock_write(vma->anon_vma);
    2395                 :            :         khugepaged_enter_vma_merge(vma, vma->vm_flags);
    2396                 :            :         validate_mm(mm);
    2397                 :            :         return error;
    2398                 :            : }
    2399                 :            : #endif /* CONFIG_STACK_GROWSUP || CONFIG_IA64 */
    2400                 :            : 
    2401                 :            : /*
    2402                 :            :  * vma is the first one with address < vma->vm_start.  Have to extend vma.
    2403                 :            :  */
    2404                 :      10686 : int expand_downwards(struct vm_area_struct *vma,
    2405                 :            :                                    unsigned long address)
    2406                 :            : {
    2407                 :      10686 :         struct mm_struct *mm = vma->vm_mm;
    2408                 :      10686 :         struct vm_area_struct *prev;
    2409                 :      10686 :         int error = 0;
    2410                 :            : 
    2411                 :      10686 :         address &= PAGE_MASK;
    2412         [ +  - ]:      10686 :         if (address < mmap_min_addr)
    2413                 :            :                 return -EPERM;
    2414                 :            : 
    2415                 :            :         /* Enforce stack_guard_gap */
    2416                 :      10686 :         prev = vma->vm_prev;
    2417                 :            :         /* Check that both stack segments have the same anon_vma? */
    2418   [ -  +  -  - ]:      10686 :         if (prev && !(prev->vm_flags & VM_GROWSDOWN) &&
    2419         [ #  # ]:          0 :                         (prev->vm_flags & (VM_WRITE|VM_READ|VM_EXEC))) {
    2420         [ #  # ]:          0 :                 if (address - prev->vm_end < stack_guard_gap)
    2421                 :            :                         return -ENOMEM;
    2422                 :            :         }
    2423                 :            : 
    2424                 :            :         /* We must make sure the anon_vma is allocated. */
    2425   [ -  +  +  - ]:      10686 :         if (unlikely(anon_vma_prepare(vma)))
    2426                 :            :                 return -ENOMEM;
    2427                 :            : 
    2428                 :            :         /*
    2429                 :            :          * vma->vm_start/vm_end cannot change under us because the caller
    2430                 :            :          * is required to hold the mmap_sem in read mode.  We need the
    2431                 :            :          * anon_vma lock to serialize against concurrent expand_stacks.
    2432                 :            :          */
    2433                 :      10686 :         anon_vma_lock_write(vma->anon_vma);
    2434                 :            : 
    2435                 :            :         /* Somebody else might have raced and expanded it already */
    2436         [ +  - ]:      10686 :         if (address < vma->vm_start) {
    2437                 :      10686 :                 unsigned long size, grow;
    2438                 :            : 
    2439                 :      10686 :                 size = vma->vm_end - address;
    2440                 :      10686 :                 grow = (vma->vm_start - address) >> PAGE_SHIFT;
    2441                 :            : 
    2442                 :      10686 :                 error = -ENOMEM;
    2443         [ +  - ]:      10686 :                 if (grow <= vma->vm_pgoff) {
    2444                 :      10686 :                         error = acct_stack_growth(vma, size, grow);
    2445         [ +  - ]:      10686 :                         if (!error) {
    2446                 :            :                                 /*
    2447                 :            :                                  * vma_gap_update() doesn't support concurrent
    2448                 :            :                                  * updates, but we only hold a shared mmap_sem
    2449                 :            :                                  * lock here, so we need to protect against
    2450                 :            :                                  * concurrent vma expansions.
    2451                 :            :                                  * anon_vma_lock_write() doesn't help here, as
    2452                 :            :                                  * we don't guarantee that all growable vmas
    2453                 :            :                                  * in a mm share the same root anon vma.
    2454                 :            :                                  * So, we reuse mm->page_table_lock to guard
    2455                 :            :                                  * against concurrent vma expansions.
    2456                 :            :                                  */
    2457                 :      10686 :                                 spin_lock(&mm->page_table_lock);
    2458         [ -  + ]:      10686 :                                 if (vma->vm_flags & VM_LOCKED)
    2459                 :          0 :                                         mm->locked_vm += grow;
    2460                 :      10686 :                                 vm_stat_account(mm, vma->vm_flags, grow);
    2461                 :      10686 :                                 anon_vma_interval_tree_pre_update_vma(vma);
    2462                 :      10686 :                                 vma->vm_start = address;
    2463                 :      10686 :                                 vma->vm_pgoff -= grow;
    2464                 :      10686 :                                 anon_vma_interval_tree_post_update_vma(vma);
    2465                 :      10686 :                                 vma_gap_update(vma);
    2466                 :      10686 :                                 spin_unlock(&mm->page_table_lock);
    2467                 :            : 
    2468                 :      10686 :                                 perf_event_mmap(vma);
    2469                 :            :                         }
    2470                 :            :                 }
    2471                 :            :         }
    2472                 :      10686 :         anon_vma_unlock_write(vma->anon_vma);
    2473                 :      10686 :         khugepaged_enter_vma_merge(vma, vma->vm_flags);
    2474                 :      10686 :         validate_mm(mm);
    2475                 :      10686 :         return error;
    2476                 :            : }
    2477                 :            : 
    2478                 :            : /* enforced gap between the expanding stack and other mappings. */
    2479                 :            : unsigned long stack_guard_gap = 256UL<<PAGE_SHIFT;
    2480                 :            : 
    2481                 :          0 : static int __init cmdline_parse_stack_guard_gap(char *p)
    2482                 :            : {
    2483                 :          0 :         unsigned long val;
    2484                 :          0 :         char *endptr;
    2485                 :            : 
    2486                 :          0 :         val = simple_strtoul(p, &endptr, 10);
    2487         [ #  # ]:          0 :         if (!*endptr)
    2488                 :          0 :                 stack_guard_gap = val << PAGE_SHIFT;
    2489                 :            : 
    2490                 :          0 :         return 0;
    2491                 :            : }
    2492                 :            : __setup("stack_guard_gap=", cmdline_parse_stack_guard_gap);
    2493                 :            : 
    2494                 :            : #ifdef CONFIG_STACK_GROWSUP
    2495                 :            : int expand_stack(struct vm_area_struct *vma, unsigned long address)
    2496                 :            : {
    2497                 :            :         return expand_upwards(vma, address);
    2498                 :            : }
    2499                 :            : 
    2500                 :            : struct vm_area_struct *
    2501                 :            : find_extend_vma(struct mm_struct *mm, unsigned long addr)
    2502                 :            : {
    2503                 :            :         struct vm_area_struct *vma, *prev;
    2504                 :            : 
    2505                 :            :         addr &= PAGE_MASK;
    2506                 :            :         vma = find_vma_prev(mm, addr, &prev);
    2507                 :            :         if (vma && (vma->vm_start <= addr))
    2508                 :            :                 return vma;
    2509                 :            :         /* don't alter vm_end if the coredump is running */
    2510                 :            :         if (!prev || !mmget_still_valid(mm) || expand_stack(prev, addr))
    2511                 :            :                 return NULL;
    2512                 :            :         if (prev->vm_flags & VM_LOCKED)
    2513                 :            :                 populate_vma_page_range(prev, addr, prev->vm_end, NULL);
    2514                 :            :         return prev;
    2515                 :            : }
    2516                 :            : #else
    2517                 :      10686 : int expand_stack(struct vm_area_struct *vma, unsigned long address)
    2518                 :            : {
    2519                 :      10686 :         return expand_downwards(vma, address);
    2520                 :            : }
    2521                 :            : 
    2522                 :            : struct vm_area_struct *
    2523                 :      44021 : find_extend_vma(struct mm_struct *mm, unsigned long addr)
    2524                 :            : {
    2525                 :      44021 :         struct vm_area_struct *vma;
    2526                 :      44021 :         unsigned long start;
    2527                 :            : 
    2528                 :      44021 :         addr &= PAGE_MASK;
    2529                 :      44021 :         vma = find_vma(mm, addr);
    2530         [ +  - ]:      44021 :         if (!vma)
    2531                 :            :                 return NULL;
    2532         [ -  + ]:      44021 :         if (vma->vm_start <= addr)
    2533                 :            :                 return vma;
    2534         [ #  # ]:          0 :         if (!(vma->vm_flags & VM_GROWSDOWN))
    2535                 :            :                 return NULL;
    2536                 :            :         /* don't alter vm_start if the coredump is running */
    2537         [ #  # ]:          0 :         if (!mmget_still_valid(mm))
    2538                 :            :                 return NULL;
    2539                 :          0 :         start = vma->vm_start;
    2540         [ #  # ]:          0 :         if (expand_stack(vma, addr))
    2541                 :            :                 return NULL;
    2542         [ #  # ]:          0 :         if (vma->vm_flags & VM_LOCKED)
    2543                 :          0 :                 populate_vma_page_range(vma, addr, start, NULL);
    2544                 :            :         return vma;
    2545                 :            : }
    2546                 :            : #endif
    2547                 :            : 
    2548                 :            : EXPORT_SYMBOL_GPL(find_extend_vma);
    2549                 :            : 
    2550                 :            : /*
    2551                 :            :  * Ok - we have the memory areas we should free on the vma list,
    2552                 :            :  * so release them, and do the vma updates.
    2553                 :            :  *
    2554                 :            :  * Called with the mm semaphore held.
    2555                 :            :  */
    2556                 :     158034 : static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma)
    2557                 :            : {
    2558                 :     158034 :         unsigned long nr_accounted = 0;
    2559                 :            : 
    2560                 :            :         /* Update high watermark before we lower total_vm */
    2561         [ +  + ]:     158034 :         update_hiwater_vm(mm);
    2562                 :     158086 :         do {
    2563         [ +  + ]:     158086 :                 long nrpages = vma_pages(vma);
    2564                 :            : 
    2565         [ +  + ]:     158086 :                 if (vma->vm_flags & VM_ACCOUNT)
    2566                 :       3787 :                         nr_accounted += nrpages;
    2567                 :     158086 :                 vm_stat_account(mm, vma->vm_flags, -nrpages);
    2568                 :     158086 :                 vma = remove_vma(vma);
    2569         [ +  + ]:     158086 :         } while (vma);
    2570                 :     158034 :         vm_unacct_memory(nr_accounted);
    2571                 :     158034 :         validate_mm(mm);
    2572                 :     158034 : }
    2573                 :            : 
    2574                 :            : /*
    2575                 :            :  * Get rid of page table information in the indicated region.
    2576                 :            :  *
    2577                 :            :  * Called with the mm semaphore held.
    2578                 :            :  */
    2579                 :     158034 : static void unmap_region(struct mm_struct *mm,
    2580                 :            :                 struct vm_area_struct *vma, struct vm_area_struct *prev,
    2581                 :            :                 unsigned long start, unsigned long end)
    2582                 :            : {
    2583         [ +  - ]:     158034 :         struct vm_area_struct *next = prev ? prev->vm_next : mm->mmap;
    2584                 :     158034 :         struct mmu_gather tlb;
    2585                 :            : 
    2586                 :     158034 :         lru_add_drain();
    2587                 :     158034 :         tlb_gather_mmu(&tlb, mm, start, end);
    2588                 :     158034 :         update_hiwater_rss(mm);
    2589                 :     158034 :         unmap_vmas(&tlb, vma, start, end);
    2590   [ +  -  +  - ]:     158034 :         free_pgtables(&tlb, vma, prev ? prev->vm_end : FIRST_USER_ADDRESS,
    2591                 :            :                                  next ? next->vm_start : USER_PGTABLES_CEILING);
    2592                 :     158034 :         tlb_finish_mmu(&tlb, start, end);
    2593                 :     158034 : }
    2594                 :            : 
    2595                 :            : /*
    2596                 :            :  * Create a list of vma's touched by the unmap, removing them from the mm's
    2597                 :            :  * vma list as we go..
    2598                 :            :  */
    2599                 :            : static void
    2600                 :     158034 : detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
    2601                 :            :         struct vm_area_struct *prev, unsigned long end)
    2602                 :            : {
    2603                 :     158034 :         struct vm_area_struct **insertion_point;
    2604                 :     158034 :         struct vm_area_struct *tail_vma = NULL;
    2605                 :            : 
    2606         [ +  - ]:     158034 :         insertion_point = (prev ? &prev->vm_next : &mm->mmap);
    2607                 :     158034 :         vma->vm_prev = NULL;
    2608                 :     158086 :         do {
    2609                 :     158086 :                 vma_rb_erase(vma, &mm->mm_rb);
    2610                 :     158086 :                 mm->map_count--;
    2611                 :     158086 :                 tail_vma = vma;
    2612                 :     158086 :                 vma = vma->vm_next;
    2613   [ +  -  +  + ]:     158086 :         } while (vma && vma->vm_start < end);
    2614                 :     158034 :         *insertion_point = vma;
    2615         [ +  - ]:     158034 :         if (vma) {
    2616                 :     158034 :                 vma->vm_prev = prev;
    2617                 :     158034 :                 vma_gap_update(vma);
    2618                 :            :         } else
    2619         [ #  # ]:          0 :                 mm->highest_vm_end = prev ? vm_end_gap(prev) : 0;
    2620                 :     158034 :         tail_vma->vm_next = NULL;
    2621                 :            : 
    2622                 :            :         /* Kill the cache */
    2623                 :     158034 :         vmacache_invalidate(mm);
    2624                 :     158034 : }
    2625                 :            : 
    2626                 :            : /*
    2627                 :            :  * __split_vma() bypasses sysctl_max_map_count checking.  We use this where it
    2628                 :            :  * has already been checked or doesn't make sense to fail.
    2629                 :            :  */
    2630                 :     238528 : int __split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
    2631                 :            :                 unsigned long addr, int new_below)
    2632                 :            : {
    2633                 :     238528 :         struct vm_area_struct *new;
    2634                 :     238528 :         int err;
    2635                 :            : 
    2636   [ +  +  -  + ]:     238528 :         if (vma->vm_ops && vma->vm_ops->split) {
    2637                 :          0 :                 err = vma->vm_ops->split(vma, addr);
    2638         [ #  # ]:          0 :                 if (err)
    2639                 :            :                         return err;
    2640                 :            :         }
    2641                 :            : 
    2642                 :     238528 :         new = vm_area_dup(vma);
    2643         [ +  - ]:     238528 :         if (!new)
    2644                 :            :                 return -ENOMEM;
    2645                 :            : 
    2646         [ +  + ]:     238528 :         if (new_below)
    2647                 :     103767 :                 new->vm_end = addr;
    2648                 :            :         else {
    2649                 :     134761 :                 new->vm_start = addr;
    2650                 :     134761 :                 new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
    2651                 :            :         }
    2652                 :            : 
    2653                 :     238528 :         err = vma_dup_policy(vma, new);
    2654         [ -  + ]:     238528 :         if (err)
    2655                 :          0 :                 goto out_free_vma;
    2656                 :            : 
    2657                 :     238528 :         err = anon_vma_clone(new, vma);
    2658         [ -  + ]:     238528 :         if (err)
    2659                 :          0 :                 goto out_free_mpol;
    2660                 :            : 
    2661         [ +  + ]:     238528 :         if (new->vm_file)
    2662                 :     237640 :                 get_file(new->vm_file);
    2663                 :            : 
    2664   [ +  +  -  + ]:     238528 :         if (new->vm_ops && new->vm_ops->open)
    2665                 :          0 :                 new->vm_ops->open(new);
    2666                 :            : 
    2667         [ +  + ]:     238528 :         if (new_below)
    2668                 :     103767 :                 err = vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
    2669                 :     103767 :                         ((addr - new->vm_start) >> PAGE_SHIFT), new);
    2670                 :            :         else
    2671                 :     134761 :                 err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
    2672                 :            : 
    2673                 :            :         /* Success. */
    2674         [ -  + ]:     238528 :         if (!err)
    2675                 :            :                 return 0;
    2676                 :            : 
    2677                 :            :         /* Clean everything up if vma_adjust failed. */
    2678   [ #  #  #  # ]:          0 :         if (new->vm_ops && new->vm_ops->close)
    2679                 :          0 :                 new->vm_ops->close(new);
    2680         [ #  # ]:          0 :         if (new->vm_file)
    2681                 :          0 :                 fput(new->vm_file);
    2682                 :          0 :         unlink_anon_vmas(new);
    2683                 :          0 :  out_free_mpol:
    2684         [ #  # ]:          0 :         mpol_put(vma_policy(new));
    2685                 :          0 :  out_free_vma:
    2686                 :          0 :         vm_area_free(new);
    2687                 :          0 :         return err;
    2688                 :            : }
    2689                 :            : 
    2690                 :            : /*
    2691                 :            :  * Split a vma into two pieces at address 'addr', a new vma is allocated
    2692                 :            :  * either for the first part or the tail.
    2693                 :            :  */
    2694                 :      95587 : int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
    2695                 :            :               unsigned long addr, int new_below)
    2696                 :            : {
    2697         [ +  - ]:      95587 :         if (mm->map_count >= sysctl_max_map_count)
    2698                 :            :                 return -ENOMEM;
    2699                 :            : 
    2700                 :      95587 :         return __split_vma(mm, vma, addr, new_below);
    2701                 :            : }
    2702                 :            : 
    2703                 :            : /* Munmap is split into 2 main parts -- this part which finds
    2704                 :            :  * what needs doing, and the areas themselves, which do the
    2705                 :            :  * work.  This now handles partial unmappings.
    2706                 :            :  * Jeremy Fitzhardinge <jeremy@goop.org>
    2707                 :            :  */
    2708                 :     158034 : int __do_munmap(struct mm_struct *mm, unsigned long start, size_t len,
    2709                 :            :                 struct list_head *uf, bool downgrade)
    2710                 :            : {
    2711                 :     158034 :         unsigned long end;
    2712                 :     158034 :         struct vm_area_struct *vma, *prev, *last;
    2713                 :            : 
    2714   [ +  -  -  +  :     790170 :         if ((offset_in_page(start)) || start > TASK_SIZE || len > TASK_SIZE-start)
          -  -  +  -  -  
             +  -  -  -  
                      + ]
    2715                 :          0 :                 return -EINVAL;
    2716                 :            : 
    2717                 :     158034 :         len = PAGE_ALIGN(len);
    2718                 :     158034 :         end = start + len;
    2719         [ +  - ]:     158034 :         if (len == 0)
    2720                 :            :                 return -EINVAL;
    2721                 :            : 
    2722                 :            :         /*
    2723                 :            :          * arch_unmap() might do unmaps itself.  It must be called
    2724                 :            :          * and finish any rbtree manipulation before this code
    2725                 :            :          * runs and also starts to manipulate the rbtree.
    2726                 :            :          */
    2727                 :     158034 :         arch_unmap(mm, start, end);
    2728                 :            : 
    2729                 :            :         /* Find the first overlapping VMA */
    2730                 :     158034 :         vma = find_vma(mm, start);
    2731         [ +  - ]:     158034 :         if (!vma)
    2732                 :            :                 return 0;
    2733                 :     158034 :         prev = vma->vm_prev;
    2734                 :            :         /* we have  start < vma->vm_end  */
    2735                 :            : 
    2736                 :            :         /* if it doesn't overlap, we have nothing.. */
    2737         [ +  - ]:     158034 :         if (vma->vm_start >= end)
    2738                 :            :                 return 0;
    2739                 :            : 
    2740                 :            :         /*
    2741                 :            :          * If we need to split any vma, do it now to save pain later.
    2742                 :            :          *
    2743                 :            :          * Note: mremap's move_vma VM_ACCOUNT handling assumes a partially
    2744                 :            :          * unmapped vm_area_struct will remain in use: so lower split_vma
    2745                 :            :          * places tmp vma above, and higher split_vma places tmp vma below.
    2746                 :            :          */
    2747         [ +  + ]:     158034 :         if (start > vma->vm_start) {
    2748                 :      57919 :                 int error;
    2749                 :            : 
    2750                 :            :                 /*
    2751                 :            :                  * Make sure that map_count on return from munmap() will
    2752                 :            :                  * not exceed its limit; but let map_count go just above
    2753                 :            :                  * its limit temporarily, to help free resources as expected.
    2754                 :            :                  */
    2755   [ +  +  +  - ]:      57919 :                 if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
    2756                 :            :                         return -ENOMEM;
    2757                 :            : 
    2758                 :      57919 :                 error = __split_vma(mm, vma, start, 0);
    2759         [ +  - ]:      57919 :                 if (error)
    2760                 :            :                         return error;
    2761                 :            :                 prev = vma;
    2762                 :            :         }
    2763                 :            : 
    2764                 :            :         /* Does it split the last one? */
    2765                 :     158034 :         last = find_vma(mm, end);
    2766   [ +  -  +  + ]:     158034 :         if (last && end > last->vm_start) {
    2767                 :      85022 :                 int error = __split_vma(mm, last, end, 1);
    2768         [ +  - ]:      85022 :                 if (error)
    2769                 :            :                         return error;
    2770                 :            :         }
    2771         [ +  - ]:     158034 :         vma = prev ? prev->vm_next : mm->mmap;
    2772                 :            : 
    2773                 :     158034 :         if (unlikely(uf)) {
    2774                 :            :                 /*
    2775                 :            :                  * If userfaultfd_unmap_prep returns an error the vmas
    2776                 :            :                  * will remain splitted, but userland will get a
    2777                 :            :                  * highly unexpected error anyway. This is no
    2778                 :            :                  * different than the case where the first of the two
    2779                 :            :                  * __split_vma fails, but we don't undo the first
    2780                 :            :                  * split, despite we could. This is unlikely enough
    2781                 :            :                  * failure that it's not worth optimizing it for.
    2782                 :            :                  */
    2783                 :            :                 int error = userfaultfd_unmap_prep(vma, start, end, uf);
    2784                 :            :                 if (error)
    2785                 :            :                         return error;
    2786                 :            :         }
    2787                 :            : 
    2788                 :            :         /*
    2789                 :            :          * unlock any mlock()ed ranges before detaching vmas
    2790                 :            :          */
    2791         [ -  + ]:     158034 :         if (mm->locked_vm) {
    2792                 :            :                 struct vm_area_struct *tmp = vma;
    2793   [ #  #  #  # ]:          0 :                 while (tmp && tmp->vm_start < end) {
    2794         [ #  # ]:          0 :                         if (tmp->vm_flags & VM_LOCKED) {
    2795                 :          0 :                                 mm->locked_vm -= vma_pages(tmp);
    2796                 :          0 :                                 munlock_vma_pages_all(tmp);
    2797                 :            :                         }
    2798                 :            : 
    2799                 :          0 :                         tmp = tmp->vm_next;
    2800                 :            :                 }
    2801                 :            :         }
    2802                 :            : 
    2803                 :            :         /* Detach vmas from rbtree */
    2804                 :     158034 :         detach_vmas_to_be_unmapped(mm, vma, prev, end);
    2805                 :            : 
    2806         [ +  + ]:     158034 :         if (downgrade)
    2807                 :      15138 :                 downgrade_write(&mm->mmap_sem);
    2808                 :            : 
    2809                 :     158034 :         unmap_region(mm, vma, prev, start, end);
    2810                 :            : 
    2811                 :            :         /* Fix up all other VM information */
    2812                 :     158034 :         remove_vma_list(mm, vma);
    2813                 :            : 
    2814                 :     158034 :         return downgrade ? 1 : 0;
    2815                 :            : }
    2816                 :            : 
    2817                 :     121524 : int do_munmap(struct mm_struct *mm, unsigned long start, size_t len,
    2818                 :            :               struct list_head *uf)
    2819                 :            : {
    2820                 :     121524 :         return __do_munmap(mm, start, len, uf, false);
    2821                 :            : }
    2822                 :            : 
    2823                 :      36078 : static int __vm_munmap(unsigned long start, size_t len, bool downgrade)
    2824                 :            : {
    2825                 :      36078 :         int ret;
    2826                 :      36078 :         struct mm_struct *mm = current->mm;
    2827                 :      36078 :         LIST_HEAD(uf);
    2828                 :            : 
    2829         [ +  - ]:      36078 :         if (down_write_killable(&mm->mmap_sem))
    2830                 :            :                 return -EINTR;
    2831                 :            : 
    2832                 :      36078 :         ret = __do_munmap(mm, start, len, &uf, downgrade);
    2833                 :            :         /*
    2834                 :            :          * Returning 1 indicates mmap_sem is downgraded.
    2835                 :            :          * But 1 is not legal return value of vm_munmap() and munmap(), reset
    2836                 :            :          * it to 0 before return.
    2837                 :            :          */
    2838         [ +  + ]:      36078 :         if (ret == 1) {
    2839                 :      14706 :                 up_read(&mm->mmap_sem);
    2840                 :      14706 :                 ret = 0;
    2841                 :            :         } else
    2842                 :      21372 :                 up_write(&mm->mmap_sem);
    2843                 :            : 
    2844                 :            :         userfaultfd_unmap_complete(mm, &uf);
    2845                 :            :         return ret;
    2846                 :            : }
    2847                 :            : 
    2848                 :      21372 : int vm_munmap(unsigned long start, size_t len)
    2849                 :            : {
    2850                 :      21372 :         return __vm_munmap(start, len, false);
    2851                 :            : }
    2852                 :            : EXPORT_SYMBOL(vm_munmap);
    2853                 :            : 
    2854                 :      29412 : SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
    2855                 :            : {
    2856                 :      14706 :         addr = untagged_addr(addr);
    2857                 :      14706 :         profile_munmap(addr);
    2858                 :      14706 :         return __vm_munmap(addr, len, true);
    2859                 :            : }
    2860                 :            : 
    2861                 :            : 
    2862                 :            : /*
    2863                 :            :  * Emulation of deprecated remap_file_pages() syscall.
    2864                 :            :  */
    2865                 :          0 : SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size,
    2866                 :            :                 unsigned long, prot, unsigned long, pgoff, unsigned long, flags)
    2867                 :            : {
    2868                 :            : 
    2869         [ #  # ]:          0 :         struct mm_struct *mm = current->mm;
    2870                 :          0 :         struct vm_area_struct *vma;
    2871                 :          0 :         unsigned long populate = 0;
    2872                 :          0 :         unsigned long ret = -EINVAL;
    2873                 :          0 :         struct file *file;
    2874                 :            : 
    2875         [ #  # ]:          0 :         pr_warn_once("%s (%d) uses deprecated remap_file_pages() syscall. See Documentation/vm/remap_file_pages.rst.\n",
    2876                 :            :                      current->comm, current->pid);
    2877                 :            : 
    2878         [ #  # ]:          0 :         if (prot)
    2879                 :            :                 return ret;
    2880                 :          0 :         start = start & PAGE_MASK;
    2881                 :          0 :         size = size & PAGE_MASK;
    2882                 :            : 
    2883         [ #  # ]:          0 :         if (start + size <= start)
    2884                 :            :                 return ret;
    2885                 :            : 
    2886                 :            :         /* Does pgoff wrap? */
    2887         [ #  # ]:          0 :         if (pgoff + (size >> PAGE_SHIFT) < pgoff)
    2888                 :            :                 return ret;
    2889                 :            : 
    2890         [ #  # ]:          0 :         if (down_write_killable(&mm->mmap_sem))
    2891                 :            :                 return -EINTR;
    2892                 :            : 
    2893                 :          0 :         vma = find_vma(mm, start);
    2894                 :            : 
    2895   [ #  #  #  # ]:          0 :         if (!vma || !(vma->vm_flags & VM_SHARED))
    2896                 :          0 :                 goto out;
    2897                 :            : 
    2898         [ #  # ]:          0 :         if (start < vma->vm_start)
    2899                 :          0 :                 goto out;
    2900                 :            : 
    2901         [ #  # ]:          0 :         if (start + size > vma->vm_end) {
    2902                 :          0 :                 struct vm_area_struct *next;
    2903                 :            : 
    2904         [ #  # ]:          0 :                 for (next = vma->vm_next; next; next = next->vm_next) {
    2905                 :            :                         /* hole between vmas ? */
    2906         [ #  # ]:          0 :                         if (next->vm_start != next->vm_prev->vm_end)
    2907                 :          0 :                                 goto out;
    2908                 :            : 
    2909         [ #  # ]:          0 :                         if (next->vm_file != vma->vm_file)
    2910                 :          0 :                                 goto out;
    2911                 :            : 
    2912         [ #  # ]:          0 :                         if (next->vm_flags != vma->vm_flags)
    2913                 :          0 :                                 goto out;
    2914                 :            : 
    2915         [ #  # ]:          0 :                         if (start + size <= next->vm_end)
    2916                 :            :                                 break;
    2917                 :            :                 }
    2918                 :            : 
    2919         [ #  # ]:          0 :                 if (!next)
    2920                 :          0 :                         goto out;
    2921                 :            :         }
    2922                 :            : 
    2923                 :          0 :         prot |= vma->vm_flags & VM_READ ? PROT_READ : 0;
    2924                 :          0 :         prot |= vma->vm_flags & VM_WRITE ? PROT_WRITE : 0;
    2925                 :          0 :         prot |= vma->vm_flags & VM_EXEC ? PROT_EXEC : 0;
    2926                 :            : 
    2927                 :          0 :         flags &= MAP_NONBLOCK;
    2928                 :          0 :         flags |= MAP_SHARED | MAP_FIXED | MAP_POPULATE;
    2929         [ #  # ]:          0 :         if (vma->vm_flags & VM_LOCKED) {
    2930                 :          0 :                 struct vm_area_struct *tmp;
    2931                 :          0 :                 flags |= MAP_LOCKED;
    2932                 :            : 
    2933                 :            :                 /* drop PG_Mlocked flag for over-mapped range */
    2934         [ #  # ]:          0 :                 for (tmp = vma; tmp->vm_start >= start + size;
    2935                 :          0 :                                 tmp = tmp->vm_next) {
    2936                 :            :                         /*
    2937                 :            :                          * Split pmd and munlock page on the border
    2938                 :            :                          * of the range.
    2939                 :            :                          */
    2940                 :          0 :                         vma_adjust_trans_huge(tmp, start, start + size, 0);
    2941                 :            : 
    2942                 :          0 :                         munlock_vma_pages_range(tmp,
    2943                 :          0 :                                         max(tmp->vm_start, start),
    2944                 :          0 :                                         min(tmp->vm_end, start + size));
    2945                 :            :                 }
    2946                 :            :         }
    2947                 :            : 
    2948                 :          0 :         file = get_file(vma->vm_file);
    2949                 :          0 :         ret = do_mmap_pgoff(vma->vm_file, start, size,
    2950                 :            :                         prot, flags, pgoff, &populate, NULL);
    2951                 :          0 :         fput(file);
    2952                 :          0 : out:
    2953                 :          0 :         up_write(&mm->mmap_sem);
    2954         [ #  # ]:          0 :         if (populate)
    2955                 :          0 :                 mm_populate(ret, populate);
    2956         [ #  # ]:          0 :         if (!IS_ERR_VALUE(ret))
    2957                 :          0 :                 ret = 0;
    2958                 :          0 :         return ret;
    2959                 :            : }
    2960                 :            : 
    2961                 :            : /*
    2962                 :            :  *  this is really a simplified "do_mmap".  it only handles
    2963                 :            :  *  anonymous maps.  eventually we may be able to do some
    2964                 :            :  *  brk-specific accounting here.
    2965                 :            :  */
    2966                 :      25223 : static int do_brk_flags(unsigned long addr, unsigned long len, unsigned long flags, struct list_head *uf)
    2967                 :            : {
    2968         [ +  - ]:      25223 :         struct mm_struct *mm = current->mm;
    2969                 :      25223 :         struct vm_area_struct *vma, *prev;
    2970                 :      25223 :         struct rb_node **rb_link, *rb_parent;
    2971                 :      25223 :         pgoff_t pgoff = addr >> PAGE_SHIFT;
    2972                 :      25223 :         int error;
    2973                 :      25223 :         unsigned long mapped_addr;
    2974                 :            : 
    2975                 :            :         /* Until we need other flags, refuse anything except VM_EXEC. */
    2976         [ +  - ]:      25223 :         if ((flags & (~VM_EXEC)) != 0)
    2977                 :            :                 return -EINVAL;
    2978         [ +  - ]:      25223 :         flags |= VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
    2979                 :            : 
    2980                 :      25223 :         mapped_addr = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
    2981         [ -  + ]:      25223 :         if (IS_ERR_VALUE(mapped_addr))
    2982                 :          0 :                 return mapped_addr;
    2983                 :            : 
    2984                 :      25223 :         error = mlock_future_check(mm, mm->def_flags, len);
    2985         [ +  - ]:      25223 :         if (error)
    2986                 :            :                 return error;
    2987                 :            : 
    2988                 :            :         /*
    2989                 :            :          * Clear old maps.  this also does some error checking for us
    2990                 :            :          */
    2991         [ -  + ]:      50446 :         while (find_vma_links(mm, addr, addr + len, &prev, &rb_link,
    2992                 :            :                               &rb_parent)) {
    2993         [ #  # ]:          0 :                 if (do_munmap(mm, addr, len, uf))
    2994                 :            :                         return -ENOMEM;
    2995                 :            :         }
    2996                 :            : 
    2997                 :            :         /* Check against address space limits *after* clearing old maps... */
    2998         [ +  - ]:      25223 :         if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT))
    2999                 :            :                 return -ENOMEM;
    3000                 :            : 
    3001         [ +  - ]:      25223 :         if (mm->map_count > sysctl_max_map_count)
    3002                 :            :                 return -ENOMEM;
    3003                 :            : 
    3004         [ +  - ]:      25223 :         if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT))
    3005                 :            :                 return -ENOMEM;
    3006                 :            : 
    3007                 :            :         /* Can we just expand an old private anonymous mapping? */
    3008                 :      25223 :         vma = vma_merge(mm, prev, addr, addr + len, flags,
    3009                 :            :                         NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX);
    3010         [ +  + ]:      25223 :         if (vma)
    3011                 :        227 :                 goto out;
    3012                 :            : 
    3013                 :            :         /*
    3014                 :            :          * create a vma struct for an anonymous mapping
    3015                 :            :          */
    3016                 :      24996 :         vma = vm_area_alloc(mm);
    3017         [ -  + ]:      24996 :         if (!vma) {
    3018                 :          0 :                 vm_unacct_memory(len >> PAGE_SHIFT);
    3019                 :          0 :                 return -ENOMEM;
    3020                 :            :         }
    3021                 :            : 
    3022         [ +  - ]:      24996 :         vma_set_anonymous(vma);
    3023                 :      24996 :         vma->vm_start = addr;
    3024                 :      24996 :         vma->vm_end = addr + len;
    3025                 :      24996 :         vma->vm_pgoff = pgoff;
    3026                 :      24996 :         vma->vm_flags = flags;
    3027         [ +  - ]:      24996 :         vma->vm_page_prot = vm_get_page_prot(flags);
    3028                 :      24996 :         vma_link(mm, vma, prev, rb_link, rb_parent);
    3029                 :      25223 : out:
    3030                 :      25223 :         perf_event_mmap(vma);
    3031                 :      25223 :         mm->total_vm += len >> PAGE_SHIFT;
    3032                 :      25223 :         mm->data_vm += len >> PAGE_SHIFT;
    3033         [ -  + ]:      25223 :         if (flags & VM_LOCKED)
    3034                 :          0 :                 mm->locked_vm += (len >> PAGE_SHIFT);
    3035                 :      25223 :         vma->vm_flags |= VM_SOFTDIRTY;
    3036                 :      25223 :         return 0;
    3037                 :            : }
    3038                 :            : 
    3039                 :      14313 : int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags)
    3040                 :            : {
    3041         [ +  - ]:      14313 :         struct mm_struct *mm = current->mm;
    3042                 :      14313 :         unsigned long len;
    3043                 :      14313 :         int ret;
    3044                 :      14313 :         bool populate;
    3045                 :      14313 :         LIST_HEAD(uf);
    3046                 :            : 
    3047                 :      14313 :         len = PAGE_ALIGN(request);
    3048         [ +  - ]:      14313 :         if (len < request)
    3049                 :            :                 return -ENOMEM;
    3050         [ +  - ]:      14313 :         if (!len)
    3051                 :            :                 return 0;
    3052                 :            : 
    3053         [ +  - ]:      14313 :         if (down_write_killable(&mm->mmap_sem))
    3054                 :            :                 return -EINTR;
    3055                 :            : 
    3056                 :      14313 :         ret = do_brk_flags(addr, len, flags, &uf);
    3057                 :      14313 :         populate = ((mm->def_flags & VM_LOCKED) != 0);
    3058                 :      14313 :         up_write(&mm->mmap_sem);
    3059         [ -  + ]:      14313 :         userfaultfd_unmap_complete(mm, &uf);
    3060         [ -  + ]:      14313 :         if (populate && !ret)
    3061                 :          0 :                 mm_populate(addr, len);
    3062                 :            :         return ret;
    3063                 :            : }
    3064                 :            : EXPORT_SYMBOL(vm_brk_flags);
    3065                 :            : 
    3066                 :          0 : int vm_brk(unsigned long addr, unsigned long len)
    3067                 :            : {
    3068                 :          0 :         return vm_brk_flags(addr, len, 0);
    3069                 :            : }
    3070                 :            : EXPORT_SYMBOL(vm_brk);
    3071                 :            : 
    3072                 :            : /* Release all mmaps. */
    3073                 :      21164 : void exit_mmap(struct mm_struct *mm)
    3074                 :            : {
    3075                 :      21164 :         struct mmu_gather tlb;
    3076                 :      21164 :         struct vm_area_struct *vma;
    3077                 :      21164 :         unsigned long nr_accounted = 0;
    3078                 :            : 
    3079                 :            :         /* mm's last user has gone, and its about to be pulled down */
    3080         [ -  + ]:      21164 :         mmu_notifier_release(mm);
    3081                 :            : 
    3082         [ -  + ]:      21164 :         if (unlikely(mm_is_oom_victim(mm))) {
    3083                 :            :                 /*
    3084                 :            :                  * Manually reap the mm to free as much memory as possible.
    3085                 :            :                  * Then, as the oom reaper does, set MMF_OOM_SKIP to disregard
    3086                 :            :                  * this mm from further consideration.  Taking mm->mmap_sem for
    3087                 :            :                  * write after setting MMF_OOM_SKIP will guarantee that the oom
    3088                 :            :                  * reaper will not run on this mm again after mmap_sem is
    3089                 :            :                  * dropped.
    3090                 :            :                  *
    3091                 :            :                  * Nothing can be holding mm->mmap_sem here and the above call
    3092                 :            :                  * to mmu_notifier_release(mm) ensures mmu notifier callbacks in
    3093                 :            :                  * __oom_reap_task_mm() will not block.
    3094                 :            :                  *
    3095                 :            :                  * This needs to be done before calling munlock_vma_pages_all(),
    3096                 :            :                  * which clears VM_LOCKED, otherwise the oom reaper cannot
    3097                 :            :                  * reliably test it.
    3098                 :            :                  */
    3099                 :          0 :                 (void)__oom_reap_task_mm(mm);
    3100                 :            : 
    3101                 :          0 :                 set_bit(MMF_OOM_SKIP, &mm->flags);
    3102                 :          0 :                 down_write(&mm->mmap_sem);
    3103                 :          0 :                 up_write(&mm->mmap_sem);
    3104                 :            :         }
    3105                 :            : 
    3106         [ -  + ]:      21164 :         if (mm->locked_vm) {
    3107                 :          0 :                 vma = mm->mmap;
    3108         [ #  # ]:          0 :                 while (vma) {
    3109         [ #  # ]:          0 :                         if (vma->vm_flags & VM_LOCKED)
    3110                 :          0 :                                 munlock_vma_pages_all(vma);
    3111                 :          0 :                         vma = vma->vm_next;
    3112                 :            :                 }
    3113                 :            :         }
    3114                 :            : 
    3115                 :      21164 :         arch_exit_mmap(mm);
    3116                 :            : 
    3117                 :      21164 :         vma = mm->mmap;
    3118         [ -  + ]:      21164 :         if (!vma)       /* Can happen if dup_mmap() received an OOM */
    3119                 :          0 :                 return;
    3120                 :            : 
    3121                 :      21164 :         lru_add_drain();
    3122                 :      21164 :         flush_cache_mm(mm);
    3123                 :      21164 :         tlb_gather_mmu(&tlb, mm, 0, -1);
    3124                 :            :         /* update_hiwater_rss(mm) here? but nobody should be looking */
    3125                 :            :         /* Use -1 here to ensure all VMAs in the mm are unmapped */
    3126                 :      21164 :         unmap_vmas(&tlb, vma, 0, -1);
    3127                 :      21164 :         free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, USER_PGTABLES_CEILING);
    3128                 :      21164 :         tlb_finish_mmu(&tlb, 0, -1);
    3129                 :            : 
    3130                 :            :         /*
    3131                 :            :          * Walk the list again, actually closing and freeing it,
    3132                 :            :          * with preemption enabled, without holding any MM locks.
    3133                 :            :          */
    3134         [ +  + ]:    1089410 :         while (vma) {
    3135         [ +  + ]:    1068246 :                 if (vma->vm_flags & VM_ACCOUNT)
    3136                 :     422835 :                         nr_accounted += vma_pages(vma);
    3137                 :    1068246 :                 vma = remove_vma(vma);
    3138                 :            :         }
    3139                 :      21164 :         vm_unacct_memory(nr_accounted);
    3140                 :            : }
    3141                 :            : 
    3142                 :            : /* Insert vm structure into process list sorted by address
    3143                 :            :  * and into the inode's i_mmap tree.  If vm_file is non-NULL
    3144                 :            :  * then i_mmap_rwsem is taken here.
    3145                 :            :  */
    3146                 :      32058 : int insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
    3147                 :            : {
    3148                 :      32058 :         struct vm_area_struct *prev;
    3149                 :      32058 :         struct rb_node **rb_link, *rb_parent;
    3150                 :            : 
    3151         [ +  - ]:      64116 :         if (find_vma_links(mm, vma->vm_start, vma->vm_end,
    3152                 :            :                            &prev, &rb_link, &rb_parent))
    3153                 :            :                 return -ENOMEM;
    3154   [ +  +  +  - ]:      42744 :         if ((vma->vm_flags & VM_ACCOUNT) &&
    3155                 :      10686 :              security_vm_enough_memory_mm(mm, vma_pages(vma)))
    3156                 :            :                 return -ENOMEM;
    3157                 :            : 
    3158                 :            :         /*
    3159                 :            :          * The vm_pgoff of a purely anonymous vma should be irrelevant
    3160                 :            :          * until its first write fault, when page's anon_vma and index
    3161                 :            :          * are set.  But now set the vm_pgoff it will almost certainly
    3162                 :            :          * end up with (unless mremap moves it elsewhere before that
    3163                 :            :          * first wfault), so /proc/pid/maps tells a consistent story.
    3164                 :            :          *
    3165                 :            :          * By setting it to reflect the virtual start address of the
    3166                 :            :          * vma, merges and splits can happen in a seamless way, just
    3167                 :            :          * using the existing file pgoff checks and manipulations.
    3168                 :            :          * Similarly in do_mmap_pgoff and in do_brk.
    3169                 :            :          */
    3170         [ +  + ]:      32058 :         if (vma_is_anonymous(vma)) {
    3171         [ -  + ]:      10686 :                 BUG_ON(vma->anon_vma);
    3172                 :      10686 :                 vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
    3173                 :            :         }
    3174                 :            : 
    3175                 :      32058 :         vma_link(mm, vma, prev, rb_link, rb_parent);
    3176                 :      32058 :         return 0;
    3177                 :            : }
    3178                 :            : 
    3179                 :            : /*
    3180                 :            :  * Copy the vma structure to a new location in the same mm,
    3181                 :            :  * prior to moving page table entries, to effect an mremap move.
    3182                 :            :  */
    3183                 :          0 : struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
    3184                 :            :         unsigned long addr, unsigned long len, pgoff_t pgoff,
    3185                 :            :         bool *need_rmap_locks)
    3186                 :            : {
    3187                 :          0 :         struct vm_area_struct *vma = *vmap;
    3188                 :          0 :         unsigned long vma_start = vma->vm_start;
    3189                 :          0 :         struct mm_struct *mm = vma->vm_mm;
    3190                 :          0 :         struct vm_area_struct *new_vma, *prev;
    3191                 :          0 :         struct rb_node **rb_link, *rb_parent;
    3192                 :          0 :         bool faulted_in_anon_vma = true;
    3193                 :            : 
    3194                 :            :         /*
    3195                 :            :          * If anonymous vma has not yet been faulted, update new pgoff
    3196                 :            :          * to match new location, to increase its chance of merging.
    3197                 :            :          */
    3198   [ #  #  #  # ]:          0 :         if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) {
    3199                 :          0 :                 pgoff = addr >> PAGE_SHIFT;
    3200                 :          0 :                 faulted_in_anon_vma = false;
    3201                 :            :         }
    3202                 :            : 
    3203         [ #  # ]:          0 :         if (find_vma_links(mm, addr, addr + len, &prev, &rb_link, &rb_parent))
    3204                 :            :                 return NULL;    /* should never get here */
    3205                 :          0 :         new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
    3206                 :            :                             vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
    3207                 :            :                             vma->vm_userfaultfd_ctx);
    3208         [ #  # ]:          0 :         if (new_vma) {
    3209                 :            :                 /*
    3210                 :            :                  * Source vma may have been merged into new_vma
    3211                 :            :                  */
    3212   [ #  #  #  # ]:          0 :                 if (unlikely(vma_start >= new_vma->vm_start &&
    3213                 :            :                              vma_start < new_vma->vm_end)) {
    3214                 :            :                         /*
    3215                 :            :                          * The only way we can get a vma_merge with
    3216                 :            :                          * self during an mremap is if the vma hasn't
    3217                 :            :                          * been faulted in yet and we were allowed to
    3218                 :            :                          * reset the dst vma->vm_pgoff to the
    3219                 :            :                          * destination address of the mremap to allow
    3220                 :            :                          * the merge to happen. mremap must change the
    3221                 :            :                          * vm_pgoff linearity between src and dst vmas
    3222                 :            :                          * (in turn preventing a vma_merge) to be
    3223                 :            :                          * safe. It is only safe to keep the vm_pgoff
    3224                 :            :                          * linear if there are no pages mapped yet.
    3225                 :            :                          */
    3226                 :          0 :                         VM_BUG_ON_VMA(faulted_in_anon_vma, new_vma);
    3227                 :          0 :                         *vmap = vma = new_vma;
    3228                 :            :                 }
    3229                 :          0 :                 *need_rmap_locks = (new_vma->vm_pgoff <= vma->vm_pgoff);
    3230                 :            :         } else {
    3231                 :          0 :                 new_vma = vm_area_dup(vma);
    3232         [ #  # ]:          0 :                 if (!new_vma)
    3233                 :          0 :                         goto out;
    3234                 :          0 :                 new_vma->vm_start = addr;
    3235                 :          0 :                 new_vma->vm_end = addr + len;
    3236                 :          0 :                 new_vma->vm_pgoff = pgoff;
    3237         [ #  # ]:          0 :                 if (vma_dup_policy(vma, new_vma))
    3238                 :          0 :                         goto out_free_vma;
    3239         [ #  # ]:          0 :                 if (anon_vma_clone(new_vma, vma))
    3240                 :          0 :                         goto out_free_mempol;
    3241         [ #  # ]:          0 :                 if (new_vma->vm_file)
    3242                 :          0 :                         get_file(new_vma->vm_file);
    3243   [ #  #  #  # ]:          0 :                 if (new_vma->vm_ops && new_vma->vm_ops->open)
    3244                 :          0 :                         new_vma->vm_ops->open(new_vma);
    3245                 :          0 :                 vma_link(mm, new_vma, prev, rb_link, rb_parent);
    3246                 :          0 :                 *need_rmap_locks = false;
    3247                 :            :         }
    3248                 :            :         return new_vma;
    3249                 :            : 
    3250                 :            : out_free_mempol:
    3251         [ #  # ]:          0 :         mpol_put(vma_policy(new_vma));
    3252                 :          0 : out_free_vma:
    3253                 :          0 :         vm_area_free(new_vma);
    3254                 :            : out:
    3255                 :            :         return NULL;
    3256                 :            : }
    3257                 :            : 
    3258                 :            : /*
    3259                 :            :  * Return true if the calling process may expand its vm space by the passed
    3260                 :            :  * number of pages
    3261                 :            :  */
    3262                 :     398743 : bool may_expand_vm(struct mm_struct *mm, vm_flags_t flags, unsigned long npages)
    3263                 :            : {
    3264         [ +  - ]:     398743 :         if (mm->total_vm + npages > rlimit(RLIMIT_AS) >> PAGE_SHIFT)
    3265                 :            :                 return false;
    3266                 :            : 
    3267   [ +  +  -  + ]:     398743 :         if (is_data_mapping(flags) &&
    3268         [ -  + ]:     125366 :             mm->data_vm + npages > rlimit(RLIMIT_DATA) >> PAGE_SHIFT) {
    3269                 :            :                 /* Workaround for Valgrind */
    3270         [ #  # ]:          0 :                 if (rlimit(RLIMIT_DATA) == 0 &&
    3271         [ #  # ]:          0 :                     mm->data_vm + npages <= rlimit_max(RLIMIT_DATA) >> PAGE_SHIFT)
    3272                 :            :                         return true;
    3273                 :            : 
    3274   [ #  #  #  # ]:          0 :                 pr_warn_once("%s (%d): VmData %lu exceed data ulimit %lu. Update limits%s.\n",
    3275                 :            :                              current->comm, current->pid,
    3276                 :            :                              (mm->data_vm + npages) << PAGE_SHIFT,
    3277                 :            :                              rlimit(RLIMIT_DATA),
    3278                 :            :                              ignore_rlimit_data ? "" : " or use boot option ignore_rlimit_data");
    3279                 :            : 
    3280         [ #  # ]:          0 :                 if (!ignore_rlimit_data)
    3281                 :          0 :                         return false;
    3282                 :            :         }
    3283                 :            : 
    3284                 :            :         return true;
    3285                 :            : }
    3286                 :            : 
    3287                 :     707369 : void vm_stat_account(struct mm_struct *mm, vm_flags_t flags, long npages)
    3288                 :            : {
    3289                 :     707369 :         mm->total_vm += npages;
    3290                 :            : 
    3291   [ +  +  +  +  :     685997 :         if (is_exec_mapping(flags))
             -  +  +  + ]
    3292                 :      81016 :                 mm->exec_vm += npages;
    3293   [ -  +  -  +  :     626353 :         else if (is_stack_mapping(flags))
          -  +  +  -  -  
                      + ]
    3294                 :      10686 :                 mm->stack_vm += npages;
    3295   [ -  +  +  +  :     615667 :         else if (is_data_mapping(flags))
          +  +  -  -  +  
                      + ]
    3296                 :     161975 :                 mm->data_vm += npages;
    3297                 :     155252 : }
    3298                 :            : 
    3299                 :            : static vm_fault_t special_mapping_fault(struct vm_fault *vmf);
    3300                 :            : 
    3301                 :            : /*
    3302                 :            :  * Having a close hook prevents vma merging regardless of flags.
    3303                 :            :  */
    3304                 :      42328 : static void special_mapping_close(struct vm_area_struct *vma)
    3305                 :            : {
    3306                 :      42328 : }
    3307                 :            : 
    3308                 :          0 : static const char *special_mapping_name(struct vm_area_struct *vma)
    3309                 :            : {
    3310                 :          0 :         return ((struct vm_special_mapping *)vma->vm_private_data)->name;
    3311                 :            : }
    3312                 :            : 
    3313                 :          0 : static int special_mapping_mremap(struct vm_area_struct *new_vma)
    3314                 :            : {
    3315                 :          0 :         struct vm_special_mapping *sm = new_vma->vm_private_data;
    3316                 :            : 
    3317   [ #  #  #  # ]:          0 :         if (WARN_ON_ONCE(current->mm != new_vma->vm_mm))
    3318                 :            :                 return -EFAULT;
    3319                 :            : 
    3320         [ #  # ]:          0 :         if (sm->mremap)
    3321                 :          0 :                 return sm->mremap(sm, new_vma);
    3322                 :            : 
    3323                 :            :         return 0;
    3324                 :            : }
    3325                 :            : 
    3326                 :            : static const struct vm_operations_struct special_mapping_vmops = {
    3327                 :            :         .close = special_mapping_close,
    3328                 :            :         .fault = special_mapping_fault,
    3329                 :            :         .mremap = special_mapping_mremap,
    3330                 :            :         .name = special_mapping_name,
    3331                 :            :         /* vDSO code relies that VVAR can't be accessed remotely */
    3332                 :            :         .access = NULL,
    3333                 :            : };
    3334                 :            : 
    3335                 :            : static const struct vm_operations_struct legacy_special_mapping_vmops = {
    3336                 :            :         .close = special_mapping_close,
    3337                 :            :         .fault = special_mapping_fault,
    3338                 :            : };
    3339                 :            : 
    3340                 :      11724 : static vm_fault_t special_mapping_fault(struct vm_fault *vmf)
    3341                 :            : {
    3342                 :      11724 :         struct vm_area_struct *vma = vmf->vma;
    3343                 :      11724 :         pgoff_t pgoff;
    3344                 :      11724 :         struct page **pages;
    3345                 :            : 
    3346         [ -  + ]:      11724 :         if (vma->vm_ops == &legacy_special_mapping_vmops) {
    3347                 :          0 :                 pages = vma->vm_private_data;
    3348                 :            :         } else {
    3349                 :      11724 :                 struct vm_special_mapping *sm = vma->vm_private_data;
    3350                 :            : 
    3351         [ +  - ]:      11724 :                 if (sm->fault)
    3352                 :      11724 :                         return sm->fault(sm, vmf->vma, vmf);
    3353                 :            : 
    3354                 :          0 :                 pages = sm->pages;
    3355                 :            :         }
    3356                 :            : 
    3357   [ #  #  #  # ]:          0 :         for (pgoff = vmf->pgoff; pgoff && *pages; ++pages)
    3358                 :          0 :                 pgoff--;
    3359                 :            : 
    3360         [ #  # ]:          0 :         if (*pages) {
    3361                 :          0 :                 struct page *page = *pages;
    3362         [ #  # ]:          0 :                 get_page(page);
    3363                 :          0 :                 vmf->page = page;
    3364                 :          0 :                 return 0;
    3365                 :            :         }
    3366                 :            : 
    3367                 :            :         return VM_FAULT_SIGBUS;
    3368                 :            : }
    3369                 :            : 
    3370                 :      21372 : static struct vm_area_struct *__install_special_mapping(
    3371                 :            :         struct mm_struct *mm,
    3372                 :            :         unsigned long addr, unsigned long len,
    3373                 :            :         unsigned long vm_flags, void *priv,
    3374                 :            :         const struct vm_operations_struct *ops)
    3375                 :            : {
    3376                 :      21372 :         int ret;
    3377                 :      21372 :         struct vm_area_struct *vma;
    3378                 :            : 
    3379                 :      21372 :         vma = vm_area_alloc(mm);
    3380         [ +  - ]:      21372 :         if (unlikely(vma == NULL))
    3381                 :            :                 return ERR_PTR(-ENOMEM);
    3382                 :            : 
    3383                 :      21372 :         vma->vm_start = addr;
    3384                 :      21372 :         vma->vm_end = addr + len;
    3385                 :            : 
    3386                 :      21372 :         vma->vm_flags = vm_flags | mm->def_flags | VM_DONTEXPAND | VM_SOFTDIRTY;
    3387         [ +  - ]:      21372 :         vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
    3388                 :            : 
    3389                 :      21372 :         vma->vm_ops = ops;
    3390                 :      21372 :         vma->vm_private_data = priv;
    3391                 :            : 
    3392                 :      21372 :         ret = insert_vm_struct(mm, vma);
    3393         [ -  + ]:      21372 :         if (ret)
    3394                 :          0 :                 goto out;
    3395                 :            : 
    3396         [ +  + ]:      21372 :         vm_stat_account(mm, vma->vm_flags, len >> PAGE_SHIFT);
    3397                 :            : 
    3398                 :      21372 :         perf_event_mmap(vma);
    3399                 :            : 
    3400                 :      21372 :         return vma;
    3401                 :            : 
    3402                 :            : out:
    3403                 :          0 :         vm_area_free(vma);
    3404                 :          0 :         return ERR_PTR(ret);
    3405                 :            : }
    3406                 :            : 
    3407                 :          0 : bool vma_is_special_mapping(const struct vm_area_struct *vma,
    3408                 :            :         const struct vm_special_mapping *sm)
    3409                 :            : {
    3410         [ #  # ]:          0 :         return vma->vm_private_data == sm &&
    3411   [ #  #  #  # ]:          0 :                 (vma->vm_ops == &special_mapping_vmops ||
    3412                 :            :                  vma->vm_ops == &legacy_special_mapping_vmops);
    3413                 :            : }
    3414                 :            : 
    3415                 :            : /*
    3416                 :            :  * Called with mm->mmap_sem held for writing.
    3417                 :            :  * Insert a new vma covering the given region, with the given flags.
    3418                 :            :  * Its pages are supplied by the given array of struct page *.
    3419                 :            :  * The array can be shorter than len >> PAGE_SHIFT if it's null-terminated.
    3420                 :            :  * The region past the last page supplied will always produce SIGBUS.
    3421                 :            :  * The array pointer and the pages it points to are assumed to stay alive
    3422                 :            :  * for as long as this mapping might exist.
    3423                 :            :  */
    3424                 :      21372 : struct vm_area_struct *_install_special_mapping(
    3425                 :            :         struct mm_struct *mm,
    3426                 :            :         unsigned long addr, unsigned long len,
    3427                 :            :         unsigned long vm_flags, const struct vm_special_mapping *spec)
    3428                 :            : {
    3429                 :      21372 :         return __install_special_mapping(mm, addr, len, vm_flags, (void *)spec,
    3430                 :            :                                         &special_mapping_vmops);
    3431                 :            : }
    3432                 :            : 
    3433                 :          0 : int install_special_mapping(struct mm_struct *mm,
    3434                 :            :                             unsigned long addr, unsigned long len,
    3435                 :            :                             unsigned long vm_flags, struct page **pages)
    3436                 :            : {
    3437                 :          0 :         struct vm_area_struct *vma = __install_special_mapping(
    3438                 :            :                 mm, addr, len, vm_flags, (void *)pages,
    3439                 :            :                 &legacy_special_mapping_vmops);
    3440                 :            : 
    3441         [ #  # ]:          0 :         return PTR_ERR_OR_ZERO(vma);
    3442                 :            : }
    3443                 :            : 
    3444                 :            : static DEFINE_MUTEX(mm_all_locks_mutex);
    3445                 :            : 
    3446                 :            : static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
    3447                 :            : {
    3448                 :            :         if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
    3449                 :            :                 /*
    3450                 :            :                  * The LSB of head.next can't change from under us
    3451                 :            :                  * because we hold the mm_all_locks_mutex.
    3452                 :            :                  */
    3453                 :            :                 down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_sem);
    3454                 :            :                 /*
    3455                 :            :                  * We can safely modify head.next after taking the
    3456                 :            :                  * anon_vma->root->rwsem. If some other vma in this mm shares
    3457                 :            :                  * the same anon_vma we won't take it again.
    3458                 :            :                  *
    3459                 :            :                  * No need of atomic instructions here, head.next
    3460                 :            :                  * can't change from under us thanks to the
    3461                 :            :                  * anon_vma->root->rwsem.
    3462                 :            :                  */
    3463                 :            :                 if (__test_and_set_bit(0, (unsigned long *)
    3464                 :            :                                        &anon_vma->root->rb_root.rb_root.rb_node))
    3465                 :            :                         BUG();
    3466                 :            :         }
    3467                 :            : }
    3468                 :            : 
    3469                 :            : static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
    3470                 :            : {
    3471                 :            :         if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
    3472                 :            :                 /*
    3473                 :            :                  * AS_MM_ALL_LOCKS can't change from under us because
    3474                 :            :                  * we hold the mm_all_locks_mutex.
    3475                 :            :                  *
    3476                 :            :                  * Operations on ->flags have to be atomic because
    3477                 :            :                  * even if AS_MM_ALL_LOCKS is stable thanks to the
    3478                 :            :                  * mm_all_locks_mutex, there may be other cpus
    3479                 :            :                  * changing other bitflags in parallel to us.
    3480                 :            :                  */
    3481                 :            :                 if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
    3482                 :            :                         BUG();
    3483                 :            :                 down_write_nest_lock(&mapping->i_mmap_rwsem, &mm->mmap_sem);
    3484                 :            :         }
    3485                 :            : }
    3486                 :            : 
    3487                 :            : /*
    3488                 :            :  * This operation locks against the VM for all pte/vma/mm related
    3489                 :            :  * operations that could ever happen on a certain mm. This includes
    3490                 :            :  * vmtruncate, try_to_unmap, and all page faults.
    3491                 :            :  *
    3492                 :            :  * The caller must take the mmap_sem in write mode before calling
    3493                 :            :  * mm_take_all_locks(). The caller isn't allowed to release the
    3494                 :            :  * mmap_sem until mm_drop_all_locks() returns.
    3495                 :            :  *
    3496                 :            :  * mmap_sem in write mode is required in order to block all operations
    3497                 :            :  * that could modify pagetables and free pages without need of
    3498                 :            :  * altering the vma layout. It's also needed in write mode to avoid new
    3499                 :            :  * anon_vmas to be associated with existing vmas.
    3500                 :            :  *
    3501                 :            :  * A single task can't take more than one mm_take_all_locks() in a row
    3502                 :            :  * or it would deadlock.
    3503                 :            :  *
    3504                 :            :  * The LSB in anon_vma->rb_root.rb_node and the AS_MM_ALL_LOCKS bitflag in
    3505                 :            :  * mapping->flags avoid to take the same lock twice, if more than one
    3506                 :            :  * vma in this mm is backed by the same anon_vma or address_space.
    3507                 :            :  *
    3508                 :            :  * We take locks in following order, accordingly to comment at beginning
    3509                 :            :  * of mm/rmap.c:
    3510                 :            :  *   - all hugetlbfs_i_mmap_rwsem_key locks (aka mapping->i_mmap_rwsem for
    3511                 :            :  *     hugetlb mapping);
    3512                 :            :  *   - all i_mmap_rwsem locks;
    3513                 :            :  *   - all anon_vma->rwseml
    3514                 :            :  *
    3515                 :            :  * We can take all locks within these types randomly because the VM code
    3516                 :            :  * doesn't nest them and we protected from parallel mm_take_all_locks() by
    3517                 :            :  * mm_all_locks_mutex.
    3518                 :            :  *
    3519                 :            :  * mm_take_all_locks() and mm_drop_all_locks are expensive operations
    3520                 :            :  * that may have to take thousand of locks.
    3521                 :            :  *
    3522                 :            :  * mm_take_all_locks() can fail if it's interrupted by signals.
    3523                 :            :  */
    3524                 :          0 : int mm_take_all_locks(struct mm_struct *mm)
    3525                 :            : {
    3526                 :          0 :         struct vm_area_struct *vma;
    3527                 :          0 :         struct anon_vma_chain *avc;
    3528                 :            : 
    3529         [ #  # ]:          0 :         BUG_ON(down_read_trylock(&mm->mmap_sem));
    3530                 :            : 
    3531                 :          0 :         mutex_lock(&mm_all_locks_mutex);
    3532                 :            : 
    3533         [ #  # ]:          0 :         for (vma = mm->mmap; vma; vma = vma->vm_next) {
    3534         [ #  # ]:          0 :                 if (signal_pending(current))
    3535                 :          0 :                         goto out_unlock;
    3536   [ #  #  #  #  :          0 :                 if (vma->vm_file && vma->vm_file->f_mapping &&
                   #  # ]
    3537         [ #  # ]:          0 :                                 is_vm_hugetlb_page(vma))
    3538                 :          0 :                         vm_lock_mapping(mm, vma->vm_file->f_mapping);
    3539                 :            :         }
    3540                 :            : 
    3541         [ #  # ]:          0 :         for (vma = mm->mmap; vma; vma = vma->vm_next) {
    3542         [ #  # ]:          0 :                 if (signal_pending(current))
    3543                 :          0 :                         goto out_unlock;
    3544   [ #  #  #  #  :          0 :                 if (vma->vm_file && vma->vm_file->f_mapping &&
                   #  # ]
    3545         [ #  # ]:          0 :                                 !is_vm_hugetlb_page(vma))
    3546                 :          0 :                         vm_lock_mapping(mm, vma->vm_file->f_mapping);
    3547                 :            :         }
    3548                 :            : 
    3549         [ #  # ]:          0 :         for (vma = mm->mmap; vma; vma = vma->vm_next) {
    3550         [ #  # ]:          0 :                 if (signal_pending(current))
    3551                 :          0 :                         goto out_unlock;
    3552         [ #  # ]:          0 :                 if (vma->anon_vma)
    3553         [ #  # ]:          0 :                         list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
    3554                 :          0 :                                 vm_lock_anon_vma(mm, avc->anon_vma);
    3555                 :            :         }
    3556                 :            : 
    3557                 :            :         return 0;
    3558                 :            : 
    3559                 :          0 : out_unlock:
    3560                 :          0 :         mm_drop_all_locks(mm);
    3561                 :          0 :         return -EINTR;
    3562                 :            : }
    3563                 :            : 
    3564                 :          0 : static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
    3565                 :            : {
    3566         [ #  # ]:          0 :         if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
    3567                 :            :                 /*
    3568                 :            :                  * The LSB of head.next can't change to 0 from under
    3569                 :            :                  * us because we hold the mm_all_locks_mutex.
    3570                 :            :                  *
    3571                 :            :                  * We must however clear the bitflag before unlocking
    3572                 :            :                  * the vma so the users using the anon_vma->rb_root will
    3573                 :            :                  * never see our bitflag.
    3574                 :            :                  *
    3575                 :            :                  * No need of atomic instructions here, head.next
    3576                 :            :                  * can't change from under us until we release the
    3577                 :            :                  * anon_vma->root->rwsem.
    3578                 :            :                  */
    3579         [ #  # ]:          0 :                 if (!__test_and_clear_bit(0, (unsigned long *)
    3580                 :          0 :                                           &anon_vma->root->rb_root.rb_root.rb_node))
    3581                 :          0 :                         BUG();
    3582                 :          0 :                 anon_vma_unlock_write(anon_vma);
    3583                 :            :         }
    3584                 :          0 : }
    3585                 :            : 
    3586                 :          0 : static void vm_unlock_mapping(struct address_space *mapping)
    3587                 :            : {
    3588         [ #  # ]:          0 :         if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
    3589                 :            :                 /*
    3590                 :            :                  * AS_MM_ALL_LOCKS can't change to 0 from under us
    3591                 :            :                  * because we hold the mm_all_locks_mutex.
    3592                 :            :                  */
    3593                 :          0 :                 i_mmap_unlock_write(mapping);
    3594         [ #  # ]:          0 :                 if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
    3595                 :            :                                         &mapping->flags))
    3596                 :          0 :                         BUG();
    3597                 :            :         }
    3598                 :          0 : }
    3599                 :            : 
    3600                 :            : /*
    3601                 :            :  * The mmap_sem cannot be released by the caller until
    3602                 :            :  * mm_drop_all_locks() returns.
    3603                 :            :  */
    3604                 :          0 : void mm_drop_all_locks(struct mm_struct *mm)
    3605                 :            : {
    3606                 :          0 :         struct vm_area_struct *vma;
    3607                 :          0 :         struct anon_vma_chain *avc;
    3608                 :            : 
    3609         [ #  # ]:          0 :         BUG_ON(down_read_trylock(&mm->mmap_sem));
    3610         [ #  # ]:          0 :         BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
    3611                 :            : 
    3612         [ #  # ]:          0 :         for (vma = mm->mmap; vma; vma = vma->vm_next) {
    3613         [ #  # ]:          0 :                 if (vma->anon_vma)
    3614         [ #  # ]:          0 :                         list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
    3615                 :          0 :                                 vm_unlock_anon_vma(avc->anon_vma);
    3616   [ #  #  #  # ]:          0 :                 if (vma->vm_file && vma->vm_file->f_mapping)
    3617                 :          0 :                         vm_unlock_mapping(vma->vm_file->f_mapping);
    3618                 :            :         }
    3619                 :            : 
    3620                 :          0 :         mutex_unlock(&mm_all_locks_mutex);
    3621                 :          0 : }
    3622                 :            : 
    3623                 :            : /*
    3624                 :            :  * initialise the percpu counter for VM
    3625                 :            :  */
    3626                 :         13 : void __init mmap_init(void)
    3627                 :            : {
    3628                 :         13 :         int ret;
    3629                 :            : 
    3630                 :         13 :         ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL);
    3631                 :         13 :         VM_BUG_ON(ret);
    3632                 :         13 : }
    3633                 :            : 
    3634                 :            : /*
    3635                 :            :  * Initialise sysctl_user_reserve_kbytes.
    3636                 :            :  *
    3637                 :            :  * This is intended to prevent a user from starting a single memory hogging
    3638                 :            :  * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER
    3639                 :            :  * mode.
    3640                 :            :  *
    3641                 :            :  * The default value is min(3% of free memory, 128MB)
    3642                 :            :  * 128MB is enough to recover with sshd/login, bash, and top/kill.
    3643                 :            :  */
    3644                 :         13 : static int init_user_reserve(void)
    3645                 :            : {
    3646                 :         13 :         unsigned long free_kbytes;
    3647                 :            : 
    3648                 :         13 :         free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
    3649                 :            : 
    3650                 :         13 :         sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17);
    3651                 :         13 :         return 0;
    3652                 :            : }
    3653                 :            : subsys_initcall(init_user_reserve);
    3654                 :            : 
    3655                 :            : /*
    3656                 :            :  * Initialise sysctl_admin_reserve_kbytes.
    3657                 :            :  *
    3658                 :            :  * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin
    3659                 :            :  * to log in and kill a memory hogging process.
    3660                 :            :  *
    3661                 :            :  * Systems with more than 256MB will reserve 8MB, enough to recover
    3662                 :            :  * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will
    3663                 :            :  * only reserve 3% of free pages by default.
    3664                 :            :  */
    3665                 :         13 : static int init_admin_reserve(void)
    3666                 :            : {
    3667                 :         13 :         unsigned long free_kbytes;
    3668                 :            : 
    3669                 :         13 :         free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
    3670                 :            : 
    3671                 :         13 :         sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13);
    3672                 :         13 :         return 0;
    3673                 :            : }
    3674                 :            : subsys_initcall(init_admin_reserve);
    3675                 :            : 
    3676                 :            : /*
    3677                 :            :  * Reinititalise user and admin reserves if memory is added or removed.
    3678                 :            :  *
    3679                 :            :  * The default user reserve max is 128MB, and the default max for the
    3680                 :            :  * admin reserve is 8MB. These are usually, but not always, enough to
    3681                 :            :  * enable recovery from a memory hogging process using login/sshd, a shell,
    3682                 :            :  * and tools like top. It may make sense to increase or even disable the
    3683                 :            :  * reserve depending on the existence of swap or variations in the recovery
    3684                 :            :  * tools. So, the admin may have changed them.
    3685                 :            :  *
    3686                 :            :  * If memory is added and the reserves have been eliminated or increased above
    3687                 :            :  * the default max, then we'll trust the admin.
    3688                 :            :  *
    3689                 :            :  * If memory is removed and there isn't enough free memory, then we
    3690                 :            :  * need to reset the reserves.
    3691                 :            :  *
    3692                 :            :  * Otherwise keep the reserve set by the admin.
    3693                 :            :  */
    3694                 :            : static int reserve_mem_notifier(struct notifier_block *nb,
    3695                 :            :                              unsigned long action, void *data)
    3696                 :            : {
    3697                 :            :         unsigned long tmp, free_kbytes;
    3698                 :            : 
    3699                 :            :         switch (action) {
    3700                 :            :         case MEM_ONLINE:
    3701                 :            :                 /* Default max is 128MB. Leave alone if modified by operator. */
    3702                 :            :                 tmp = sysctl_user_reserve_kbytes;
    3703                 :            :                 if (0 < tmp && tmp < (1UL << 17))
    3704                 :            :                         init_user_reserve();
    3705                 :            : 
    3706                 :            :                 /* Default max is 8MB.  Leave alone if modified by operator. */
    3707                 :            :                 tmp = sysctl_admin_reserve_kbytes;
    3708                 :            :                 if (0 < tmp && tmp < (1UL << 13))
    3709                 :            :                         init_admin_reserve();
    3710                 :            : 
    3711                 :            :                 break;
    3712                 :            :         case MEM_OFFLINE:
    3713                 :            :                 free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10);
    3714                 :            : 
    3715                 :            :                 if (sysctl_user_reserve_kbytes > free_kbytes) {
    3716                 :            :                         init_user_reserve();
    3717                 :            :                         pr_info("vm.user_reserve_kbytes reset to %lu\n",
    3718                 :            :                                 sysctl_user_reserve_kbytes);
    3719                 :            :                 }
    3720                 :            : 
    3721                 :            :                 if (sysctl_admin_reserve_kbytes > free_kbytes) {
    3722                 :            :                         init_admin_reserve();
    3723                 :            :                         pr_info("vm.admin_reserve_kbytes reset to %lu\n",
    3724                 :            :                                 sysctl_admin_reserve_kbytes);
    3725                 :            :                 }
    3726                 :            :                 break;
    3727                 :            :         default:
    3728                 :            :                 break;
    3729                 :            :         }
    3730                 :            :         return NOTIFY_OK;
    3731                 :            : }
    3732                 :            : 
    3733                 :            : static struct notifier_block reserve_mem_nb = {
    3734                 :            :         .notifier_call = reserve_mem_notifier,
    3735                 :            : };
    3736                 :            : 
    3737                 :         13 : static int __meminit init_reserve_notifier(void)
    3738                 :            : {
    3739                 :         13 :         if (register_hotmemory_notifier(&reserve_mem_nb))
    3740                 :            :                 pr_err("Failed registering memory add/remove notifier for admin reserve\n");
    3741                 :            : 
    3742                 :         13 :         return 0;
    3743                 :            : }
    3744                 :            : subsys_initcall(init_reserve_notifier);

Generated by: LCOV version 1.14