LCOV - code coverage report
Current view: top level - include/linux - mm.h (source / functions) Hit Total Coverage
Test: combined.info Lines: 242 342 70.8 %
Date: 2022-03-28 16:04:14 Functions: 22 26 84.6 %
Branches: 165 450 36.7 %

           Branch data     Line data    Source code
       1                 :            : /* SPDX-License-Identifier: GPL-2.0 */
       2                 :            : #ifndef _LINUX_MM_H
       3                 :            : #define _LINUX_MM_H
       4                 :            : 
       5                 :            : #include <linux/errno.h>
       6                 :            : 
       7                 :            : #ifdef __KERNEL__
       8                 :            : 
       9                 :            : #include <linux/mmdebug.h>
      10                 :            : #include <linux/gfp.h>
      11                 :            : #include <linux/bug.h>
      12                 :            : #include <linux/list.h>
      13                 :            : #include <linux/mmzone.h>
      14                 :            : #include <linux/rbtree.h>
      15                 :            : #include <linux/atomic.h>
      16                 :            : #include <linux/debug_locks.h>
      17                 :            : #include <linux/mm_types.h>
      18                 :            : #include <linux/range.h>
      19                 :            : #include <linux/pfn.h>
      20                 :            : #include <linux/percpu-refcount.h>
      21                 :            : #include <linux/bit_spinlock.h>
      22                 :            : #include <linux/shrinker.h>
      23                 :            : #include <linux/resource.h>
      24                 :            : #include <linux/page_ext.h>
      25                 :            : #include <linux/err.h>
      26                 :            : #include <linux/page_ref.h>
      27                 :            : #include <linux/memremap.h>
      28                 :            : #include <linux/overflow.h>
      29                 :            : #include <linux/sizes.h>
      30                 :            : 
      31                 :            : struct mempolicy;
      32                 :            : struct anon_vma;
      33                 :            : struct anon_vma_chain;
      34                 :            : struct file_ra_state;
      35                 :            : struct user_struct;
      36                 :            : struct writeback_control;
      37                 :            : struct bdi_writeback;
      38                 :            : 
      39                 :            : void init_mm_internals(void);
      40                 :            : 
      41                 :            : #ifndef CONFIG_NEED_MULTIPLE_NODES      /* Don't use mapnrs, do it properly */
      42                 :            : extern unsigned long max_mapnr;
      43                 :            : 
      44                 :            : static inline void set_max_mapnr(unsigned long limit)
      45                 :            : {
      46                 :            :         max_mapnr = limit;
      47                 :            : }
      48                 :            : #else
      49                 :            : static inline void set_max_mapnr(unsigned long limit) { }
      50                 :            : #endif
      51                 :            : 
      52                 :            : extern atomic_long_t _totalram_pages;
      53                 :     578328 : static inline unsigned long totalram_pages(void)
      54                 :            : {
      55         [ +  - ]:     578328 :         return (unsigned long)atomic_long_read(&_totalram_pages);
      56                 :            : }
      57                 :            : 
      58                 :          0 : static inline void totalram_pages_inc(void)
      59                 :            : {
      60                 :          0 :         atomic_long_inc(&_totalram_pages);
      61                 :            : }
      62                 :            : 
      63                 :            : static inline void totalram_pages_dec(void)
      64                 :            : {
      65                 :            :         atomic_long_dec(&_totalram_pages);
      66                 :            : }
      67                 :            : 
      68                 :      18382 : static inline void totalram_pages_add(long count)
      69                 :            : {
      70                 :      18382 :         atomic_long_add(count, &_totalram_pages);
      71                 :      18382 : }
      72                 :            : 
      73                 :            : extern void * high_memory;
      74                 :            : extern int page_cluster;
      75                 :            : 
      76                 :            : #ifdef CONFIG_SYSCTL
      77                 :            : extern int sysctl_legacy_va_layout;
      78                 :            : #else
      79                 :            : #define sysctl_legacy_va_layout 0
      80                 :            : #endif
      81                 :            : 
      82                 :            : #ifdef CONFIG_HAVE_ARCH_MMAP_RND_BITS
      83                 :            : extern const int mmap_rnd_bits_min;
      84                 :            : extern const int mmap_rnd_bits_max;
      85                 :            : extern int mmap_rnd_bits __read_mostly;
      86                 :            : #endif
      87                 :            : #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
      88                 :            : extern const int mmap_rnd_compat_bits_min;
      89                 :            : extern const int mmap_rnd_compat_bits_max;
      90                 :            : extern int mmap_rnd_compat_bits __read_mostly;
      91                 :            : #endif
      92                 :            : 
      93                 :            : #include <asm/page.h>
      94                 :            : #include <asm/pgtable.h>
      95                 :            : #include <asm/processor.h>
      96                 :            : 
      97                 :            : /*
      98                 :            :  * Architectures that support memory tagging (assigning tags to memory regions,
      99                 :            :  * embedding these tags into addresses that point to these memory regions, and
     100                 :            :  * checking that the memory and the pointer tags match on memory accesses)
     101                 :            :  * redefine this macro to strip tags from pointers.
     102                 :            :  * It's defined as noop for arcitectures that don't support memory tagging.
     103                 :            :  */
     104                 :            : #ifndef untagged_addr
     105                 :            : #define untagged_addr(addr) (addr)
     106                 :            : #endif
     107                 :            : 
     108                 :            : #ifndef __pa_symbol
     109                 :            : #define __pa_symbol(x)  __pa(RELOC_HIDE((unsigned long)(x), 0))
     110                 :            : #endif
     111                 :            : 
     112                 :            : #ifndef page_to_virt
     113                 :            : #define page_to_virt(x) __va(PFN_PHYS(page_to_pfn(x)))
     114                 :            : #endif
     115                 :            : 
     116                 :            : #ifndef lm_alias
     117                 :            : #define lm_alias(x)     __va(__pa_symbol(x))
     118                 :            : #endif
     119                 :            : 
     120                 :            : /*
     121                 :            :  * To prevent common memory management code establishing
     122                 :            :  * a zero page mapping on a read fault.
     123                 :            :  * This macro should be defined within <asm/pgtable.h>.
     124                 :            :  * s390 does this to prevent multiplexing of hardware bits
     125                 :            :  * related to the physical page in case of virtualization.
     126                 :            :  */
     127                 :            : #ifndef mm_forbids_zeropage
     128                 :            : #define mm_forbids_zeropage(X)  (0)
     129                 :            : #endif
     130                 :            : 
     131                 :            : /*
     132                 :            :  * On some architectures it is expensive to call memset() for small sizes.
     133                 :            :  * If an architecture decides to implement their own version of
     134                 :            :  * mm_zero_struct_page they should wrap the defines below in a #ifndef and
     135                 :            :  * define their own version of this macro in <asm/pgtable.h>
     136                 :            :  */
     137                 :            : #if BITS_PER_LONG == 64
     138                 :            : /* This function must be updated when the size of struct page grows above 80
     139                 :            :  * or reduces below 56. The idea that compiler optimizes out switch()
     140                 :            :  * statement, and only leaves move/store instructions. Also the compiler can
     141                 :            :  * combine write statments if they are both assignments and can be reordered,
     142                 :            :  * this can result in several of the writes here being dropped.
     143                 :            :  */
     144                 :            : #define mm_zero_struct_page(pp) __mm_zero_struct_page(pp)
     145                 :            : static inline void __mm_zero_struct_page(struct page *page)
     146                 :            : {
     147                 :            :         unsigned long *_pp = (void *)page;
     148                 :            : 
     149                 :            :          /* Check that struct page is either 56, 64, 72, or 80 bytes */
     150                 :            :         BUILD_BUG_ON(sizeof(struct page) & 7);
     151                 :            :         BUILD_BUG_ON(sizeof(struct page) < 56);
     152                 :            :         BUILD_BUG_ON(sizeof(struct page) > 80);
     153                 :            : 
     154                 :            :         switch (sizeof(struct page)) {
     155                 :            :         case 80:
     156                 :            :                 _pp[9] = 0;     /* fallthrough */
     157                 :            :         case 72:
     158                 :            :                 _pp[8] = 0;     /* fallthrough */
     159                 :            :         case 64:
     160                 :            :                 _pp[7] = 0;     /* fallthrough */
     161                 :            :         case 56:
     162                 :            :                 _pp[6] = 0;
     163                 :            :                 _pp[5] = 0;
     164                 :            :                 _pp[4] = 0;
     165                 :            :                 _pp[3] = 0;
     166                 :            :                 _pp[2] = 0;
     167                 :            :                 _pp[1] = 0;
     168                 :            :                 _pp[0] = 0;
     169                 :            :         }
     170                 :            : }
     171                 :            : #else
     172                 :            : #define mm_zero_struct_page(pp)  ((void)memset((pp), 0, sizeof(struct page)))
     173                 :            : #endif
     174                 :            : 
     175                 :            : /*
     176                 :            :  * Default maximum number of active map areas, this limits the number of vmas
     177                 :            :  * per mm struct. Users can overwrite this number by sysctl but there is a
     178                 :            :  * problem.
     179                 :            :  *
     180                 :            :  * When a program's coredump is generated as ELF format, a section is created
     181                 :            :  * per a vma. In ELF, the number of sections is represented in unsigned short.
     182                 :            :  * This means the number of sections should be smaller than 65535 at coredump.
     183                 :            :  * Because the kernel adds some informative sections to a image of program at
     184                 :            :  * generating coredump, we need some margin. The number of extra sections is
     185                 :            :  * 1-3 now and depends on arch. We use "5" as safe margin, here.
     186                 :            :  *
     187                 :            :  * ELF extended numbering allows more than 65535 sections, so 16-bit bound is
     188                 :            :  * not a hard limit any more. Although some userspace tools can be surprised by
     189                 :            :  * that.
     190                 :            :  */
     191                 :            : #define MAPCOUNT_ELF_CORE_MARGIN        (5)
     192                 :            : #define DEFAULT_MAX_MAP_COUNT   (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN)
     193                 :            : 
     194                 :            : extern int sysctl_max_map_count;
     195                 :            : 
     196                 :            : extern unsigned long sysctl_user_reserve_kbytes;
     197                 :            : extern unsigned long sysctl_admin_reserve_kbytes;
     198                 :            : 
     199                 :            : extern int sysctl_overcommit_memory;
     200                 :            : extern int sysctl_overcommit_ratio;
     201                 :            : extern unsigned long sysctl_overcommit_kbytes;
     202                 :            : 
     203                 :            : extern int overcommit_ratio_handler(struct ctl_table *, int, void __user *,
     204                 :            :                                     size_t *, loff_t *);
     205                 :            : extern int overcommit_kbytes_handler(struct ctl_table *, int, void __user *,
     206                 :            :                                     size_t *, loff_t *);
     207                 :            : 
     208                 :            : #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
     209                 :            : 
     210                 :            : /* to align the pointer to the (next) page boundary */
     211                 :            : #define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
     212                 :            : 
     213                 :            : /* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */
     214                 :            : #define PAGE_ALIGNED(addr)      IS_ALIGNED((unsigned long)(addr), PAGE_SIZE)
     215                 :            : 
     216                 :            : #define lru_to_page(head) (list_entry((head)->prev, struct page, lru))
     217                 :            : 
     218                 :            : /*
     219                 :            :  * Linux kernel virtual memory manager primitives.
     220                 :            :  * The idea being to have a "virtual" mm in the same way
     221                 :            :  * we have a virtual fs - giving a cleaner interface to the
     222                 :            :  * mm details, and allowing different kinds of memory mappings
     223                 :            :  * (from shared memory to executable loading to arbitrary
     224                 :            :  * mmap() functions).
     225                 :            :  */
     226                 :            : 
     227                 :            : struct vm_area_struct *vm_area_alloc(struct mm_struct *);
     228                 :            : struct vm_area_struct *vm_area_dup(struct vm_area_struct *);
     229                 :            : void vm_area_free(struct vm_area_struct *);
     230                 :            : 
     231                 :            : #ifndef CONFIG_MMU
     232                 :            : extern struct rb_root nommu_region_tree;
     233                 :            : extern struct rw_semaphore nommu_region_sem;
     234                 :            : 
     235                 :            : extern unsigned int kobjsize(const void *objp);
     236                 :            : #endif
     237                 :            : 
     238                 :            : /*
     239                 :            :  * vm_flags in vm_area_struct, see mm_types.h.
     240                 :            :  * When changing, update also include/trace/events/mmflags.h
     241                 :            :  */
     242                 :            : #define VM_NONE         0x00000000
     243                 :            : 
     244                 :            : #define VM_READ         0x00000001      /* currently active flags */
     245                 :            : #define VM_WRITE        0x00000002
     246                 :            : #define VM_EXEC         0x00000004
     247                 :            : #define VM_SHARED       0x00000008
     248                 :            : 
     249                 :            : /* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */
     250                 :            : #define VM_MAYREAD      0x00000010      /* limits for mprotect() etc */
     251                 :            : #define VM_MAYWRITE     0x00000020
     252                 :            : #define VM_MAYEXEC      0x00000040
     253                 :            : #define VM_MAYSHARE     0x00000080
     254                 :            : 
     255                 :            : #define VM_GROWSDOWN    0x00000100      /* general info on the segment */
     256                 :            : #define VM_UFFD_MISSING 0x00000200      /* missing pages tracking */
     257                 :            : #define VM_PFNMAP       0x00000400      /* Page-ranges managed without "struct page", just pure PFN */
     258                 :            : #define VM_DENYWRITE    0x00000800      /* ETXTBSY on write attempts.. */
     259                 :            : #define VM_UFFD_WP      0x00001000      /* wrprotect pages tracking */
     260                 :            : 
     261                 :            : #define VM_LOCKED       0x00002000
     262                 :            : #define VM_IO           0x00004000      /* Memory mapped I/O or similar */
     263                 :            : 
     264                 :            :                                         /* Used by sys_madvise() */
     265                 :            : #define VM_SEQ_READ     0x00008000      /* App will access data sequentially */
     266                 :            : #define VM_RAND_READ    0x00010000      /* App will not benefit from clustered reads */
     267                 :            : 
     268                 :            : #define VM_DONTCOPY     0x00020000      /* Do not copy this vma on fork */
     269                 :            : #define VM_DONTEXPAND   0x00040000      /* Cannot expand with mremap() */
     270                 :            : #define VM_LOCKONFAULT  0x00080000      /* Lock the pages covered when they are faulted in */
     271                 :            : #define VM_ACCOUNT      0x00100000      /* Is a VM accounted object */
     272                 :            : #define VM_NORESERVE    0x00200000      /* should the VM suppress accounting */
     273                 :            : #define VM_HUGETLB      0x00400000      /* Huge TLB Page VM */
     274                 :            : #define VM_SYNC         0x00800000      /* Synchronous page faults */
     275                 :            : #define VM_ARCH_1       0x01000000      /* Architecture-specific flag */
     276                 :            : #define VM_WIPEONFORK   0x02000000      /* Wipe VMA contents in child. */
     277                 :            : #define VM_DONTDUMP     0x04000000      /* Do not include in the core dump */
     278                 :            : 
     279                 :            : #ifdef CONFIG_MEM_SOFT_DIRTY
     280                 :            : # define VM_SOFTDIRTY   0x08000000      /* Not soft dirty clean area */
     281                 :            : #else
     282                 :            : # define VM_SOFTDIRTY   0
     283                 :            : #endif
     284                 :            : 
     285                 :            : #define VM_MIXEDMAP     0x10000000      /* Can contain "struct page" and pure PFN pages */
     286                 :            : #define VM_HUGEPAGE     0x20000000      /* MADV_HUGEPAGE marked this vma */
     287                 :            : #define VM_NOHUGEPAGE   0x40000000      /* MADV_NOHUGEPAGE marked this vma */
     288                 :            : #define VM_MERGEABLE    0x80000000      /* KSM may merge identical pages */
     289                 :            : 
     290                 :            : #ifdef CONFIG_ARCH_USES_HIGH_VMA_FLAGS
     291                 :            : #define VM_HIGH_ARCH_BIT_0      32      /* bit only usable on 64-bit architectures */
     292                 :            : #define VM_HIGH_ARCH_BIT_1      33      /* bit only usable on 64-bit architectures */
     293                 :            : #define VM_HIGH_ARCH_BIT_2      34      /* bit only usable on 64-bit architectures */
     294                 :            : #define VM_HIGH_ARCH_BIT_3      35      /* bit only usable on 64-bit architectures */
     295                 :            : #define VM_HIGH_ARCH_BIT_4      36      /* bit only usable on 64-bit architectures */
     296                 :            : #define VM_HIGH_ARCH_0  BIT(VM_HIGH_ARCH_BIT_0)
     297                 :            : #define VM_HIGH_ARCH_1  BIT(VM_HIGH_ARCH_BIT_1)
     298                 :            : #define VM_HIGH_ARCH_2  BIT(VM_HIGH_ARCH_BIT_2)
     299                 :            : #define VM_HIGH_ARCH_3  BIT(VM_HIGH_ARCH_BIT_3)
     300                 :            : #define VM_HIGH_ARCH_4  BIT(VM_HIGH_ARCH_BIT_4)
     301                 :            : #endif /* CONFIG_ARCH_USES_HIGH_VMA_FLAGS */
     302                 :            : 
     303                 :            : #ifdef CONFIG_ARCH_HAS_PKEYS
     304                 :            : # define VM_PKEY_SHIFT  VM_HIGH_ARCH_BIT_0
     305                 :            : # define VM_PKEY_BIT0   VM_HIGH_ARCH_0  /* A protection key is a 4-bit value */
     306                 :            : # define VM_PKEY_BIT1   VM_HIGH_ARCH_1  /* on x86 and 5-bit value on ppc64   */
     307                 :            : # define VM_PKEY_BIT2   VM_HIGH_ARCH_2
     308                 :            : # define VM_PKEY_BIT3   VM_HIGH_ARCH_3
     309                 :            : #ifdef CONFIG_PPC
     310                 :            : # define VM_PKEY_BIT4  VM_HIGH_ARCH_4
     311                 :            : #else
     312                 :            : # define VM_PKEY_BIT4  0
     313                 :            : #endif
     314                 :            : #endif /* CONFIG_ARCH_HAS_PKEYS */
     315                 :            : 
     316                 :            : #if defined(CONFIG_X86)
     317                 :            : # define VM_PAT         VM_ARCH_1       /* PAT reserves whole VMA at once (x86) */
     318                 :            : #elif defined(CONFIG_PPC)
     319                 :            : # define VM_SAO         VM_ARCH_1       /* Strong Access Ordering (powerpc) */
     320                 :            : #elif defined(CONFIG_PARISC)
     321                 :            : # define VM_GROWSUP     VM_ARCH_1
     322                 :            : #elif defined(CONFIG_IA64)
     323                 :            : # define VM_GROWSUP     VM_ARCH_1
     324                 :            : #elif defined(CONFIG_SPARC64)
     325                 :            : # define VM_SPARC_ADI   VM_ARCH_1       /* Uses ADI tag for access control */
     326                 :            : # define VM_ARCH_CLEAR  VM_SPARC_ADI
     327                 :            : #elif !defined(CONFIG_MMU)
     328                 :            : # define VM_MAPPED_COPY VM_ARCH_1       /* T if mapped copy of data (nommu mmap) */
     329                 :            : #endif
     330                 :            : 
     331                 :            : #if defined(CONFIG_X86_INTEL_MPX)
     332                 :            : /* MPX specific bounds table or bounds directory */
     333                 :            : # define VM_MPX         VM_HIGH_ARCH_4
     334                 :            : #else
     335                 :            : # define VM_MPX         VM_NONE
     336                 :            : #endif
     337                 :            : 
     338                 :            : #ifndef VM_GROWSUP
     339                 :            : # define VM_GROWSUP     VM_NONE
     340                 :            : #endif
     341                 :            : 
     342                 :            : /* Bits set in the VMA until the stack is in its final location */
     343                 :            : #define VM_STACK_INCOMPLETE_SETUP       (VM_RAND_READ | VM_SEQ_READ)
     344                 :            : 
     345                 :            : #ifndef VM_STACK_DEFAULT_FLAGS          /* arch can override this */
     346                 :            : #define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
     347                 :            : #endif
     348                 :            : 
     349                 :            : #ifdef CONFIG_STACK_GROWSUP
     350                 :            : #define VM_STACK        VM_GROWSUP
     351                 :            : #else
     352                 :            : #define VM_STACK        VM_GROWSDOWN
     353                 :            : #endif
     354                 :            : 
     355                 :            : #define VM_STACK_FLAGS  (VM_STACK | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
     356                 :            : 
     357                 :            : /*
     358                 :            :  * Special vmas that are non-mergable, non-mlock()able.
     359                 :            :  * Note: mm/huge_memory.c VM_NO_THP depends on this definition.
     360                 :            :  */
     361                 :            : #define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP)
     362                 :            : 
     363                 :            : /* This mask defines which mm->def_flags a process can inherit its parent */
     364                 :            : #define VM_INIT_DEF_MASK        VM_NOHUGEPAGE
     365                 :            : 
     366                 :            : /* This mask is used to clear all the VMA flags used by mlock */
     367                 :            : #define VM_LOCKED_CLEAR_MASK    (~(VM_LOCKED | VM_LOCKONFAULT))
     368                 :            : 
     369                 :            : /* Arch-specific flags to clear when updating VM flags on protection change */
     370                 :            : #ifndef VM_ARCH_CLEAR
     371                 :            : # define VM_ARCH_CLEAR  VM_NONE
     372                 :            : #endif
     373                 :            : #define VM_FLAGS_CLEAR  (ARCH_VM_PKEY_FLAGS | VM_ARCH_CLEAR)
     374                 :            : 
     375                 :            : /*
     376                 :            :  * mapping from the currently active vm_flags protection bits (the
     377                 :            :  * low four bits) to a page protection mask..
     378                 :            :  */
     379                 :            : extern pgprot_t protection_map[16];
     380                 :            : 
     381                 :            : #define FAULT_FLAG_WRITE        0x01    /* Fault was a write access */
     382                 :            : #define FAULT_FLAG_MKWRITE      0x02    /* Fault was mkwrite of existing pte */
     383                 :            : #define FAULT_FLAG_ALLOW_RETRY  0x04    /* Retry fault if blocking */
     384                 :            : #define FAULT_FLAG_RETRY_NOWAIT 0x08    /* Don't drop mmap_sem and wait when retrying */
     385                 :            : #define FAULT_FLAG_KILLABLE     0x10    /* The fault task is in SIGKILL killable region */
     386                 :            : #define FAULT_FLAG_TRIED        0x20    /* Second try */
     387                 :            : #define FAULT_FLAG_USER         0x40    /* The fault originated in userspace */
     388                 :            : #define FAULT_FLAG_REMOTE       0x80    /* faulting for non current tsk/mm */
     389                 :            : #define FAULT_FLAG_INSTRUCTION  0x100   /* The fault was during an instruction fetch */
     390                 :            : 
     391                 :            : #define FAULT_FLAG_TRACE \
     392                 :            :         { FAULT_FLAG_WRITE,             "WRITE" }, \
     393                 :            :         { FAULT_FLAG_MKWRITE,           "MKWRITE" }, \
     394                 :            :         { FAULT_FLAG_ALLOW_RETRY,       "ALLOW_RETRY" }, \
     395                 :            :         { FAULT_FLAG_RETRY_NOWAIT,      "RETRY_NOWAIT" }, \
     396                 :            :         { FAULT_FLAG_KILLABLE,          "KILLABLE" }, \
     397                 :            :         { FAULT_FLAG_TRIED,             "TRIED" }, \
     398                 :            :         { FAULT_FLAG_USER,              "USER" }, \
     399                 :            :         { FAULT_FLAG_REMOTE,            "REMOTE" }, \
     400                 :            :         { FAULT_FLAG_INSTRUCTION,       "INSTRUCTION" }
     401                 :            : 
     402                 :            : /*
     403                 :            :  * vm_fault is filled by the the pagefault handler and passed to the vma's
     404                 :            :  * ->fault function. The vma's ->fault is responsible for returning a bitmask
     405                 :            :  * of VM_FAULT_xxx flags that give details about how the fault was handled.
     406                 :            :  *
     407                 :            :  * MM layer fills up gfp_mask for page allocations but fault handler might
     408                 :            :  * alter it if its implementation requires a different allocation context.
     409                 :            :  *
     410                 :            :  * pgoff should be used in favour of virtual_address, if possible.
     411                 :            :  */
     412                 :            : struct vm_fault {
     413                 :            :         struct vm_area_struct *vma;     /* Target VMA */
     414                 :            :         unsigned int flags;             /* FAULT_FLAG_xxx flags */
     415                 :            :         gfp_t gfp_mask;                 /* gfp mask to be used for allocations */
     416                 :            :         pgoff_t pgoff;                  /* Logical page offset based on vma */
     417                 :            :         unsigned long address;          /* Faulting virtual address */
     418                 :            :         pmd_t *pmd;                     /* Pointer to pmd entry matching
     419                 :            :                                          * the 'address' */
     420                 :            :         pud_t *pud;                     /* Pointer to pud entry matching
     421                 :            :                                          * the 'address'
     422                 :            :                                          */
     423                 :            :         pte_t orig_pte;                 /* Value of PTE at the time of fault */
     424                 :            : 
     425                 :            :         struct page *cow_page;          /* Page handler may use for COW fault */
     426                 :            :         struct mem_cgroup *memcg;       /* Cgroup cow_page belongs to */
     427                 :            :         struct page *page;              /* ->fault handlers should return a
     428                 :            :                                          * page here, unless VM_FAULT_NOPAGE
     429                 :            :                                          * is set (which is also implied by
     430                 :            :                                          * VM_FAULT_ERROR).
     431                 :            :                                          */
     432                 :            :         /* These three entries are valid only while holding ptl lock */
     433                 :            :         pte_t *pte;                     /* Pointer to pte entry matching
     434                 :            :                                          * the 'address'. NULL if the page
     435                 :            :                                          * table hasn't been allocated.
     436                 :            :                                          */
     437                 :            :         spinlock_t *ptl;                /* Page table lock.
     438                 :            :                                          * Protects pte page table if 'pte'
     439                 :            :                                          * is not NULL, otherwise pmd.
     440                 :            :                                          */
     441                 :            :         pgtable_t prealloc_pte;         /* Pre-allocated pte page table.
     442                 :            :                                          * vm_ops->map_pages() calls
     443                 :            :                                          * alloc_set_pte() from atomic context.
     444                 :            :                                          * do_fault_around() pre-allocates
     445                 :            :                                          * page table to avoid allocation from
     446                 :            :                                          * atomic context.
     447                 :            :                                          */
     448                 :            : };
     449                 :            : 
     450                 :            : /* page entry size for vm->huge_fault() */
     451                 :            : enum page_entry_size {
     452                 :            :         PE_SIZE_PTE = 0,
     453                 :            :         PE_SIZE_PMD,
     454                 :            :         PE_SIZE_PUD,
     455                 :            : };
     456                 :            : 
     457                 :            : /*
     458                 :            :  * These are the virtual MM functions - opening of an area, closing and
     459                 :            :  * unmapping it (needed to keep files on disk up-to-date etc), pointer
     460                 :            :  * to the functions called when a no-page or a wp-page exception occurs.
     461                 :            :  */
     462                 :            : struct vm_operations_struct {
     463                 :            :         void (*open)(struct vm_area_struct * area);
     464                 :            :         void (*close)(struct vm_area_struct * area);
     465                 :            :         int (*split)(struct vm_area_struct * area, unsigned long addr);
     466                 :            :         int (*mremap)(struct vm_area_struct * area);
     467                 :            :         vm_fault_t (*fault)(struct vm_fault *vmf);
     468                 :            :         vm_fault_t (*huge_fault)(struct vm_fault *vmf,
     469                 :            :                         enum page_entry_size pe_size);
     470                 :            :         void (*map_pages)(struct vm_fault *vmf,
     471                 :            :                         pgoff_t start_pgoff, pgoff_t end_pgoff);
     472                 :            :         unsigned long (*pagesize)(struct vm_area_struct * area);
     473                 :            : 
     474                 :            :         /* notification that a previously read-only page is about to become
     475                 :            :          * writable, if an error is returned it will cause a SIGBUS */
     476                 :            :         vm_fault_t (*page_mkwrite)(struct vm_fault *vmf);
     477                 :            : 
     478                 :            :         /* same as page_mkwrite when using VM_PFNMAP|VM_MIXEDMAP */
     479                 :            :         vm_fault_t (*pfn_mkwrite)(struct vm_fault *vmf);
     480                 :            : 
     481                 :            :         /* called by access_process_vm when get_user_pages() fails, typically
     482                 :            :          * for use by special VMAs that can switch between memory and hardware
     483                 :            :          */
     484                 :            :         int (*access)(struct vm_area_struct *vma, unsigned long addr,
     485                 :            :                       void *buf, int len, int write);
     486                 :            : 
     487                 :            :         /* Called by the /proc/PID/maps code to ask the vma whether it
     488                 :            :          * has a special name.  Returning non-NULL will also cause this
     489                 :            :          * vma to be dumped unconditionally. */
     490                 :            :         const char *(*name)(struct vm_area_struct *vma);
     491                 :            : 
     492                 :            : #ifdef CONFIG_NUMA
     493                 :            :         /*
     494                 :            :          * set_policy() op must add a reference to any non-NULL @new mempolicy
     495                 :            :          * to hold the policy upon return.  Caller should pass NULL @new to
     496                 :            :          * remove a policy and fall back to surrounding context--i.e. do not
     497                 :            :          * install a MPOL_DEFAULT policy, nor the task or system default
     498                 :            :          * mempolicy.
     499                 :            :          */
     500                 :            :         int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
     501                 :            : 
     502                 :            :         /*
     503                 :            :          * get_policy() op must add reference [mpol_get()] to any policy at
     504                 :            :          * (vma,addr) marked as MPOL_SHARED.  The shared policy infrastructure
     505                 :            :          * in mm/mempolicy.c will do this automatically.
     506                 :            :          * get_policy() must NOT add a ref if the policy at (vma,addr) is not
     507                 :            :          * marked as MPOL_SHARED. vma policies are protected by the mmap_sem.
     508                 :            :          * If no [shared/vma] mempolicy exists at the addr, get_policy() op
     509                 :            :          * must return NULL--i.e., do not "fallback" to task or system default
     510                 :            :          * policy.
     511                 :            :          */
     512                 :            :         struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
     513                 :            :                                         unsigned long addr);
     514                 :            : #endif
     515                 :            :         /*
     516                 :            :          * Called by vm_normal_page() for special PTEs to find the
     517                 :            :          * page for @addr.  This is useful if the default behavior
     518                 :            :          * (using pte_page()) would not find the correct page.
     519                 :            :          */
     520                 :            :         struct page *(*find_special_page)(struct vm_area_struct *vma,
     521                 :            :                                           unsigned long addr);
     522                 :            : };
     523                 :            : 
     524                 :     245348 : static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm)
     525                 :            : {
     526                 :     245348 :         static const struct vm_operations_struct dummy_vm_ops = {};
     527                 :            : 
     528                 :     245348 :         memset(vma, 0, sizeof(*vma));
     529                 :     245348 :         vma->vm_mm = mm;
     530                 :     245348 :         vma->vm_ops = &dummy_vm_ops;
     531                 :     245348 :         INIT_LIST_HEAD(&vma->anon_vma_chain);
     532                 :     228253 : }
     533                 :            : 
     534                 :      66481 : static inline void vma_set_anonymous(struct vm_area_struct *vma)
     535                 :            : {
     536         [ +  - ]:      36722 :         vma->vm_ops = NULL;
     537                 :      29759 : }
     538                 :            : 
     539                 :    1013844 : static inline bool vma_is_anonymous(struct vm_area_struct *vma)
     540                 :            : {
     541   [ +  +  +  + ]:    1013844 :         return !vma->vm_ops;
     542                 :            : }
     543                 :            : 
     544                 :            : #ifdef CONFIG_SHMEM
     545                 :            : /*
     546                 :            :  * The vma_is_shmem is not inline because it is used only by slow
     547                 :            :  * paths in userfault.
     548                 :            :  */
     549                 :            : bool vma_is_shmem(struct vm_area_struct *vma);
     550                 :            : #else
     551                 :            : static inline bool vma_is_shmem(struct vm_area_struct *vma) { return false; }
     552                 :            : #endif
     553                 :            : 
     554                 :            : int vma_is_stack_for_current(struct vm_area_struct *vma);
     555                 :            : 
     556                 :            : /* flush_tlb_range() takes a vma, not a mm, and can care about flags */
     557                 :            : #define TLB_FLUSH_VMA(mm,flags) { .vm_mm = (mm), .vm_flags = (flags) }
     558                 :            : 
     559                 :            : struct mmu_gather;
     560                 :            : struct inode;
     561                 :            : 
     562                 :            : /*
     563                 :            :  * FIXME: take this include out, include page-flags.h in
     564                 :            :  * files which need it (119 of them)
     565                 :            :  */
     566                 :            : #include <linux/page-flags.h>
     567                 :            : #include <linux/huge_mm.h>
     568                 :            : 
     569                 :            : /*
     570                 :            :  * Methods to modify the page usage count.
     571                 :            :  *
     572                 :            :  * What counts for a page usage:
     573                 :            :  * - cache mapping   (page->mapping)
     574                 :            :  * - private data    (page->private)
     575                 :            :  * - page mapped in a task's page tables, each mapping
     576                 :            :  *   is counted separately
     577                 :            :  *
     578                 :            :  * Also, many kernel routines increase the page count before a critical
     579                 :            :  * routine so they can be sure the page doesn't go away from under them.
     580                 :            :  */
     581                 :            : 
     582                 :            : /*
     583                 :            :  * Drop a ref, return true if the refcount fell to zero (the page has no users)
     584                 :            :  */
     585                 :   10605543 : static inline int put_page_testzero(struct page *page)
     586                 :            : {
     587                 :   10605543 :         VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
     588                 :   10605543 :         return page_ref_dec_and_test(page);
     589                 :            : }
     590                 :            : 
     591                 :            : /*
     592                 :            :  * Try to grab a ref unless the page has a refcount of zero, return false if
     593                 :            :  * that is the case.
     594                 :            :  * This can be called when MMU is off so it must not access
     595                 :            :  * any of the virtual mappings.
     596                 :            :  */
     597                 :          0 : static inline int get_page_unless_zero(struct page *page)
     598                 :            : {
     599                 :          0 :         return page_ref_add_unless(page, 1, 0);
     600                 :            : }
     601                 :            : 
     602                 :            : extern int page_is_ram(unsigned long pfn);
     603                 :            : 
     604                 :            : enum {
     605                 :            :         REGION_INTERSECTS,
     606                 :            :         REGION_DISJOINT,
     607                 :            :         REGION_MIXED,
     608                 :            : };
     609                 :            : 
     610                 :            : int region_intersects(resource_size_t offset, size_t size, unsigned long flags,
     611                 :            :                       unsigned long desc);
     612                 :            : 
     613                 :            : /* Support for virtually mapped pages */
     614                 :            : struct page *vmalloc_to_page(const void *addr);
     615                 :            : unsigned long vmalloc_to_pfn(const void *addr);
     616                 :            : 
     617                 :            : /*
     618                 :            :  * Determine if an address is within the vmalloc range
     619                 :            :  *
     620                 :            :  * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there
     621                 :            :  * is no special casing required.
     622                 :            :  */
     623                 :            : 
     624                 :            : #ifndef is_ioremap_addr
     625                 :            : #define is_ioremap_addr(x) is_vmalloc_addr(x)
     626                 :            : #endif
     627                 :            : 
     628                 :            : #ifdef CONFIG_MMU
     629                 :            : extern bool is_vmalloc_addr(const void *x);
     630                 :            : extern int is_vmalloc_or_module_addr(const void *x);
     631                 :            : #else
     632                 :            : static inline bool is_vmalloc_addr(const void *x)
     633                 :            : {
     634                 :            :         return false;
     635                 :            : }
     636                 :            : static inline int is_vmalloc_or_module_addr(const void *x)
     637                 :            : {
     638                 :            :         return 0;
     639                 :            : }
     640                 :            : #endif
     641                 :            : 
     642                 :            : extern void *kvmalloc_node(size_t size, gfp_t flags, int node);
     643                 :      25851 : static inline void *kvmalloc(size_t size, gfp_t flags)
     644                 :            : {
     645                 :      25526 :         return kvmalloc_node(size, flags, NUMA_NO_NODE);
     646                 :            : }
     647                 :          0 : static inline void *kvzalloc_node(size_t size, gfp_t flags, int node)
     648                 :            : {
     649                 :          0 :         return kvmalloc_node(size, flags | __GFP_ZERO, node);
     650                 :            : }
     651                 :       1159 : static inline void *kvzalloc(size_t size, gfp_t flags)
     652                 :            : {
     653                 :       1159 :         return kvmalloc(size, flags | __GFP_ZERO);
     654                 :            : }
     655                 :            : 
     656                 :        417 : static inline void *kvmalloc_array(size_t n, size_t size, gfp_t flags)
     657                 :            : {
     658                 :        417 :         size_t bytes;
     659                 :            : 
     660   [ +  -  -  -  :        417 :         if (unlikely(check_mul_overflow(n, size, &bytes)))
                   #  # ]
     661                 :            :                 return NULL;
     662                 :            : 
     663                 :        417 :         return kvmalloc(bytes, flags);
     664                 :            : }
     665                 :            : 
     666                 :        104 : static inline void *kvcalloc(size_t n, size_t size, gfp_t flags)
     667                 :            : {
     668                 :        104 :         return kvmalloc_array(n, size, flags | __GFP_ZERO);
     669                 :            : }
     670                 :            : 
     671                 :            : extern void kvfree(const void *addr);
     672                 :            : 
     673                 :          0 : static inline int compound_mapcount(struct page *page)
     674                 :            : {
     675                 :          0 :         VM_BUG_ON_PAGE(!PageCompound(page), page);
     676         [ #  # ]:          0 :         page = compound_head(page);
     677                 :          0 :         return atomic_read(compound_mapcount_ptr(page)) + 1;
     678                 :            : }
     679                 :            : 
     680                 :            : /*
     681                 :            :  * The atomic page->_mapcount, starts from -1: so that transitions
     682                 :            :  * both from it and to it can be tracked, using atomic_inc_and_test
     683                 :            :  * and atomic_add_negative(-1).
     684                 :            :  */
     685                 :          0 : static inline void page_mapcount_reset(struct page *page)
     686                 :            : {
     687                 :          0 :         atomic_set(&(page)->_mapcount, -1);
     688                 :            : }
     689                 :            : 
     690                 :            : int __page_mapcount(struct page *page);
     691                 :            : 
     692                 :    8111459 : static inline int page_mapcount(struct page *page)
     693                 :            : {
     694                 :    8111459 :         VM_BUG_ON_PAGE(PageSlab(page), page);
     695                 :            : 
     696         [ -  + ]:   16222904 :         if (unlikely(PageCompound(page)))
     697                 :          0 :                 return __page_mapcount(page);
     698                 :    8111459 :         return atomic_read(&page->_mapcount) + 1;
     699                 :            : }
     700                 :            : 
     701                 :            : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
     702                 :            : int total_mapcount(struct page *page);
     703                 :            : int page_trans_huge_mapcount(struct page *page, int *total_mapcount);
     704                 :            : #else
     705                 :          0 : static inline int total_mapcount(struct page *page)
     706                 :            : {
     707                 :          0 :         return page_mapcount(page);
     708                 :            : }
     709                 :     261462 : static inline int page_trans_huge_mapcount(struct page *page,
     710                 :            :                                            int *total_mapcount)
     711                 :            : {
     712                 :     261462 :         int mapcount = page_mapcount(page);
     713         [ +  - ]:     261462 :         if (total_mapcount)
     714                 :     261462 :                 *total_mapcount = mapcount;
     715         [ -  + ]:     261462 :         return mapcount;
     716                 :            : }
     717                 :            : #endif
     718                 :            : 
     719                 :   38395286 : static inline struct page *virt_to_head_page(const void *x)
     720                 :            : {
     721   [ +  -  +  -  :   38395286 :         struct page *page = virt_to_page(x);
          +  -  +  -  +  
          -  +  -  -  -  
          #  #  #  #  #  
                      # ]
     722                 :            : 
     723   [ +  +  +  +  :   38395286 :         return compound_head(page);
          +  +  +  +  +  
          +  +  +  -  -  
          #  #  #  #  #  
                      # ]
     724                 :            : }
     725                 :            : 
     726                 :            : void __put_page(struct page *page);
     727                 :            : 
     728                 :            : void put_pages_list(struct list_head *pages);
     729                 :            : 
     730                 :            : void split_page(struct page *page, unsigned int order);
     731                 :            : 
     732                 :            : /*
     733                 :            :  * Compound pages have a destructor function.  Provide a
     734                 :            :  * prototype for that function and accessor functions.
     735                 :            :  * These are _only_ valid on the head of a compound page.
     736                 :            :  */
     737                 :            : typedef void compound_page_dtor(struct page *);
     738                 :            : 
     739                 :            : /* Keep the enum in sync with compound_page_dtors array in mm/page_alloc.c */
     740                 :            : enum compound_dtor_id {
     741                 :            :         NULL_COMPOUND_DTOR,
     742                 :            :         COMPOUND_PAGE_DTOR,
     743                 :            : #ifdef CONFIG_HUGETLB_PAGE
     744                 :            :         HUGETLB_PAGE_DTOR,
     745                 :            : #endif
     746                 :            : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
     747                 :            :         TRANSHUGE_PAGE_DTOR,
     748                 :            : #endif
     749                 :            :         NR_COMPOUND_DTORS,
     750                 :            : };
     751                 :            : extern compound_page_dtor * const compound_page_dtors[];
     752                 :            : 
     753                 :     136035 : static inline void set_compound_page_dtor(struct page *page,
     754                 :            :                 enum compound_dtor_id compound_dtor)
     755                 :            : {
     756                 :     136035 :         VM_BUG_ON_PAGE(compound_dtor >= NR_COMPOUND_DTORS, page);
     757                 :     136035 :         page[1].compound_dtor = compound_dtor;
     758                 :            : }
     759                 :            : 
     760                 :          0 : static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
     761                 :            : {
     762                 :          0 :         VM_BUG_ON_PAGE(page[1].compound_dtor >= NR_COMPOUND_DTORS, page);
     763                 :          0 :         return compound_page_dtors[page[1].compound_dtor];
     764                 :            : }
     765                 :            : 
     766                 :    9796385 : static inline unsigned int compound_order(struct page *page)
     767                 :            : {
     768   [ +  +  +  +  :    9728005 :         if (!PageHead(page))
          +  +  +  +  -  
             +  -  +  -  
                      - ]
     769                 :            :                 return 0;
     770   [ #  #  #  # ]:    1260272 :         return page[1].compound_order;
     771                 :            : }
     772                 :            : 
     773                 :     136035 : static inline void set_compound_order(struct page *page, unsigned int order)
     774                 :            : {
     775                 :     136035 :         page[1].compound_order = order;
     776                 :            : }
     777                 :            : 
     778                 :            : /* Returns the number of pages in this potentially compound page. */
     779                 :    9083044 : static inline unsigned long compound_nr(struct page *page)
     780                 :            : {
     781         [ +  + ]:   10547786 :         return 1UL << compound_order(page);
     782                 :            : }
     783                 :            : 
     784                 :            : /* Returns the number of bytes in this potentially compound page. */
     785                 :     370424 : static inline unsigned long page_size(struct page *page)
     786                 :            : {
     787         [ -  - ]:     397438 :         return PAGE_SIZE << compound_order(page);
     788                 :            : }
     789                 :            : 
     790                 :            : /* Returns the number of bits needed for the number of bytes in a page */
     791                 :            : static inline unsigned int page_shift(struct page *page)
     792                 :            : {
     793                 :            :         return PAGE_SHIFT + compound_order(page);
     794                 :            : }
     795                 :            : 
     796                 :            : void free_compound_page(struct page *page);
     797                 :            : 
     798                 :            : #ifdef CONFIG_MMU
     799                 :            : /*
     800                 :            :  * Do pte_mkwrite, but only if the vma says VM_WRITE.  We do this when
     801                 :            :  * servicing faults for write access.  In the normal case, do always want
     802                 :            :  * pte_mkwrite.  But get_user_pages can cause write faults for mappings
     803                 :            :  * that do not have writing enabled, when used by access_process_vm.
     804                 :            :  */
     805                 :     478815 : static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
     806                 :            : {
     807   [ +  -  -  -  :     478815 :         if (likely(vma->vm_flags & VM_WRITE))
          +  -  +  -  -  
                -  -  - ]
     808                 :     478815 :                 pte = pte_mkwrite(pte);
     809                 :     386665 :         return pte;
     810                 :            : }
     811                 :            : 
     812                 :            : vm_fault_t alloc_set_pte(struct vm_fault *vmf, struct mem_cgroup *memcg,
     813                 :            :                 struct page *page);
     814                 :            : vm_fault_t finish_fault(struct vm_fault *vmf);
     815                 :            : vm_fault_t finish_mkwrite_fault(struct vm_fault *vmf);
     816                 :            : #endif
     817                 :            : 
     818                 :            : /*
     819                 :            :  * Multiple processes may "see" the same page. E.g. for untouched
     820                 :            :  * mappings of /dev/null, all processes see the same page full of
     821                 :            :  * zeroes, and text pages of executables and shared libraries have
     822                 :            :  * only one copy in memory, at most, normally.
     823                 :            :  *
     824                 :            :  * For the non-reserved pages, page_count(page) denotes a reference count.
     825                 :            :  *   page_count() == 0 means the page is free. page->lru is then used for
     826                 :            :  *   freelist management in the buddy allocator.
     827                 :            :  *   page_count() > 0  means the page has been allocated.
     828                 :            :  *
     829                 :            :  * Pages are allocated by the slab allocator in order to provide memory
     830                 :            :  * to kmalloc and kmem_cache_alloc. In this case, the management of the
     831                 :            :  * page, and the fields in 'struct page' are the responsibility of mm/slab.c
     832                 :            :  * unless a particular usage is carefully commented. (the responsibility of
     833                 :            :  * freeing the kmalloc memory is the caller's, of course).
     834                 :            :  *
     835                 :            :  * A page may be used by anyone else who does a __get_free_page().
     836                 :            :  * In this case, page_count still tracks the references, and should only
     837                 :            :  * be used through the normal accessor functions. The top bits of page->flags
     838                 :            :  * and page->virtual store page management information, but all other fields
     839                 :            :  * are unused and could be used privately, carefully. The management of this
     840                 :            :  * page is the responsibility of the one who allocated it, and those who have
     841                 :            :  * subsequently been given references to it.
     842                 :            :  *
     843                 :            :  * The other pages (we may call them "pagecache pages") are completely
     844                 :            :  * managed by the Linux memory manager: I/O, buffers, swapping etc.
     845                 :            :  * The following discussion applies only to them.
     846                 :            :  *
     847                 :            :  * A pagecache page contains an opaque `private' member, which belongs to the
     848                 :            :  * page's address_space. Usually, this is the address of a circular list of
     849                 :            :  * the page's disk buffers. PG_private must be set to tell the VM to call
     850                 :            :  * into the filesystem to release these pages.
     851                 :            :  *
     852                 :            :  * A page may belong to an inode's memory mapping. In this case, page->mapping
     853                 :            :  * is the pointer to the inode, and page->index is the file offset of the page,
     854                 :            :  * in units of PAGE_SIZE.
     855                 :            :  *
     856                 :            :  * If pagecache pages are not associated with an inode, they are said to be
     857                 :            :  * anonymous pages. These may become associated with the swapcache, and in that
     858                 :            :  * case PG_swapcache is set, and page->private is an offset into the swapcache.
     859                 :            :  *
     860                 :            :  * In either case (swapcache or inode backed), the pagecache itself holds one
     861                 :            :  * reference to the page. Setting PG_private should also increment the
     862                 :            :  * refcount. The each user mapping also has a reference to the page.
     863                 :            :  *
     864                 :            :  * The pagecache pages are stored in a per-mapping radix tree, which is
     865                 :            :  * rooted at mapping->i_pages, and indexed by offset.
     866                 :            :  * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space
     867                 :            :  * lists, we instead now tag pages as dirty/writeback in the radix tree.
     868                 :            :  *
     869                 :            :  * All pagecache pages may be subject to I/O:
     870                 :            :  * - inode pages may need to be read from disk,
     871                 :            :  * - inode pages which have been modified and are MAP_SHARED may need
     872                 :            :  *   to be written back to the inode on disk,
     873                 :            :  * - anonymous pages (including MAP_PRIVATE file mappings) which have been
     874                 :            :  *   modified may need to be swapped out to swap space and (later) to be read
     875                 :            :  *   back into memory.
     876                 :            :  */
     877                 :            : 
     878                 :            : /*
     879                 :            :  * The zone field is never updated after free_area_init_core()
     880                 :            :  * sets it, so none of the operations on it need to be atomic.
     881                 :            :  */
     882                 :            : 
     883                 :            : /* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */
     884                 :            : #define SECTIONS_PGOFF          ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
     885                 :            : #define NODES_PGOFF             (SECTIONS_PGOFF - NODES_WIDTH)
     886                 :            : #define ZONES_PGOFF             (NODES_PGOFF - ZONES_WIDTH)
     887                 :            : #define LAST_CPUPID_PGOFF       (ZONES_PGOFF - LAST_CPUPID_WIDTH)
     888                 :            : #define KASAN_TAG_PGOFF         (LAST_CPUPID_PGOFF - KASAN_TAG_WIDTH)
     889                 :            : 
     890                 :            : /*
     891                 :            :  * Define the bit shifts to access each section.  For non-existent
     892                 :            :  * sections we define the shift as 0; that plus a 0 mask ensures
     893                 :            :  * the compiler will optimise away reference to them.
     894                 :            :  */
     895                 :            : #define SECTIONS_PGSHIFT        (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
     896                 :            : #define NODES_PGSHIFT           (NODES_PGOFF * (NODES_WIDTH != 0))
     897                 :            : #define ZONES_PGSHIFT           (ZONES_PGOFF * (ZONES_WIDTH != 0))
     898                 :            : #define LAST_CPUPID_PGSHIFT     (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))
     899                 :            : #define KASAN_TAG_PGSHIFT       (KASAN_TAG_PGOFF * (KASAN_TAG_WIDTH != 0))
     900                 :            : 
     901                 :            : /* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */
     902                 :            : #ifdef NODE_NOT_IN_PAGE_FLAGS
     903                 :            : #define ZONEID_SHIFT            (SECTIONS_SHIFT + ZONES_SHIFT)
     904                 :            : #define ZONEID_PGOFF            ((SECTIONS_PGOFF < ZONES_PGOFF)? \
     905                 :            :                                                 SECTIONS_PGOFF : ZONES_PGOFF)
     906                 :            : #else
     907                 :            : #define ZONEID_SHIFT            (NODES_SHIFT + ZONES_SHIFT)
     908                 :            : #define ZONEID_PGOFF            ((NODES_PGOFF < ZONES_PGOFF)? \
     909                 :            :                                                 NODES_PGOFF : ZONES_PGOFF)
     910                 :            : #endif
     911                 :            : 
     912                 :            : #define ZONEID_PGSHIFT          (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
     913                 :            : 
     914                 :            : #define ZONES_MASK              ((1UL << ZONES_WIDTH) - 1)
     915                 :            : #define NODES_MASK              ((1UL << NODES_WIDTH) - 1)
     916                 :            : #define SECTIONS_MASK           ((1UL << SECTIONS_WIDTH) - 1)
     917                 :            : #define LAST_CPUPID_MASK        ((1UL << LAST_CPUPID_SHIFT) - 1)
     918                 :            : #define KASAN_TAG_MASK          ((1UL << KASAN_TAG_WIDTH) - 1)
     919                 :            : #define ZONEID_MASK             ((1UL << ZONEID_SHIFT) - 1)
     920                 :            : 
     921                 :    1938252 : static inline enum zone_type page_zonenum(const struct page *page)
     922                 :            : {
     923         [ -  + ]:    1763968 :         return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
     924                 :            : }
     925                 :            : 
     926                 :            : #ifdef CONFIG_ZONE_DEVICE
     927                 :            : static inline bool is_zone_device_page(const struct page *page)
     928                 :            : {
     929                 :            :         return page_zonenum(page) == ZONE_DEVICE;
     930                 :            : }
     931                 :            : extern void memmap_init_zone_device(struct zone *, unsigned long,
     932                 :            :                                     unsigned long, struct dev_pagemap *);
     933                 :            : #else
     934                 :    8540278 : static inline bool is_zone_device_page(const struct page *page)
     935                 :            : {
     936         [ -  + ]:    8540278 :         return false;
     937                 :            : }
     938                 :            : #endif
     939                 :            : 
     940                 :            : #ifdef CONFIG_DEV_PAGEMAP_OPS
     941                 :            : void free_devmap_managed_page(struct page *page);
     942                 :            : DECLARE_STATIC_KEY_FALSE(devmap_managed_key);
     943                 :            : 
     944                 :            : static inline bool page_is_devmap_managed(struct page *page)
     945                 :            : {
     946                 :            :         if (!static_branch_unlikely(&devmap_managed_key))
     947                 :            :                 return false;
     948                 :            :         if (!is_zone_device_page(page))
     949                 :            :                 return false;
     950                 :            :         switch (page->pgmap->type) {
     951                 :            :         case MEMORY_DEVICE_PRIVATE:
     952                 :            :         case MEMORY_DEVICE_FS_DAX:
     953                 :            :                 return true;
     954                 :            :         default:
     955                 :            :                 break;
     956                 :            :         }
     957                 :            :         return false;
     958                 :            : }
     959                 :            : 
     960                 :            : void put_devmap_managed_page(struct page *page);
     961                 :            : 
     962                 :            : #else /* CONFIG_DEV_PAGEMAP_OPS */
     963                 :    1530141 : static inline bool page_is_devmap_managed(struct page *page)
     964                 :            : {
     965                 :    1530141 :         return false;
     966                 :            : }
     967                 :            : 
     968                 :            : static inline void put_devmap_managed_page(struct page *page)
     969                 :            : {
     970                 :            : }
     971                 :            : #endif /* CONFIG_DEV_PAGEMAP_OPS */
     972                 :            : 
     973                 :          0 : static inline bool is_device_private_page(const struct page *page)
     974                 :            : {
     975   [ #  #  #  # ]:          0 :         return IS_ENABLED(CONFIG_DEV_PAGEMAP_OPS) &&
     976                 :            :                 IS_ENABLED(CONFIG_DEVICE_PRIVATE) &&
     977                 :            :                 is_zone_device_page(page) &&
     978                 :            :                 page->pgmap->type == MEMORY_DEVICE_PRIVATE;
     979                 :            : }
     980                 :            : 
     981                 :            : static inline bool is_pci_p2pdma_page(const struct page *page)
     982                 :            : {
     983                 :            :         return IS_ENABLED(CONFIG_DEV_PAGEMAP_OPS) &&
     984                 :            :                 IS_ENABLED(CONFIG_PCI_P2PDMA) &&
     985                 :            :                 is_zone_device_page(page) &&
     986                 :            :                 page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA;
     987                 :            : }
     988                 :            : 
     989                 :            : /* 127: arbitrary random number, small enough to assemble well */
     990                 :            : #define page_ref_zero_or_close_to_overflow(page) \
     991                 :            :         ((unsigned int) page_ref_count(page) + 127u <= 127u)
     992                 :            : 
     993                 :    1862615 : static inline void get_page(struct page *page)
     994                 :            : {
     995   [ -  +  -  +  :    1862615 :         page = compound_head(page);
          -  +  -  +  -  
          -  -  -  #  #  
          #  #  #  #  #  
             #  #  #  #  
                      # ]
     996                 :            :         /*
     997                 :            :          * Getting a normal page or the head of a compound page
     998                 :            :          * requires to already have an elevated page->_refcount.
     999                 :            :          */
    1000                 :    1862615 :         VM_BUG_ON_PAGE(page_ref_zero_or_close_to_overflow(page), page);
    1001                 :    1862615 :         page_ref_inc(page);
    1002                 :         65 : }
    1003                 :            : 
    1004                 :      33023 : static inline __must_check bool try_get_page(struct page *page)
    1005                 :            : {
    1006         [ -  + ]:      33023 :         page = compound_head(page);
    1007   [ -  +  +  - ]:      33023 :         if (WARN_ON_ONCE(page_ref_count(page) <= 0))
    1008                 :            :                 return false;
    1009                 :      33023 :         page_ref_inc(page);
    1010                 :      33023 :         return true;
    1011                 :            : }
    1012                 :            : 
    1013                 :    1530141 : static inline void put_page(struct page *page)
    1014                 :            : {
    1015         [ -  + ]:    1530141 :         page = compound_head(page);
    1016                 :            : 
    1017                 :            :         /*
    1018                 :            :          * For devmap managed pages we need to catch refcount transition from
    1019                 :            :          * 2 to 1, when refcount reach one it means the page is free and we
    1020                 :            :          * need to inform the device driver through callback. See
    1021                 :            :          * include/linux/memremap.h and HMM for details.
    1022                 :            :          */
    1023                 :    1530141 :         if (page_is_devmap_managed(page)) {
    1024                 :            :                 put_devmap_managed_page(page);
    1025                 :            :                 return;
    1026                 :            :         }
    1027                 :            : 
    1028         [ +  + ]:    1530141 :         if (put_page_testzero(page))
    1029                 :       1458 :                 __put_page(page);
    1030                 :            : }
    1031                 :            : 
    1032                 :            : /**
    1033                 :            :  * unpin_user_page() - release a gup-pinned page
    1034                 :            :  * @page:            pointer to page to be released
    1035                 :            :  *
    1036                 :            :  * Pages that were pinned via pin_user_pages*() must be released via either
    1037                 :            :  * unpin_user_page(), or one of the unpin_user_pages*() routines. This is so
    1038                 :            :  * that eventually such pages can be separately tracked and uniquely handled. In
    1039                 :            :  * particular, interactions with RDMA and filesystems need special handling.
    1040                 :            :  *
    1041                 :            :  * unpin_user_page() and put_page() are not interchangeable, despite this early
    1042                 :            :  * implementation that makes them look the same. unpin_user_page() calls must
    1043                 :            :  * be perfectly matched up with pin*() calls.
    1044                 :            :  */
    1045                 :          0 : static inline void unpin_user_page(struct page *page)
    1046                 :            : {
    1047                 :          0 :         put_page(page);
    1048                 :            : }
    1049                 :            : 
    1050                 :            : void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
    1051                 :            :                                  bool make_dirty);
    1052                 :            : 
    1053                 :            : void unpin_user_pages(struct page **pages, unsigned long npages);
    1054                 :            : 
    1055                 :            : #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
    1056                 :            : #define SECTION_IN_PAGE_FLAGS
    1057                 :            : #endif
    1058                 :            : 
    1059                 :            : /*
    1060                 :            :  * The identification function is mainly used by the buddy allocator for
    1061                 :            :  * determining if two pages could be buddies. We are not really identifying
    1062                 :            :  * the zone since we could be using the section number id if we do not have
    1063                 :            :  * node id available in page flags.
    1064                 :            :  * We only guarantee that it will return the same value for two combinable
    1065                 :            :  * pages in a zone.
    1066                 :            :  */
    1067                 :     311439 : static inline int page_zone_id(struct page *page)
    1068                 :            : {
    1069   [ -  +  +  -  :     311439 :         return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
                   +  - ]
    1070                 :            : }
    1071                 :            : 
    1072                 :            : #ifdef NODE_NOT_IN_PAGE_FLAGS
    1073                 :            : extern int page_to_nid(const struct page *page);
    1074                 :            : #else
    1075                 :    5088977 : static inline int page_to_nid(const struct page *page)
    1076                 :            : {
    1077                 :    5088977 :         struct page *p = (struct page *)page;
    1078                 :            : 
    1079   [ +  -  -  -  :    4394087 :         return (PF_POISONED_CHECK(p)->flags >> NODES_PGSHIFT) & NODES_MASK;
          -  -  -  +  -  
          +  -  +  -  +  
             -  +  +  + ]
    1080                 :            : }
    1081                 :            : #endif
    1082                 :            : 
    1083                 :            : #ifdef CONFIG_NUMA_BALANCING
    1084                 :            : static inline int cpu_pid_to_cpupid(int cpu, int pid)
    1085                 :            : {
    1086                 :            :         return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
    1087                 :            : }
    1088                 :            : 
    1089                 :            : static inline int cpupid_to_pid(int cpupid)
    1090                 :            : {
    1091                 :            :         return cpupid & LAST__PID_MASK;
    1092                 :            : }
    1093                 :            : 
    1094                 :            : static inline int cpupid_to_cpu(int cpupid)
    1095                 :            : {
    1096                 :            :         return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;
    1097                 :            : }
    1098                 :            : 
    1099                 :            : static inline int cpupid_to_nid(int cpupid)
    1100                 :            : {
    1101                 :            :         return cpu_to_node(cpupid_to_cpu(cpupid));
    1102                 :            : }
    1103                 :            : 
    1104                 :            : static inline bool cpupid_pid_unset(int cpupid)
    1105                 :            : {
    1106                 :            :         return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);
    1107                 :            : }
    1108                 :            : 
    1109                 :            : static inline bool cpupid_cpu_unset(int cpupid)
    1110                 :            : {
    1111                 :            :         return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);
    1112                 :            : }
    1113                 :            : 
    1114                 :            : static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid)
    1115                 :            : {
    1116                 :            :         return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);
    1117                 :            : }
    1118                 :            : 
    1119                 :            : #define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid)
    1120                 :            : #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
    1121                 :            : static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
    1122                 :            : {
    1123                 :            :         return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK);
    1124                 :            : }
    1125                 :            : 
    1126                 :            : static inline int page_cpupid_last(struct page *page)
    1127                 :            : {
    1128                 :            :         return page->_last_cpupid;
    1129                 :            : }
    1130                 :            : static inline void page_cpupid_reset_last(struct page *page)
    1131                 :            : {
    1132                 :            :         page->_last_cpupid = -1 & LAST_CPUPID_MASK;
    1133                 :            : }
    1134                 :            : #else
    1135                 :            : static inline int page_cpupid_last(struct page *page)
    1136                 :            : {
    1137                 :            :         return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
    1138                 :            : }
    1139                 :            : 
    1140                 :            : extern int page_cpupid_xchg_last(struct page *page, int cpupid);
    1141                 :            : 
    1142                 :            : static inline void page_cpupid_reset_last(struct page *page)
    1143                 :            : {
    1144                 :            :         page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;
    1145                 :            : }
    1146                 :            : #endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
    1147                 :            : #else /* !CONFIG_NUMA_BALANCING */
    1148                 :          0 : static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
    1149                 :            : {
    1150         [ #  # ]:          0 :         return page_to_nid(page); /* XXX */
    1151                 :            : }
    1152                 :            : 
    1153                 :            : static inline int page_cpupid_last(struct page *page)
    1154                 :            : {
    1155                 :            :         return page_to_nid(page); /* XXX */
    1156                 :            : }
    1157                 :            : 
    1158                 :            : static inline int cpupid_to_nid(int cpupid)
    1159                 :            : {
    1160                 :            :         return -1;
    1161                 :            : }
    1162                 :            : 
    1163                 :            : static inline int cpupid_to_pid(int cpupid)
    1164                 :            : {
    1165                 :            :         return -1;
    1166                 :            : }
    1167                 :            : 
    1168                 :            : static inline int cpupid_to_cpu(int cpupid)
    1169                 :            : {
    1170                 :            :         return -1;
    1171                 :            : }
    1172                 :            : 
    1173                 :            : static inline int cpu_pid_to_cpupid(int nid, int pid)
    1174                 :            : {
    1175                 :            :         return -1;
    1176                 :            : }
    1177                 :            : 
    1178                 :            : static inline bool cpupid_pid_unset(int cpupid)
    1179                 :            : {
    1180                 :            :         return 1;
    1181                 :            : }
    1182                 :            : 
    1183                 :            : static inline void page_cpupid_reset_last(struct page *page)
    1184                 :            : {
    1185                 :            : }
    1186                 :            : 
    1187                 :            : static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
    1188                 :            : {
    1189                 :            :         return false;
    1190                 :            : }
    1191                 :            : #endif /* CONFIG_NUMA_BALANCING */
    1192                 :            : 
    1193                 :            : #ifdef CONFIG_KASAN_SW_TAGS
    1194                 :            : static inline u8 page_kasan_tag(const struct page *page)
    1195                 :            : {
    1196                 :            :         return (page->flags >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK;
    1197                 :            : }
    1198                 :            : 
    1199                 :            : static inline void page_kasan_tag_set(struct page *page, u8 tag)
    1200                 :            : {
    1201                 :            :         page->flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT);
    1202                 :            :         page->flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT;
    1203                 :            : }
    1204                 :            : 
    1205                 :            : static inline void page_kasan_tag_reset(struct page *page)
    1206                 :            : {
    1207                 :            :         page_kasan_tag_set(page, 0xff);
    1208                 :            : }
    1209                 :            : #else
    1210                 :            : static inline u8 page_kasan_tag(const struct page *page)
    1211                 :            : {
    1212                 :            :         return 0xff;
    1213                 :            : }
    1214                 :            : 
    1215                 :            : static inline void page_kasan_tag_set(struct page *page, u8 tag) { }
    1216                 :    1096879 : static inline void page_kasan_tag_reset(struct page *page) { }
    1217                 :            : #endif
    1218                 :            : 
    1219                 :     577696 : static inline struct zone *page_zone(const struct page *page)
    1220                 :            : {
    1221   [ -  -  -  -  :     577696 :         return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
                   +  - ]
    1222                 :            : }
    1223                 :            : 
    1224                 :    3710774 : static inline pg_data_t *page_pgdat(const struct page *page)
    1225                 :            : {
    1226   [ +  +  +  + ]:    3693679 :         return NODE_DATA(page_to_nid(page));
    1227                 :            : }
    1228                 :            : 
    1229                 :            : #ifdef SECTION_IN_PAGE_FLAGS
    1230                 :            : static inline void set_page_section(struct page *page, unsigned long section)
    1231                 :            : {
    1232                 :            :         page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
    1233                 :            :         page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
    1234                 :            : }
    1235                 :            : 
    1236                 :            : static inline unsigned long page_to_section(const struct page *page)
    1237                 :            : {
    1238                 :            :         return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
    1239                 :            : }
    1240                 :            : #endif
    1241                 :            : 
    1242                 :            : static inline void set_page_zone(struct page *page, enum zone_type zone)
    1243                 :            : {
    1244                 :            :         page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
    1245                 :            :         page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
    1246                 :            : }
    1247                 :            : 
    1248                 :            : static inline void set_page_node(struct page *page, unsigned long node)
    1249                 :            : {
    1250                 :            :         page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
    1251                 :            :         page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
    1252                 :            : }
    1253                 :            : 
    1254                 :            : static inline void set_page_links(struct page *page, enum zone_type zone,
    1255                 :            :         unsigned long node, unsigned long pfn)
    1256                 :            : {
    1257                 :            :         set_page_zone(page, zone);
    1258                 :            :         set_page_node(page, node);
    1259                 :            : #ifdef SECTION_IN_PAGE_FLAGS
    1260                 :            :         set_page_section(page, pfn_to_section_nr(pfn));
    1261                 :            : #endif
    1262                 :            : }
    1263                 :            : 
    1264                 :            : #ifdef CONFIG_MEMCG
    1265                 :            : static inline struct mem_cgroup *page_memcg(struct page *page)
    1266                 :            : {
    1267                 :            :         return page->mem_cgroup;
    1268                 :            : }
    1269                 :            : static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
    1270                 :            : {
    1271                 :            :         WARN_ON_ONCE(!rcu_read_lock_held());
    1272                 :            :         return READ_ONCE(page->mem_cgroup);
    1273                 :            : }
    1274                 :            : #else
    1275                 :          0 : static inline struct mem_cgroup *page_memcg(struct page *page)
    1276                 :            : {
    1277                 :          0 :         return NULL;
    1278                 :            : }
    1279                 :      60147 : static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
    1280                 :            : {
    1281                 :      60147 :         WARN_ON_ONCE(!rcu_read_lock_held());
    1282                 :      60147 :         return NULL;
    1283                 :            : }
    1284                 :            : #endif
    1285                 :            : 
    1286                 :            : /*
    1287                 :            :  * Some inline functions in vmstat.h depend on page_zone()
    1288                 :            :  */
    1289                 :            : #include <linux/vmstat.h>
    1290                 :            : 
    1291                 :   15205715 : static __always_inline void *lowmem_page_address(const struct page *page)
    1292                 :            : {
    1293   [ +  +  +  +  :   15203460 :         return page_to_virt(page);
          +  +  #  #  #  
          #  #  #  #  #  
                   #  # ]
    1294                 :            : }
    1295                 :            : 
    1296                 :            : #if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
    1297                 :            : #define HASHED_PAGE_VIRTUAL
    1298                 :            : #endif
    1299                 :            : 
    1300                 :            : #if defined(WANT_PAGE_VIRTUAL)
    1301                 :            : static inline void *page_address(const struct page *page)
    1302                 :            : {
    1303                 :            :         return page->virtual;
    1304                 :            : }
    1305                 :            : static inline void set_page_address(struct page *page, void *address)
    1306                 :            : {
    1307                 :            :         page->virtual = address;
    1308                 :            : }
    1309                 :            : #define page_address_init()  do { } while(0)
    1310                 :            : #endif
    1311                 :            : 
    1312                 :            : #if defined(HASHED_PAGE_VIRTUAL)
    1313                 :            : void *page_address(const struct page *page);
    1314                 :            : void set_page_address(struct page *page, void *virtual);
    1315                 :            : void page_address_init(void);
    1316                 :            : #endif
    1317                 :            : 
    1318                 :            : #if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
    1319                 :            : #define page_address(page) lowmem_page_address(page)
    1320                 :            : #define set_page_address(page, address)  do { } while(0)
    1321                 :            : #define page_address_init()  do { } while(0)
    1322                 :            : #endif
    1323                 :            : 
    1324                 :            : extern void *page_rmapping(struct page *page);
    1325                 :            : extern struct anon_vma *page_anon_vma(struct page *page);
    1326                 :            : extern struct address_space *page_mapping(struct page *page);
    1327                 :            : 
    1328                 :            : extern struct address_space *__page_file_mapping(struct page *);
    1329                 :            : 
    1330                 :            : static inline
    1331                 :          0 : struct address_space *page_file_mapping(struct page *page)
    1332                 :            : {
    1333   [ #  #  #  # ]:          0 :         if (unlikely(PageSwapCache(page)))
    1334                 :          0 :                 return __page_file_mapping(page);
    1335                 :            : 
    1336                 :          0 :         return page->mapping;
    1337                 :            : }
    1338                 :            : 
    1339                 :            : extern pgoff_t __page_file_index(struct page *page);
    1340                 :            : 
    1341                 :            : /*
    1342                 :            :  * Return the pagecache index of the passed page.  Regular pagecache pages
    1343                 :            :  * use ->index whereas swapcache pages use swp_offset(->private)
    1344                 :            :  */
    1345                 :       8219 : static inline pgoff_t page_index(struct page *page)
    1346                 :            : {
    1347   [ -  +  -  + ]:      16438 :         if (unlikely(PageSwapCache(page)))
    1348                 :          0 :                 return __page_file_index(page);
    1349                 :       8219 :         return page->index;
    1350                 :            : }
    1351                 :            : 
    1352                 :            : bool page_mapped(struct page *page);
    1353                 :            : struct address_space *page_mapping(struct page *page);
    1354                 :            : struct address_space *page_mapping_file(struct page *page);
    1355                 :            : 
    1356                 :            : /*
    1357                 :            :  * Return true only if the page has been allocated with
    1358                 :            :  * ALLOC_NO_WATERMARKS and the low watermark was not
    1359                 :            :  * met implying that the system is under some pressure.
    1360                 :            :  */
    1361                 :     343324 : static inline bool page_is_pfmemalloc(struct page *page)
    1362                 :            : {
    1363                 :            :         /*
    1364                 :            :          * Page index cannot be this large so this must be
    1365                 :            :          * a pfmemalloc page.
    1366                 :            :          */
    1367   [ -  +  #  #  :     343324 :         return page->index == -1UL;
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  # ]
    1368                 :            : }
    1369                 :            : 
    1370                 :            : /*
    1371                 :            :  * Only to be called by the page allocator on a freshly allocated
    1372                 :            :  * page.
    1373                 :            :  */
    1374                 :          0 : static inline void set_page_pfmemalloc(struct page *page)
    1375                 :            : {
    1376                 :          0 :         page->index = -1UL;
    1377                 :          0 : }
    1378                 :            : 
    1379                 :    1632038 : static inline void clear_page_pfmemalloc(struct page *page)
    1380                 :            : {
    1381                 :    1632038 :         page->index = 0;
    1382                 :    1632038 : }
    1383                 :            : 
    1384                 :            : /*
    1385                 :            :  * Can be called by the pagefault handler when it gets a VM_FAULT_OOM.
    1386                 :            :  */
    1387                 :            : extern void pagefault_out_of_memory(void);
    1388                 :            : 
    1389                 :            : #define offset_in_page(p)       ((unsigned long)(p) & ~PAGE_MASK)
    1390                 :            : 
    1391                 :            : /*
    1392                 :            :  * Flags passed to show_mem() and show_free_areas() to suppress output in
    1393                 :            :  * various contexts.
    1394                 :            :  */
    1395                 :            : #define SHOW_MEM_FILTER_NODES           (0x0001u)       /* disallowed nodes */
    1396                 :            : 
    1397                 :            : extern void show_free_areas(unsigned int flags, nodemask_t *nodemask);
    1398                 :            : 
    1399                 :            : #ifdef CONFIG_MMU
    1400                 :            : extern bool can_do_mlock(void);
    1401                 :            : #else
    1402                 :            : static inline bool can_do_mlock(void) { return false; }
    1403                 :            : #endif
    1404                 :            : extern int user_shm_lock(size_t, struct user_struct *);
    1405                 :            : extern void user_shm_unlock(size_t, struct user_struct *);
    1406                 :            : 
    1407                 :            : /*
    1408                 :            :  * Parameter block passed down to zap_pte_range in exceptional cases.
    1409                 :            :  */
    1410                 :            : struct zap_details {
    1411                 :            :         struct address_space *check_mapping;    /* Check page->mapping if set */
    1412                 :            :         pgoff_t first_index;                    /* Lowest page->index to unmap */
    1413                 :            :         pgoff_t last_index;                     /* Highest page->index to unmap */
    1414                 :            : };
    1415                 :            : 
    1416                 :            : struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
    1417                 :            :                              pte_t pte);
    1418                 :            : struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
    1419                 :            :                                 pmd_t pmd);
    1420                 :            : 
    1421                 :            : void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
    1422                 :            :                   unsigned long size);
    1423                 :            : void zap_page_range(struct vm_area_struct *vma, unsigned long address,
    1424                 :            :                     unsigned long size);
    1425                 :            : void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
    1426                 :            :                 unsigned long start, unsigned long end);
    1427                 :            : 
    1428                 :            : struct mmu_notifier_range;
    1429                 :            : 
    1430                 :            : void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
    1431                 :            :                 unsigned long end, unsigned long floor, unsigned long ceiling);
    1432                 :            : int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
    1433                 :            :                         struct vm_area_struct *vma);
    1434                 :            : int follow_pte_pmd(struct mm_struct *mm, unsigned long address,
    1435                 :            :                    struct mmu_notifier_range *range,
    1436                 :            :                    pte_t **ptepp, pmd_t **pmdpp, spinlock_t **ptlp);
    1437                 :            : int follow_pfn(struct vm_area_struct *vma, unsigned long address,
    1438                 :            :         unsigned long *pfn);
    1439                 :            : int follow_phys(struct vm_area_struct *vma, unsigned long address,
    1440                 :            :                 unsigned int flags, unsigned long *prot, resource_size_t *phys);
    1441                 :            : int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
    1442                 :            :                         void *buf, int len, int write);
    1443                 :            : 
    1444                 :            : extern void truncate_pagecache(struct inode *inode, loff_t new);
    1445                 :            : extern void truncate_setsize(struct inode *inode, loff_t newsize);
    1446                 :            : void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
    1447                 :            : void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
    1448                 :            : int truncate_inode_page(struct address_space *mapping, struct page *page);
    1449                 :            : int generic_error_remove_page(struct address_space *mapping, struct page *page);
    1450                 :            : int invalidate_inode_page(struct page *page);
    1451                 :            : 
    1452                 :            : #ifdef CONFIG_MMU
    1453                 :            : extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
    1454                 :            :                         unsigned long address, unsigned int flags);
    1455                 :            : extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
    1456                 :            :                             unsigned long address, unsigned int fault_flags,
    1457                 :            :                             bool *unlocked);
    1458                 :            : void unmap_mapping_pages(struct address_space *mapping,
    1459                 :            :                 pgoff_t start, pgoff_t nr, bool even_cows);
    1460                 :            : void unmap_mapping_range(struct address_space *mapping,
    1461                 :            :                 loff_t const holebegin, loff_t const holelen, int even_cows);
    1462                 :            : #else
    1463                 :            : static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
    1464                 :            :                 unsigned long address, unsigned int flags)
    1465                 :            : {
    1466                 :            :         /* should never happen if there's no MMU */
    1467                 :            :         BUG();
    1468                 :            :         return VM_FAULT_SIGBUS;
    1469                 :            : }
    1470                 :            : static inline int fixup_user_fault(struct task_struct *tsk,
    1471                 :            :                 struct mm_struct *mm, unsigned long address,
    1472                 :            :                 unsigned int fault_flags, bool *unlocked)
    1473                 :            : {
    1474                 :            :         /* should never happen if there's no MMU */
    1475                 :            :         BUG();
    1476                 :            :         return -EFAULT;
    1477                 :            : }
    1478                 :            : static inline void unmap_mapping_pages(struct address_space *mapping,
    1479                 :            :                 pgoff_t start, pgoff_t nr, bool even_cows) { }
    1480                 :            : static inline void unmap_mapping_range(struct address_space *mapping,
    1481                 :            :                 loff_t const holebegin, loff_t const holelen, int even_cows) { }
    1482                 :            : #endif
    1483                 :            : 
    1484                 :            : static inline void unmap_shared_mapping_range(struct address_space *mapping,
    1485                 :            :                 loff_t const holebegin, loff_t const holelen)
    1486                 :            : {
    1487                 :            :         unmap_mapping_range(mapping, holebegin, holelen, 0);
    1488                 :            : }
    1489                 :            : 
    1490                 :            : extern int access_process_vm(struct task_struct *tsk, unsigned long addr,
    1491                 :            :                 void *buf, int len, unsigned int gup_flags);
    1492                 :            : extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
    1493                 :            :                 void *buf, int len, unsigned int gup_flags);
    1494                 :            : extern int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm,
    1495                 :            :                 unsigned long addr, void *buf, int len, unsigned int gup_flags);
    1496                 :            : 
    1497                 :            : long get_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
    1498                 :            :                             unsigned long start, unsigned long nr_pages,
    1499                 :            :                             unsigned int gup_flags, struct page **pages,
    1500                 :            :                             struct vm_area_struct **vmas, int *locked);
    1501                 :            : long pin_user_pages_remote(struct task_struct *tsk, struct mm_struct *mm,
    1502                 :            :                            unsigned long start, unsigned long nr_pages,
    1503                 :            :                            unsigned int gup_flags, struct page **pages,
    1504                 :            :                            struct vm_area_struct **vmas, int *locked);
    1505                 :            : long get_user_pages(unsigned long start, unsigned long nr_pages,
    1506                 :            :                             unsigned int gup_flags, struct page **pages,
    1507                 :            :                             struct vm_area_struct **vmas);
    1508                 :            : long pin_user_pages(unsigned long start, unsigned long nr_pages,
    1509                 :            :                     unsigned int gup_flags, struct page **pages,
    1510                 :            :                     struct vm_area_struct **vmas);
    1511                 :            : long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
    1512                 :            :                     unsigned int gup_flags, struct page **pages, int *locked);
    1513                 :            : long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
    1514                 :            :                     struct page **pages, unsigned int gup_flags);
    1515                 :            : 
    1516                 :            : int get_user_pages_fast(unsigned long start, int nr_pages,
    1517                 :            :                         unsigned int gup_flags, struct page **pages);
    1518                 :            : int pin_user_pages_fast(unsigned long start, int nr_pages,
    1519                 :            :                         unsigned int gup_flags, struct page **pages);
    1520                 :            : 
    1521                 :            : int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc);
    1522                 :            : int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
    1523                 :            :                         struct task_struct *task, bool bypass_rlim);
    1524                 :            : 
    1525                 :            : /* Container for pinned pfns / pages */
    1526                 :            : struct frame_vector {
    1527                 :            :         unsigned int nr_allocated;      /* Number of frames we have space for */
    1528                 :            :         unsigned int nr_frames; /* Number of frames stored in ptrs array */
    1529                 :            :         bool got_ref;           /* Did we pin pages by getting page ref? */
    1530                 :            :         bool is_pfns;           /* Does array contain pages or pfns? */
    1531                 :            :         void *ptrs[0];          /* Array of pinned pfns / pages. Use
    1532                 :            :                                  * pfns_vector_pages() or pfns_vector_pfns()
    1533                 :            :                                  * for access */
    1534                 :            : };
    1535                 :            : 
    1536                 :            : struct frame_vector *frame_vector_create(unsigned int nr_frames);
    1537                 :            : void frame_vector_destroy(struct frame_vector *vec);
    1538                 :            : int get_vaddr_frames(unsigned long start, unsigned int nr_pfns,
    1539                 :            :                      unsigned int gup_flags, struct frame_vector *vec);
    1540                 :            : void put_vaddr_frames(struct frame_vector *vec);
    1541                 :            : int frame_vector_to_pages(struct frame_vector *vec);
    1542                 :            : void frame_vector_to_pfns(struct frame_vector *vec);
    1543                 :            : 
    1544                 :            : static inline unsigned int frame_vector_count(struct frame_vector *vec)
    1545                 :            : {
    1546                 :            :         return vec->nr_frames;
    1547                 :            : }
    1548                 :            : 
    1549                 :            : static inline struct page **frame_vector_pages(struct frame_vector *vec)
    1550                 :            : {
    1551                 :            :         if (vec->is_pfns) {
    1552                 :            :                 int err = frame_vector_to_pages(vec);
    1553                 :            : 
    1554                 :            :                 if (err)
    1555                 :            :                         return ERR_PTR(err);
    1556                 :            :         }
    1557                 :            :         return (struct page **)(vec->ptrs);
    1558                 :            : }
    1559                 :            : 
    1560                 :            : static inline unsigned long *frame_vector_pfns(struct frame_vector *vec)
    1561                 :            : {
    1562                 :            :         if (!vec->is_pfns)
    1563                 :            :                 frame_vector_to_pfns(vec);
    1564                 :            :         return (unsigned long *)(vec->ptrs);
    1565                 :            : }
    1566                 :            : 
    1567                 :            : struct kvec;
    1568                 :            : int get_kernel_pages(const struct kvec *iov, int nr_pages, int write,
    1569                 :            :                         struct page **pages);
    1570                 :            : int get_kernel_page(unsigned long start, int write, struct page **pages);
    1571                 :            : struct page *get_dump_page(unsigned long addr);
    1572                 :            : 
    1573                 :            : extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
    1574                 :            : extern void do_invalidatepage(struct page *page, unsigned int offset,
    1575                 :            :                               unsigned int length);
    1576                 :            : 
    1577                 :            : void __set_page_dirty(struct page *, struct address_space *, int warn);
    1578                 :            : int __set_page_dirty_nobuffers(struct page *page);
    1579                 :            : int __set_page_dirty_no_writeback(struct page *page);
    1580                 :            : int redirty_page_for_writepage(struct writeback_control *wbc,
    1581                 :            :                                 struct page *page);
    1582                 :            : void account_page_dirtied(struct page *page, struct address_space *mapping);
    1583                 :            : void account_page_cleaned(struct page *page, struct address_space *mapping,
    1584                 :            :                           struct bdi_writeback *wb);
    1585                 :            : int set_page_dirty(struct page *page);
    1586                 :            : int set_page_dirty_lock(struct page *page);
    1587                 :            : void __cancel_dirty_page(struct page *page);
    1588                 :       7410 : static inline void cancel_dirty_page(struct page *page)
    1589                 :            : {
    1590                 :            :         /* Avoid atomic ops, locking, etc. when not actually needed. */
    1591   [ +  +  -  + ]:      14820 :         if (PageDirty(page))
    1592                 :        442 :                 __cancel_dirty_page(page);
    1593                 :       7410 : }
    1594                 :            : int clear_page_dirty_for_io(struct page *page);
    1595                 :            : 
    1596                 :            : int get_cmdline(struct task_struct *task, char *buffer, int buflen);
    1597                 :            : 
    1598                 :            : extern unsigned long move_page_tables(struct vm_area_struct *vma,
    1599                 :            :                 unsigned long old_addr, struct vm_area_struct *new_vma,
    1600                 :            :                 unsigned long new_addr, unsigned long len,
    1601                 :            :                 bool need_rmap_locks);
    1602                 :            : extern unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
    1603                 :            :                               unsigned long end, pgprot_t newprot,
    1604                 :            :                               int dirty_accountable, int prot_numa);
    1605                 :            : extern int mprotect_fixup(struct vm_area_struct *vma,
    1606                 :            :                           struct vm_area_struct **pprev, unsigned long start,
    1607                 :            :                           unsigned long end, unsigned long newflags);
    1608                 :            : 
    1609                 :            : /*
    1610                 :            :  * doesn't attempt to fault and will return short.
    1611                 :            :  */
    1612                 :            : int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
    1613                 :            :                           struct page **pages);
    1614                 :            : /*
    1615                 :            :  * per-process(per-mm_struct) statistics.
    1616                 :            :  */
    1617                 :     594917 : static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
    1618                 :            : {
    1619                 :     594056 :         long val = atomic_long_read(&mm->rss_stat.count[member]);
    1620                 :            : 
    1621                 :            : #ifdef SPLIT_RSS_COUNTING
    1622                 :            :         /*
    1623                 :            :          * counter is updated in asynchronous manner and may go to minus.
    1624                 :            :          * But it's never be expected number for users.
    1625                 :            :          */
    1626                 :     594917 :         if (val < 0)
    1627                 :            :                 val = 0;
    1628                 :            : #endif
    1629                 :     198210 :         return (unsigned long)val;
    1630                 :            : }
    1631                 :            : 
    1632                 :            : void mm_trace_rss_stat(struct mm_struct *mm, int member, long count);
    1633                 :            : 
    1634                 :    1165371 : static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
    1635                 :            : {
    1636                 :    1165371 :         long count = atomic_long_add_return(value, &mm->rss_stat.count[member]);
    1637                 :            : 
    1638                 :    1165371 :         mm_trace_rss_stat(mm, member, count);
    1639                 :    1011025 : }
    1640                 :            : 
    1641                 :          0 : static inline void inc_mm_counter(struct mm_struct *mm, int member)
    1642                 :            : {
    1643                 :          0 :         long count = atomic_long_inc_return(&mm->rss_stat.count[member]);
    1644                 :            : 
    1645                 :          0 :         mm_trace_rss_stat(mm, member, count);
    1646                 :          0 : }
    1647                 :            : 
    1648                 :          0 : static inline void dec_mm_counter(struct mm_struct *mm, int member)
    1649                 :            : {
    1650                 :          0 :         long count = atomic_long_dec_return(&mm->rss_stat.count[member]);
    1651                 :            : 
    1652                 :          0 :         mm_trace_rss_stat(mm, member, count);
    1653                 :          0 : }
    1654                 :            : 
    1655                 :            : /* Optimized variant when page is already known not to be PageAnon */
    1656                 :   14008390 : static inline int mm_counter_file(struct page *page)
    1657                 :            : {
    1658         [ +  + ]:   28016800 :         if (PageSwapBacked(page))
    1659                 :       1760 :                 return MM_SHMEMPAGES;
    1660                 :            :         return MM_FILEPAGES;
    1661                 :            : }
    1662                 :            : 
    1663                 :    8602088 : static inline int mm_counter(struct page *page)
    1664                 :            : {
    1665         [ +  + ]:    8602088 :         if (PageAnon(page))
    1666                 :            :                 return MM_ANONPAGES;
    1667                 :    6766845 :         return mm_counter_file(page);
    1668                 :            : }
    1669                 :            : 
    1670                 :     197923 : static inline unsigned long get_mm_rss(struct mm_struct *mm)
    1671                 :            : {
    1672                 :     395846 :         return get_mm_counter(mm, MM_FILEPAGES) +
    1673                 :     197923 :                 get_mm_counter(mm, MM_ANONPAGES) +
    1674                 :            :                 get_mm_counter(mm, MM_SHMEMPAGES);
    1675                 :            : }
    1676                 :            : 
    1677                 :      16224 : static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
    1678                 :            : {
    1679                 :      32448 :         return max(mm->hiwater_rss, get_mm_rss(mm));
    1680                 :            : }
    1681                 :            : 
    1682                 :          0 : static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm)
    1683                 :            : {
    1684                 :          0 :         return max(mm->hiwater_vm, mm->total_vm);
    1685                 :            : }
    1686                 :            : 
    1687                 :     158034 : static inline void update_hiwater_rss(struct mm_struct *mm)
    1688                 :            : {
    1689                 :     158034 :         unsigned long _rss = get_mm_rss(mm);
    1690                 :            : 
    1691   [ +  +  #  # ]:     158034 :         if ((mm)->hiwater_rss < _rss)
    1692                 :      71586 :                 (mm)->hiwater_rss = _rss;
    1693                 :            : }
    1694                 :            : 
    1695                 :     158034 : static inline void update_hiwater_vm(struct mm_struct *mm)
    1696                 :            : {
    1697         [ +  + ]:     158034 :         if (mm->hiwater_vm < mm->total_vm)
    1698                 :      65573 :                 mm->hiwater_vm = mm->total_vm;
    1699                 :            : }
    1700                 :            : 
    1701                 :          0 : static inline void reset_mm_hiwater_rss(struct mm_struct *mm)
    1702                 :            : {
    1703                 :          0 :         mm->hiwater_rss = get_mm_rss(mm);
    1704                 :            : }
    1705                 :            : 
    1706                 :      16224 : static inline void setmax_mm_hiwater_rss(unsigned long *maxrss,
    1707                 :            :                                          struct mm_struct *mm)
    1708                 :            : {
    1709                 :      16224 :         unsigned long hiwater_rss = get_mm_hiwater_rss(mm);
    1710                 :            : 
    1711         [ +  + ]:      16224 :         if (*maxrss < hiwater_rss)
    1712                 :      15980 :                 *maxrss = hiwater_rss;
    1713                 :            : }
    1714                 :            : 
    1715                 :            : #if defined(SPLIT_RSS_COUNTING)
    1716                 :            : void sync_mm_rss(struct mm_struct *mm);
    1717                 :            : #else
    1718                 :            : static inline void sync_mm_rss(struct mm_struct *mm)
    1719                 :            : {
    1720                 :            : }
    1721                 :            : #endif
    1722                 :            : 
    1723                 :            : #ifndef CONFIG_ARCH_HAS_PTE_DEVMAP
    1724                 :            : static inline int pte_devmap(pte_t pte)
    1725                 :            : {
    1726                 :            :         return 0;
    1727                 :            : }
    1728                 :            : #endif
    1729                 :            : 
    1730                 :            : int vma_wants_writenotify(struct vm_area_struct *vma, pgprot_t vm_page_prot);
    1731                 :            : 
    1732                 :            : extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
    1733                 :            :                                spinlock_t **ptl);
    1734                 :     130284 : static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
    1735                 :            :                                     spinlock_t **ptl)
    1736                 :            : {
    1737                 :     130284 :         pte_t *ptep;
    1738                 :     130284 :         __cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl));
    1739         [ +  + ]:     130284 :         return ptep;
    1740                 :            : }
    1741                 :            : 
    1742                 :            : #ifdef __PAGETABLE_P4D_FOLDED
    1743                 :            : static inline int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
    1744                 :            :                                                 unsigned long address)
    1745                 :            : {
    1746                 :            :         return 0;
    1747                 :            : }
    1748                 :            : #else
    1749                 :            : int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
    1750                 :            : #endif
    1751                 :            : 
    1752                 :            : #if defined(__PAGETABLE_PUD_FOLDED) || !defined(CONFIG_MMU)
    1753                 :            : static inline int __pud_alloc(struct mm_struct *mm, p4d_t *p4d,
    1754                 :            :                                                 unsigned long address)
    1755                 :            : {
    1756                 :            :         return 0;
    1757                 :            : }
    1758                 :            : static inline void mm_inc_nr_puds(struct mm_struct *mm) {}
    1759                 :            : static inline void mm_dec_nr_puds(struct mm_struct *mm) {}
    1760                 :            : 
    1761                 :            : #else
    1762                 :            : int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address);
    1763                 :            : 
    1764                 :      54755 : static inline void mm_inc_nr_puds(struct mm_struct *mm)
    1765                 :            : {
    1766                 :      54755 :         if (mm_pud_folded(mm))
    1767                 :            :                 return;
    1768                 :      54755 :         atomic_long_add(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes);
    1769                 :            : }
    1770                 :            : 
    1771                 :      53952 : static inline void mm_dec_nr_puds(struct mm_struct *mm)
    1772                 :            : {
    1773                 :      53952 :         if (mm_pud_folded(mm))
    1774                 :            :                 return;
    1775                 :      53952 :         atomic_long_sub(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes);
    1776                 :            : }
    1777                 :            : #endif
    1778                 :            : 
    1779                 :            : #if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU)
    1780                 :            : static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
    1781                 :            :                                                 unsigned long address)
    1782                 :            : {
    1783                 :            :         return 0;
    1784                 :            : }
    1785                 :            : 
    1786                 :            : static inline void mm_inc_nr_pmds(struct mm_struct *mm) {}
    1787                 :            : static inline void mm_dec_nr_pmds(struct mm_struct *mm) {}
    1788                 :            : 
    1789                 :            : #else
    1790                 :            : int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
    1791                 :            : 
    1792                 :      74540 : static inline void mm_inc_nr_pmds(struct mm_struct *mm)
    1793                 :            : {
    1794                 :      74540 :         if (mm_pmd_folded(mm))
    1795                 :            :                 return;
    1796                 :      74540 :         atomic_long_add(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes);
    1797                 :            : }
    1798                 :            : 
    1799                 :      73594 : static inline void mm_dec_nr_pmds(struct mm_struct *mm)
    1800                 :            : {
    1801                 :      73594 :         if (mm_pmd_folded(mm))
    1802                 :            :                 return;
    1803                 :      73594 :         atomic_long_sub(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes);
    1804                 :            : }
    1805                 :            : #endif
    1806                 :            : 
    1807                 :            : #ifdef CONFIG_MMU
    1808                 :            : static inline void mm_pgtables_bytes_init(struct mm_struct *mm)
    1809                 :            : {
    1810                 :            :         atomic_long_set(&mm->pgtables_bytes, 0);
    1811                 :            : }
    1812                 :            : 
    1813                 :       9283 : static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm)
    1814                 :            : {
    1815                 :       9283 :         return atomic_long_read(&mm->pgtables_bytes);
    1816                 :            : }
    1817                 :            : 
    1818                 :     157775 : static inline void mm_inc_nr_ptes(struct mm_struct *mm)
    1819                 :            : {
    1820                 :     157775 :         atomic_long_add(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes);
    1821                 :     157775 : }
    1822                 :            : 
    1823                 :     154465 : static inline void mm_dec_nr_ptes(struct mm_struct *mm)
    1824                 :            : {
    1825                 :     154465 :         atomic_long_sub(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes);
    1826                 :     154465 : }
    1827                 :            : #else
    1828                 :            : 
    1829                 :            : static inline void mm_pgtables_bytes_init(struct mm_struct *mm) {}
    1830                 :            : static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm)
    1831                 :            : {
    1832                 :            :         return 0;
    1833                 :            : }
    1834                 :            : 
    1835                 :            : static inline void mm_inc_nr_ptes(struct mm_struct *mm) {}
    1836                 :            : static inline void mm_dec_nr_ptes(struct mm_struct *mm) {}
    1837                 :            : #endif
    1838                 :            : 
    1839                 :            : int __pte_alloc(struct mm_struct *mm, pmd_t *pmd);
    1840                 :            : int __pte_alloc_kernel(pmd_t *pmd);
    1841                 :            : 
    1842                 :            : #if defined(CONFIG_MMU)
    1843                 :            : 
    1844                 :            : /*
    1845                 :            :  * The following ifdef needed to get the 5level-fixup.h header to work.
    1846                 :            :  * Remove it when 5level-fixup.h has been removed.
    1847                 :            :  */
    1848                 :            : #ifndef __ARCH_HAS_5LEVEL_HACK
    1849                 :    1918253 : static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
    1850                 :            :                 unsigned long address)
    1851                 :            : {
    1852         [ -  - ]:    1918253 :         return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ?
    1853         [ -  + ]:    1918253 :                 NULL : p4d_offset(pgd, address);
    1854                 :            : }
    1855                 :            : 
    1856                 :    1918227 : static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d,
    1857                 :            :                 unsigned long address)
    1858                 :            : {
    1859   [ +  +  +  - ]:    1918227 :         return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ?
    1860         [ +  + ]:    1918227 :                 NULL : pud_offset(p4d, address);
    1861                 :            : }
    1862                 :            : #endif /* !__ARCH_HAS_5LEVEL_HACK */
    1863                 :            : 
    1864                 :    1918214 : static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
    1865                 :            : {
    1866   [ +  +  +  - ]:    1918214 :         return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
    1867         [ +  + ]:    1918214 :                 NULL: pmd_offset(pud, address);
    1868                 :            : }
    1869                 :            : #endif /* CONFIG_MMU */
    1870                 :            : 
    1871                 :            : #if USE_SPLIT_PTE_PTLOCKS
    1872                 :            : #if ALLOC_SPLIT_PTLOCKS
    1873                 :            : void __init ptlock_cache_init(void);
    1874                 :            : extern bool ptlock_alloc(struct page *page);
    1875                 :            : extern void ptlock_free(struct page *page);
    1876                 :            : 
    1877                 :            : static inline spinlock_t *ptlock_ptr(struct page *page)
    1878                 :            : {
    1879                 :            :         return page->ptl;
    1880                 :            : }
    1881                 :            : #else /* ALLOC_SPLIT_PTLOCKS */
    1882                 :         13 : static inline void ptlock_cache_init(void)
    1883                 :            : {
    1884                 :         13 : }
    1885                 :            : 
    1886                 :          0 : static inline bool ptlock_alloc(struct page *page)
    1887                 :            : {
    1888                 :          0 :         return true;
    1889                 :            : }
    1890                 :            : 
    1891                 :    1388174 : static inline void ptlock_free(struct page *page)
    1892                 :            : {
    1893                 :    1388174 : }
    1894                 :            : 
    1895                 :    3639557 : static inline spinlock_t *ptlock_ptr(struct page *page)
    1896                 :            : {
    1897                 :    3639557 :         return &page->ptl;
    1898                 :            : }
    1899                 :            : #endif /* ALLOC_SPLIT_PTLOCKS */
    1900                 :            : 
    1901                 :    3639557 : static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
    1902                 :            : {
    1903   [ +  -  +  -  :    7279114 :         return ptlock_ptr(pmd_page(*pmd));
          +  +  +  -  +  
          -  +  -  -  -  
          -  -  -  -  -  
          -  +  -  -  -  
          -  -  -  -  -  
          -  +  -  +  -  
             +  -  +  - ]
    1904                 :            : }
    1905                 :            : 
    1906                 :          0 : static inline bool ptlock_init(struct page *page)
    1907                 :            : {
    1908                 :            :         /*
    1909                 :            :          * prep_new_page() initialize page->private (and therefore page->ptl)
    1910                 :            :          * with 0. Make sure nobody took it in use in between.
    1911                 :            :          *
    1912                 :            :          * It can happen if arch try to use slab for page table allocation:
    1913                 :            :          * slab code uses page->slab_cache, which share storage with page->ptl.
    1914                 :            :          */
    1915                 :          0 :         VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page);
    1916                 :          0 :         if (!ptlock_alloc(page))
    1917                 :            :                 return false;
    1918                 :          0 :         spin_lock_init(ptlock_ptr(page));
    1919                 :          0 :         return true;
    1920                 :            : }
    1921                 :            : 
    1922                 :            : #else   /* !USE_SPLIT_PTE_PTLOCKS */
    1923                 :            : /*
    1924                 :            :  * We use mm->page_table_lock to guard all pagetable pages of the mm.
    1925                 :            :  */
    1926                 :            : static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
    1927                 :            : {
    1928                 :            :         return &mm->page_table_lock;
    1929                 :            : }
    1930                 :            : static inline void ptlock_cache_init(void) {}
    1931                 :            : static inline bool ptlock_init(struct page *page) { return true; }
    1932                 :            : static inline void ptlock_free(struct page *page) {}
    1933                 :            : #endif /* USE_SPLIT_PTE_PTLOCKS */
    1934                 :            : 
    1935                 :         13 : static inline void pgtable_init(void)
    1936                 :            : {
    1937                 :         13 :         ptlock_cache_init();
    1938                 :         13 :         pgtable_cache_init();
    1939                 :            : }
    1940                 :            : 
    1941                 :            : static inline bool pgtable_pte_page_ctor(struct page *page)
    1942                 :            : {
    1943                 :            :         if (!ptlock_init(page))
    1944                 :            :                 return false;
    1945                 :            :         __SetPageTable(page);
    1946                 :            :         inc_zone_page_state(page, NR_PAGETABLE);
    1947                 :            :         return true;
    1948                 :            : }
    1949                 :            : 
    1950                 :     907674 : static inline void pgtable_pte_page_dtor(struct page *page)
    1951                 :            : {
    1952                 :     907674 :         ptlock_free(page);
    1953                 :     907674 :         __ClearPageTable(page);
    1954                 :     907674 :         dec_zone_page_state(page, NR_PAGETABLE);
    1955                 :            : }
    1956                 :            : 
    1957                 :            : #define pte_offset_map_lock(mm, pmd, address, ptlp)     \
    1958                 :            : ({                                                      \
    1959                 :            :         spinlock_t *__ptl = pte_lockptr(mm, pmd);       \
    1960                 :            :         pte_t *__pte = pte_offset_map(pmd, address);    \
    1961                 :            :         *(ptlp) = __ptl;                                \
    1962                 :            :         spin_lock(__ptl);                               \
    1963                 :            :         __pte;                                          \
    1964                 :            : })
    1965                 :            : 
    1966                 :            : #define pte_unmap_unlock(pte, ptl)      do {            \
    1967                 :            :         spin_unlock(ptl);                               \
    1968                 :            :         pte_unmap(pte);                                 \
    1969                 :            : } while (0)
    1970                 :            : 
    1971                 :            : #define pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd))
    1972                 :            : 
    1973                 :            : #define pte_alloc_map(mm, pmd, address)                 \
    1974                 :            :         (pte_alloc(mm, pmd) ? NULL : pte_offset_map(pmd, address))
    1975                 :            : 
    1976                 :            : #define pte_alloc_map_lock(mm, pmd, address, ptlp)      \
    1977                 :            :         (pte_alloc(mm, pmd) ?                   \
    1978                 :            :                  NULL : pte_offset_map_lock(mm, pmd, address, ptlp))
    1979                 :            : 
    1980                 :            : #define pte_alloc_kernel(pmd, address)                  \
    1981                 :            :         ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd))? \
    1982                 :            :                 NULL: pte_offset_kernel(pmd, address))
    1983                 :            : 
    1984                 :            : #if USE_SPLIT_PMD_PTLOCKS
    1985                 :            : 
    1986                 :          0 : static struct page *pmd_to_page(pmd_t *pmd)
    1987                 :            : {
    1988                 :          0 :         unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1);
    1989         [ #  # ]:          0 :         return virt_to_page((void *)((unsigned long) pmd & mask));
    1990                 :            : }
    1991                 :            : 
    1992                 :          0 : static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
    1993                 :            : {
    1994   [ #  #  #  #  :          0 :         return ptlock_ptr(pmd_to_page(pmd));
             #  #  #  # ]
    1995                 :            : }
    1996                 :            : 
    1997                 :          0 : static inline bool pgtable_pmd_page_ctor(struct page *page)
    1998                 :            : {
    1999                 :            : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
    2000                 :            :         page->pmd_huge_pte = NULL;
    2001                 :            : #endif
    2002                 :          0 :         return ptlock_init(page);
    2003                 :            : }
    2004                 :            : 
    2005                 :     480500 : static inline void pgtable_pmd_page_dtor(struct page *page)
    2006                 :            : {
    2007                 :            : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
    2008                 :            :         VM_BUG_ON_PAGE(page->pmd_huge_pte, page);
    2009                 :            : #endif
    2010                 :     480500 :         ptlock_free(page);
    2011                 :            : }
    2012                 :            : 
    2013                 :            : #define pmd_huge_pte(mm, pmd) (pmd_to_page(pmd)->pmd_huge_pte)
    2014                 :            : 
    2015                 :            : #else
    2016                 :            : 
    2017                 :            : static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
    2018                 :            : {
    2019                 :            :         return &mm->page_table_lock;
    2020                 :            : }
    2021                 :            : 
    2022                 :            : static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; }
    2023                 :            : static inline void pgtable_pmd_page_dtor(struct page *page) {}
    2024                 :            : 
    2025                 :            : #define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
    2026                 :            : 
    2027                 :            : #endif
    2028                 :            : 
    2029                 :            : static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
    2030                 :            : {
    2031                 :            :         spinlock_t *ptl = pmd_lockptr(mm, pmd);
    2032                 :            :         spin_lock(ptl);
    2033                 :            :         return ptl;
    2034                 :            : }
    2035                 :            : 
    2036                 :            : /*
    2037                 :            :  * No scalability reason to split PUD locks yet, but follow the same pattern
    2038                 :            :  * as the PMD locks to make it easier if we decide to.  The VM should not be
    2039                 :            :  * considered ready to switch to split PUD locks yet; there may be places
    2040                 :            :  * which need to be converted from page_table_lock.
    2041                 :            :  */
    2042                 :      74540 : static inline spinlock_t *pud_lockptr(struct mm_struct *mm, pud_t *pud)
    2043                 :            : {
    2044                 :      74540 :         return &mm->page_table_lock;
    2045                 :            : }
    2046                 :            : 
    2047                 :      74540 : static inline spinlock_t *pud_lock(struct mm_struct *mm, pud_t *pud)
    2048                 :            : {
    2049                 :      74540 :         spinlock_t *ptl = pud_lockptr(mm, pud);
    2050                 :            : 
    2051                 :      74540 :         spin_lock(ptl);
    2052         [ +  - ]:      74540 :         return ptl;
    2053                 :            : }
    2054                 :            : 
    2055                 :            : extern void __init pagecache_init(void);
    2056                 :            : extern void free_area_init(unsigned long * zones_size);
    2057                 :            : extern void __init free_area_init_node(int nid, unsigned long * zones_size,
    2058                 :            :                 unsigned long zone_start_pfn, unsigned long *zholes_size);
    2059                 :            : extern void free_initmem(void);
    2060                 :            : 
    2061                 :            : /*
    2062                 :            :  * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK)
    2063                 :            :  * into the buddy system. The freed pages will be poisoned with pattern
    2064                 :            :  * "poison" if it's within range [0, UCHAR_MAX].
    2065                 :            :  * Return pages freed into the buddy system.
    2066                 :            :  */
    2067                 :            : extern unsigned long free_reserved_area(void *start, void *end,
    2068                 :            :                                         int poison, const char *s);
    2069                 :            : 
    2070                 :            : #ifdef  CONFIG_HIGHMEM
    2071                 :            : /*
    2072                 :            :  * Free a highmem page into the buddy system, adjusting totalhigh_pages
    2073                 :            :  * and totalram_pages.
    2074                 :            :  */
    2075                 :            : extern void free_highmem_page(struct page *page);
    2076                 :            : #endif
    2077                 :            : 
    2078                 :            : extern void adjust_managed_page_count(struct page *page, long count);
    2079                 :            : extern void mem_init_print_info(const char *str);
    2080                 :            : 
    2081                 :            : extern void reserve_bootmem_region(phys_addr_t start, phys_addr_t end);
    2082                 :            : 
    2083                 :            : /* Free the reserved page into the buddy system, so it gets managed. */
    2084                 :      18369 : static inline void __free_reserved_page(struct page *page)
    2085                 :            : {
    2086                 :      18369 :         ClearPageReserved(page);
    2087                 :      18369 :         init_page_count(page);
    2088                 :      18369 :         __free_page(page);
    2089                 :      18369 : }
    2090                 :            : 
    2091                 :      18369 : static inline void free_reserved_page(struct page *page)
    2092                 :            : {
    2093                 :      18369 :         __free_reserved_page(page);
    2094                 :      18369 :         adjust_managed_page_count(page, 1);
    2095                 :            : }
    2096                 :            : 
    2097                 :            : static inline void mark_page_reserved(struct page *page)
    2098                 :            : {
    2099                 :            :         SetPageReserved(page);
    2100                 :            :         adjust_managed_page_count(page, -1);
    2101                 :            : }
    2102                 :            : 
    2103                 :            : /*
    2104                 :            :  * Default method to free all the __init memory into the buddy system.
    2105                 :            :  * The freed pages will be poisoned with pattern "poison" if it's within
    2106                 :            :  * range [0, UCHAR_MAX].
    2107                 :            :  * Return pages freed into the buddy system.
    2108                 :            :  */
    2109                 :          0 : static inline unsigned long free_initmem_default(int poison)
    2110                 :            : {
    2111                 :          0 :         extern char __init_begin[], __init_end[];
    2112                 :            : 
    2113                 :          0 :         return free_reserved_area(&__init_begin, &__init_end,
    2114                 :            :                                   poison, "unused kernel");
    2115                 :            : }
    2116                 :            : 
    2117                 :         13 : static inline unsigned long get_num_physpages(void)
    2118                 :            : {
    2119                 :         13 :         int nid;
    2120                 :         13 :         unsigned long phys_pages = 0;
    2121                 :            : 
    2122         [ +  + ]:         39 :         for_each_online_node(nid)
    2123                 :         13 :                 phys_pages += node_present_pages(nid);
    2124                 :            : 
    2125                 :         13 :         return phys_pages;
    2126                 :            : }
    2127                 :            : 
    2128                 :            : #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
    2129                 :            : /*
    2130                 :            :  * With CONFIG_HAVE_MEMBLOCK_NODE_MAP set, an architecture may initialise its
    2131                 :            :  * zones, allocate the backing mem_map and account for memory holes in a more
    2132                 :            :  * architecture independent manner. This is a substitute for creating the
    2133                 :            :  * zone_sizes[] and zholes_size[] arrays and passing them to
    2134                 :            :  * free_area_init_node()
    2135                 :            :  *
    2136                 :            :  * An architecture is expected to register range of page frames backed by
    2137                 :            :  * physical memory with memblock_add[_node]() before calling
    2138                 :            :  * free_area_init_nodes() passing in the PFN each zone ends at. At a basic
    2139                 :            :  * usage, an architecture is expected to do something like
    2140                 :            :  *
    2141                 :            :  * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn,
    2142                 :            :  *                                                       max_highmem_pfn};
    2143                 :            :  * for_each_valid_physical_page_range()
    2144                 :            :  *      memblock_add_node(base, size, nid)
    2145                 :            :  * free_area_init_nodes(max_zone_pfns);
    2146                 :            :  *
    2147                 :            :  * free_bootmem_with_active_regions() calls free_bootmem_node() for each
    2148                 :            :  * registered physical page range.  Similarly
    2149                 :            :  * sparse_memory_present_with_active_regions() calls memory_present() for
    2150                 :            :  * each range when SPARSEMEM is enabled.
    2151                 :            :  *
    2152                 :            :  * See mm/page_alloc.c for more information on each function exposed by
    2153                 :            :  * CONFIG_HAVE_MEMBLOCK_NODE_MAP.
    2154                 :            :  */
    2155                 :            : extern void free_area_init_nodes(unsigned long *max_zone_pfn);
    2156                 :            : unsigned long node_map_pfn_alignment(void);
    2157                 :            : unsigned long __absent_pages_in_range(int nid, unsigned long start_pfn,
    2158                 :            :                                                 unsigned long end_pfn);
    2159                 :            : extern unsigned long absent_pages_in_range(unsigned long start_pfn,
    2160                 :            :                                                 unsigned long end_pfn);
    2161                 :            : extern void get_pfn_range_for_nid(unsigned int nid,
    2162                 :            :                         unsigned long *start_pfn, unsigned long *end_pfn);
    2163                 :            : extern unsigned long find_min_pfn_with_active_regions(void);
    2164                 :            : extern void free_bootmem_with_active_regions(int nid,
    2165                 :            :                                                 unsigned long max_low_pfn);
    2166                 :            : extern void sparse_memory_present_with_active_regions(int nid);
    2167                 :            : 
    2168                 :            : #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
    2169                 :            : 
    2170                 :            : #if !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) && \
    2171                 :            :     !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID)
    2172                 :            : static inline int __early_pfn_to_nid(unsigned long pfn,
    2173                 :            :                                         struct mminit_pfnnid_cache *state)
    2174                 :            : {
    2175                 :            :         return 0;
    2176                 :            : }
    2177                 :            : #else
    2178                 :            : /* please see mm/page_alloc.c */
    2179                 :            : extern int __meminit early_pfn_to_nid(unsigned long pfn);
    2180                 :            : /* there is a per-arch backend function. */
    2181                 :            : extern int __meminit __early_pfn_to_nid(unsigned long pfn,
    2182                 :            :                                         struct mminit_pfnnid_cache *state);
    2183                 :            : #endif
    2184                 :            : 
    2185                 :            : extern void set_dma_reserve(unsigned long new_dma_reserve);
    2186                 :            : extern void memmap_init_zone(unsigned long, int, unsigned long, unsigned long,
    2187                 :            :                 enum memmap_context, struct vmem_altmap *);
    2188                 :            : extern void setup_per_zone_wmarks(void);
    2189                 :            : extern int __meminit init_per_zone_wmark_min(void);
    2190                 :            : extern void mem_init(void);
    2191                 :            : extern void __init mmap_init(void);
    2192                 :            : extern void show_mem(unsigned int flags, nodemask_t *nodemask);
    2193                 :            : extern long si_mem_available(void);
    2194                 :            : extern void si_meminfo(struct sysinfo * val);
    2195                 :            : extern void si_meminfo_node(struct sysinfo *val, int nid);
    2196                 :            : #ifdef __HAVE_ARCH_RESERVED_KERNEL_PAGES
    2197                 :            : extern unsigned long arch_reserved_kernel_pages(void);
    2198                 :            : #endif
    2199                 :            : 
    2200                 :            : extern __printf(3, 4)
    2201                 :            : void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...);
    2202                 :            : 
    2203                 :            : extern void setup_per_cpu_pageset(void);
    2204                 :            : 
    2205                 :            : /* page_alloc.c */
    2206                 :            : extern int min_free_kbytes;
    2207                 :            : extern int watermark_boost_factor;
    2208                 :            : extern int watermark_scale_factor;
    2209                 :            : 
    2210                 :            : /* nommu.c */
    2211                 :            : extern atomic_long_t mmap_pages_allocated;
    2212                 :            : extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
    2213                 :            : 
    2214                 :            : /* interval_tree.c */
    2215                 :            : void vma_interval_tree_insert(struct vm_area_struct *node,
    2216                 :            :                               struct rb_root_cached *root);
    2217                 :            : void vma_interval_tree_insert_after(struct vm_area_struct *node,
    2218                 :            :                                     struct vm_area_struct *prev,
    2219                 :            :                                     struct rb_root_cached *root);
    2220                 :            : void vma_interval_tree_remove(struct vm_area_struct *node,
    2221                 :            :                               struct rb_root_cached *root);
    2222                 :            : struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root_cached *root,
    2223                 :            :                                 unsigned long start, unsigned long last);
    2224                 :            : struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node,
    2225                 :            :                                 unsigned long start, unsigned long last);
    2226                 :            : 
    2227                 :            : #define vma_interval_tree_foreach(vma, root, start, last)               \
    2228                 :            :         for (vma = vma_interval_tree_iter_first(root, start, last);     \
    2229                 :            :              vma; vma = vma_interval_tree_iter_next(vma, start, last))
    2230                 :            : 
    2231                 :            : void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
    2232                 :            :                                    struct rb_root_cached *root);
    2233                 :            : void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
    2234                 :            :                                    struct rb_root_cached *root);
    2235                 :            : struct anon_vma_chain *
    2236                 :            : anon_vma_interval_tree_iter_first(struct rb_root_cached *root,
    2237                 :            :                                   unsigned long start, unsigned long last);
    2238                 :            : struct anon_vma_chain *anon_vma_interval_tree_iter_next(
    2239                 :            :         struct anon_vma_chain *node, unsigned long start, unsigned long last);
    2240                 :            : #ifdef CONFIG_DEBUG_VM_RB
    2241                 :            : void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
    2242                 :            : #endif
    2243                 :            : 
    2244                 :            : #define anon_vma_interval_tree_foreach(avc, root, start, last)           \
    2245                 :            :         for (avc = anon_vma_interval_tree_iter_first(root, start, last); \
    2246                 :            :              avc; avc = anon_vma_interval_tree_iter_next(avc, start, last))
    2247                 :            : 
    2248                 :            : /* mmap.c */
    2249                 :            : extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
    2250                 :            : extern int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
    2251                 :            :         unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert,
    2252                 :            :         struct vm_area_struct *expand);
    2253                 :     261980 : static inline int vma_adjust(struct vm_area_struct *vma, unsigned long start,
    2254                 :            :         unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
    2255                 :            : {
    2256                 :     261980 :         return __vma_adjust(vma, start, end, pgoff, insert, NULL);
    2257                 :            : }
    2258                 :            : extern struct vm_area_struct *vma_merge(struct mm_struct *,
    2259                 :            :         struct vm_area_struct *prev, unsigned long addr, unsigned long end,
    2260                 :            :         unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
    2261                 :            :         struct mempolicy *, struct vm_userfaultfd_ctx);
    2262                 :            : extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
    2263                 :            : extern int __split_vma(struct mm_struct *, struct vm_area_struct *,
    2264                 :            :         unsigned long addr, int new_below);
    2265                 :            : extern int split_vma(struct mm_struct *, struct vm_area_struct *,
    2266                 :            :         unsigned long addr, int new_below);
    2267                 :            : extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
    2268                 :            : extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
    2269                 :            :         struct rb_node **, struct rb_node *);
    2270                 :            : extern void unlink_file_vma(struct vm_area_struct *);
    2271                 :            : extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
    2272                 :            :         unsigned long addr, unsigned long len, pgoff_t pgoff,
    2273                 :            :         bool *need_rmap_locks);
    2274                 :            : extern void exit_mmap(struct mm_struct *);
    2275                 :            : 
    2276                 :      12381 : static inline int check_data_rlimit(unsigned long rlim,
    2277                 :            :                                     unsigned long new,
    2278                 :            :                                     unsigned long start,
    2279                 :            :                                     unsigned long end_data,
    2280                 :            :                                     unsigned long start_data)
    2281                 :            : {
    2282         [ -  + ]:      12381 :         if (rlim < RLIM_INFINITY) {
    2283         [ #  # ]:          0 :                 if (((new - start) + (end_data - start_data)) > rlim)
    2284                 :          0 :                         return -ENOSPC;
    2285                 :            :         }
    2286                 :            : 
    2287                 :            :         return 0;
    2288                 :            : }
    2289                 :            : 
    2290                 :            : extern int mm_take_all_locks(struct mm_struct *mm);
    2291                 :            : extern void mm_drop_all_locks(struct mm_struct *mm);
    2292                 :            : 
    2293                 :            : extern void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
    2294                 :            : extern struct file *get_mm_exe_file(struct mm_struct *mm);
    2295                 :            : extern struct file *get_task_exe_file(struct task_struct *task);
    2296                 :            : 
    2297                 :            : extern bool may_expand_vm(struct mm_struct *, vm_flags_t, unsigned long npages);
    2298                 :            : extern void vm_stat_account(struct mm_struct *, vm_flags_t, long npages);
    2299                 :            : 
    2300                 :            : extern bool vma_is_special_mapping(const struct vm_area_struct *vma,
    2301                 :            :                                    const struct vm_special_mapping *sm);
    2302                 :            : extern struct vm_area_struct *_install_special_mapping(struct mm_struct *mm,
    2303                 :            :                                    unsigned long addr, unsigned long len,
    2304                 :            :                                    unsigned long flags,
    2305                 :            :                                    const struct vm_special_mapping *spec);
    2306                 :            : /* This is an obsolete alternative to _install_special_mapping. */
    2307                 :            : extern int install_special_mapping(struct mm_struct *mm,
    2308                 :            :                                    unsigned long addr, unsigned long len,
    2309                 :            :                                    unsigned long flags, struct page **pages);
    2310                 :            : 
    2311                 :            : unsigned long randomize_stack_top(unsigned long stack_top);
    2312                 :            : 
    2313                 :            : extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
    2314                 :            : 
    2315                 :            : extern unsigned long mmap_region(struct file *file, unsigned long addr,
    2316                 :            :         unsigned long len, vm_flags_t vm_flags, unsigned long pgoff,
    2317                 :            :         struct list_head *uf);
    2318                 :            : extern unsigned long do_mmap(struct file *file, unsigned long addr,
    2319                 :            :         unsigned long len, unsigned long prot, unsigned long flags,
    2320                 :            :         vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate,
    2321                 :            :         struct list_head *uf);
    2322                 :            : extern int __do_munmap(struct mm_struct *, unsigned long, size_t,
    2323                 :            :                        struct list_head *uf, bool downgrade);
    2324                 :            : extern int do_munmap(struct mm_struct *, unsigned long, size_t,
    2325                 :            :                      struct list_head *uf);
    2326                 :            : extern int do_madvise(unsigned long start, size_t len_in, int behavior);
    2327                 :            : 
    2328                 :            : static inline unsigned long
    2329                 :     351677 : do_mmap_pgoff(struct file *file, unsigned long addr,
    2330                 :            :         unsigned long len, unsigned long prot, unsigned long flags,
    2331                 :            :         unsigned long pgoff, unsigned long *populate,
    2332                 :            :         struct list_head *uf)
    2333                 :            : {
    2334                 :     351677 :         return do_mmap(file, addr, len, prot, flags, 0, pgoff, populate, uf);
    2335                 :            : }
    2336                 :            : 
    2337                 :            : #ifdef CONFIG_MMU
    2338                 :            : extern int __mm_populate(unsigned long addr, unsigned long len,
    2339                 :            :                          int ignore_errors);
    2340                 :          0 : static inline void mm_populate(unsigned long addr, unsigned long len)
    2341                 :            : {
    2342                 :            :         /* Ignore errors */
    2343                 :          0 :         (void) __mm_populate(addr, len, 1);
    2344                 :          0 : }
    2345                 :            : #else
    2346                 :            : static inline void mm_populate(unsigned long addr, unsigned long len) {}
    2347                 :            : #endif
    2348                 :            : 
    2349                 :            : /* These take the mm semaphore themselves */
    2350                 :            : extern int __must_check vm_brk(unsigned long, unsigned long);
    2351                 :            : extern int __must_check vm_brk_flags(unsigned long, unsigned long, unsigned long);
    2352                 :            : extern int vm_munmap(unsigned long, size_t);
    2353                 :            : extern unsigned long __must_check vm_mmap(struct file *, unsigned long,
    2354                 :            :         unsigned long, unsigned long,
    2355                 :            :         unsigned long, unsigned long);
    2356                 :            : 
    2357                 :            : struct vm_unmapped_area_info {
    2358                 :            : #define VM_UNMAPPED_AREA_TOPDOWN 1
    2359                 :            :         unsigned long flags;
    2360                 :            :         unsigned long length;
    2361                 :            :         unsigned long low_limit;
    2362                 :            :         unsigned long high_limit;
    2363                 :            :         unsigned long align_mask;
    2364                 :            :         unsigned long align_offset;
    2365                 :            : };
    2366                 :            : 
    2367                 :            : extern unsigned long unmapped_area(struct vm_unmapped_area_info *info);
    2368                 :            : extern unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info);
    2369                 :            : 
    2370                 :            : /*
    2371                 :            :  * Search for an unmapped address range.
    2372                 :            :  *
    2373                 :            :  * We are looking for a range that:
    2374                 :            :  * - does not intersect with any VMA;
    2375                 :            :  * - is contained within the [low_limit, high_limit) interval;
    2376                 :            :  * - is at least the desired size.
    2377                 :            :  * - satisfies (begin_addr & align_mask) == (align_offset & align_mask)
    2378                 :            :  */
    2379                 :            : static inline unsigned long
    2380                 :     767431 : vm_unmapped_area(struct vm_unmapped_area_info *info)
    2381                 :            : {
    2382         [ +  - ]:     767431 :         if (info->flags & VM_UNMAPPED_AREA_TOPDOWN)
    2383                 :     767431 :                 return unmapped_area_topdown(info);
    2384                 :            :         else
    2385                 :          0 :                 return unmapped_area(info);
    2386                 :            : }
    2387                 :            : 
    2388                 :            : /* truncate.c */
    2389                 :            : extern void truncate_inode_pages(struct address_space *, loff_t);
    2390                 :            : extern void truncate_inode_pages_range(struct address_space *,
    2391                 :            :                                        loff_t lstart, loff_t lend);
    2392                 :            : extern void truncate_inode_pages_final(struct address_space *);
    2393                 :            : 
    2394                 :            : /* generic vm_area_ops exported for stackable file systems */
    2395                 :            : extern vm_fault_t filemap_fault(struct vm_fault *vmf);
    2396                 :            : extern void filemap_map_pages(struct vm_fault *vmf,
    2397                 :            :                 pgoff_t start_pgoff, pgoff_t end_pgoff);
    2398                 :            : extern vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf);
    2399                 :            : 
    2400                 :            : /* mm/page-writeback.c */
    2401                 :            : int __must_check write_one_page(struct page *page);
    2402                 :            : void task_dirty_inc(struct task_struct *tsk);
    2403                 :            : 
    2404                 :            : /* readahead.c */
    2405                 :            : #define VM_READAHEAD_PAGES      (SZ_128K / PAGE_SIZE)
    2406                 :            : 
    2407                 :            : int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
    2408                 :            :                         pgoff_t offset, unsigned long nr_to_read);
    2409                 :            : 
    2410                 :            : void page_cache_sync_readahead(struct address_space *mapping,
    2411                 :            :                                struct file_ra_state *ra,
    2412                 :            :                                struct file *filp,
    2413                 :            :                                pgoff_t offset,
    2414                 :            :                                unsigned long size);
    2415                 :            : 
    2416                 :            : void page_cache_async_readahead(struct address_space *mapping,
    2417                 :            :                                 struct file_ra_state *ra,
    2418                 :            :                                 struct file *filp,
    2419                 :            :                                 struct page *pg,
    2420                 :            :                                 pgoff_t offset,
    2421                 :            :                                 unsigned long size);
    2422                 :            : 
    2423                 :            : extern unsigned long stack_guard_gap;
    2424                 :            : /* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */
    2425                 :            : extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
    2426                 :            : 
    2427                 :            : /* CONFIG_STACK_GROWSUP still needs to to grow downwards at some places */
    2428                 :            : extern int expand_downwards(struct vm_area_struct *vma,
    2429                 :            :                 unsigned long address);
    2430                 :            : #if VM_GROWSUP
    2431                 :            : extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
    2432                 :            : #else
    2433                 :            :   #define expand_upwards(vma, address) (0)
    2434                 :            : #endif
    2435                 :            : 
    2436                 :            : /* Look up the first VMA which satisfies  addr < vm_end,  NULL if none. */
    2437                 :            : extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
    2438                 :            : extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
    2439                 :            :                                              struct vm_area_struct **pprev);
    2440                 :            : 
    2441                 :            : /* Look up the first VMA which intersects the interval start_addr..end_addr-1,
    2442                 :            :    NULL if none.  Assume start_addr < end_addr. */
    2443                 :          0 : static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
    2444                 :            : {
    2445                 :          0 :         struct vm_area_struct * vma = find_vma(mm,start_addr);
    2446                 :            : 
    2447   [ #  #  #  # ]:          0 :         if (vma && end_addr <= vma->vm_start)
    2448                 :            :                 vma = NULL;
    2449         [ #  # ]:          0 :         return vma;
    2450                 :            : }
    2451                 :            : 
    2452                 :    6523395 : static inline unsigned long vm_start_gap(struct vm_area_struct *vma)
    2453                 :            : {
    2454                 :    6523395 :         unsigned long vm_start = vma->vm_start;
    2455                 :            : 
    2456   [ +  +  -  -  :    6523395 :         if (vma->vm_flags & VM_GROWSDOWN) {
          -  -  +  +  -  
                      + ]
    2457                 :     446428 :                 vm_start -= stack_guard_gap;
    2458   [ -  +  -  -  :     446428 :                 if (vm_start > vma->vm_start)
          -  -  -  +  -  
                      - ]
    2459                 :          0 :                         vm_start = 0;
    2460                 :            :         }
    2461   [ +  +  -  -  :    6523395 :         return vm_start;
             +  +  -  + ]
    2462                 :            : }
    2463                 :            : 
    2464                 :    7677275 : static inline unsigned long vm_end_gap(struct vm_area_struct *vma)
    2465                 :            : {
    2466                 :    7677275 :         unsigned long vm_end = vma->vm_end;
    2467                 :            : 
    2468                 :    7677275 :         if (vma->vm_flags & VM_GROWSUP) {
    2469                 :            :                 vm_end += stack_guard_gap;
    2470                 :            :                 if (vm_end < vma->vm_end)
    2471                 :            :                         vm_end = -PAGE_SIZE;
    2472                 :            :         }
    2473   [ -  -  +  + ]:    7677275 :         return vm_end;
    2474                 :            : }
    2475                 :            : 
    2476                 :    6973827 : static inline unsigned long vma_pages(struct vm_area_struct *vma)
    2477                 :            : {
    2478   [ +  +  +  -  :    6973827 :         return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
          +  +  +  +  -  
          -  -  -  +  +  
                   +  + ]
    2479                 :            : }
    2480                 :            : 
    2481                 :            : /* Look up the first VMA which exactly match the interval vm_start ... vm_end */
    2482                 :          0 : static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
    2483                 :            :                                 unsigned long vm_start, unsigned long vm_end)
    2484                 :            : {
    2485                 :          0 :         struct vm_area_struct *vma = find_vma(mm, vm_start);
    2486                 :            : 
    2487   [ #  #  #  #  :          0 :         if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end))
          #  #  #  #  #  
          #  #  #  #  #  
             #  #  #  # ]
    2488                 :          0 :                 vma = NULL;
    2489                 :            : 
    2490   [ #  #  #  # ]:          0 :         return vma;
    2491                 :            : }
    2492                 :            : 
    2493                 :          0 : static inline bool range_in_vma(struct vm_area_struct *vma,
    2494                 :            :                                 unsigned long start, unsigned long end)
    2495                 :            : {
    2496   [ #  #  #  #  :          0 :         return (vma && vma->vm_start <= start && end <= vma->vm_end);
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    2497                 :            : }
    2498                 :            : 
    2499                 :            : #ifdef CONFIG_MMU
    2500                 :            : pgprot_t vm_get_page_prot(unsigned long vm_flags);
    2501                 :            : void vma_set_page_prot(struct vm_area_struct *vma);
    2502                 :            : #else
    2503                 :            : static inline pgprot_t vm_get_page_prot(unsigned long vm_flags)
    2504                 :            : {
    2505                 :            :         return __pgprot(0);
    2506                 :            : }
    2507                 :            : static inline void vma_set_page_prot(struct vm_area_struct *vma)
    2508                 :            : {
    2509                 :            :         vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
    2510                 :            : }
    2511                 :            : #endif
    2512                 :            : 
    2513                 :            : #ifdef CONFIG_NUMA_BALANCING
    2514                 :            : unsigned long change_prot_numa(struct vm_area_struct *vma,
    2515                 :            :                         unsigned long start, unsigned long end);
    2516                 :            : #endif
    2517                 :            : 
    2518                 :            : struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
    2519                 :            : int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
    2520                 :            :                         unsigned long pfn, unsigned long size, pgprot_t);
    2521                 :            : int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
    2522                 :            : int vm_map_pages(struct vm_area_struct *vma, struct page **pages,
    2523                 :            :                                 unsigned long num);
    2524                 :            : int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages,
    2525                 :            :                                 unsigned long num);
    2526                 :            : vm_fault_t vmf_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
    2527                 :            :                         unsigned long pfn);
    2528                 :            : vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
    2529                 :            :                         unsigned long pfn, pgprot_t pgprot);
    2530                 :            : vm_fault_t vmf_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
    2531                 :            :                         pfn_t pfn);
    2532                 :            : vm_fault_t vmf_insert_mixed_prot(struct vm_area_struct *vma, unsigned long addr,
    2533                 :            :                         pfn_t pfn, pgprot_t pgprot);
    2534                 :            : vm_fault_t vmf_insert_mixed_mkwrite(struct vm_area_struct *vma,
    2535                 :            :                 unsigned long addr, pfn_t pfn);
    2536                 :            : int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len);
    2537                 :            : 
    2538                 :          0 : static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma,
    2539                 :            :                                 unsigned long addr, struct page *page)
    2540                 :            : {
    2541                 :          0 :         int err = vm_insert_page(vma, addr, page);
    2542                 :            : 
    2543         [ #  # ]:          0 :         if (err == -ENOMEM)
    2544                 :            :                 return VM_FAULT_OOM;
    2545         [ #  # ]:          0 :         if (err < 0 && err != -EBUSY)
    2546                 :          0 :                 return VM_FAULT_SIGBUS;
    2547                 :            : 
    2548                 :            :         return VM_FAULT_NOPAGE;
    2549                 :            : }
    2550                 :            : 
    2551                 :          0 : static inline vm_fault_t vmf_error(int err)
    2552                 :            : {
    2553   [ #  #  #  # ]:          0 :         if (err == -ENOMEM)
    2554                 :          0 :                 return VM_FAULT_OOM;
    2555                 :            :         return VM_FAULT_SIGBUS;
    2556                 :            : }
    2557                 :            : 
    2558                 :            : struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
    2559                 :            :                          unsigned int foll_flags);
    2560                 :            : 
    2561                 :            : #define FOLL_WRITE      0x01    /* check pte is writable */
    2562                 :            : #define FOLL_TOUCH      0x02    /* mark page accessed */
    2563                 :            : #define FOLL_GET        0x04    /* do get_page on page */
    2564                 :            : #define FOLL_DUMP       0x08    /* give error on hole if it would be zero */
    2565                 :            : #define FOLL_FORCE      0x10    /* get_user_pages read/write w/o permission */
    2566                 :            : #define FOLL_NOWAIT     0x20    /* if a disk transfer is needed, start the IO
    2567                 :            :                                  * and return without waiting upon it */
    2568                 :            : #define FOLL_POPULATE   0x40    /* fault in page */
    2569                 :            : #define FOLL_SPLIT      0x80    /* don't return transhuge pages, split them */
    2570                 :            : #define FOLL_HWPOISON   0x100   /* check page is hwpoisoned */
    2571                 :            : #define FOLL_NUMA       0x200   /* force NUMA hinting page fault */
    2572                 :            : #define FOLL_MIGRATION  0x400   /* wait for page to replace migration entry */
    2573                 :            : #define FOLL_TRIED      0x800   /* a retry, previous pass started an IO */
    2574                 :            : #define FOLL_MLOCK      0x1000  /* lock present pages */
    2575                 :            : #define FOLL_REMOTE     0x2000  /* we are working on non-current tsk/mm */
    2576                 :            : #define FOLL_COW        0x4000  /* internal GUP flag */
    2577                 :            : #define FOLL_ANON       0x8000  /* don't do file mappings */
    2578                 :            : #define FOLL_LONGTERM   0x10000 /* mapping lifetime is indefinite: see below */
    2579                 :            : #define FOLL_SPLIT_PMD  0x20000 /* split huge pmd before returning */
    2580                 :            : #define FOLL_PIN        0x40000 /* pages must be released via unpin_user_page */
    2581                 :            : 
    2582                 :            : /*
    2583                 :            :  * FOLL_PIN and FOLL_LONGTERM may be used in various combinations with each
    2584                 :            :  * other. Here is what they mean, and how to use them:
    2585                 :            :  *
    2586                 :            :  * FOLL_LONGTERM indicates that the page will be held for an indefinite time
    2587                 :            :  * period _often_ under userspace control.  This is in contrast to
    2588                 :            :  * iov_iter_get_pages(), whose usages are transient.
    2589                 :            :  *
    2590                 :            :  * FIXME: For pages which are part of a filesystem, mappings are subject to the
    2591                 :            :  * lifetime enforced by the filesystem and we need guarantees that longterm
    2592                 :            :  * users like RDMA and V4L2 only establish mappings which coordinate usage with
    2593                 :            :  * the filesystem.  Ideas for this coordination include revoking the longterm
    2594                 :            :  * pin, delaying writeback, bounce buffer page writeback, etc.  As FS DAX was
    2595                 :            :  * added after the problem with filesystems was found FS DAX VMAs are
    2596                 :            :  * specifically failed.  Filesystem pages are still subject to bugs and use of
    2597                 :            :  * FOLL_LONGTERM should be avoided on those pages.
    2598                 :            :  *
    2599                 :            :  * FIXME: Also NOTE that FOLL_LONGTERM is not supported in every GUP call.
    2600                 :            :  * Currently only get_user_pages() and get_user_pages_fast() support this flag
    2601                 :            :  * and calls to get_user_pages_[un]locked are specifically not allowed.  This
    2602                 :            :  * is due to an incompatibility with the FS DAX check and
    2603                 :            :  * FAULT_FLAG_ALLOW_RETRY.
    2604                 :            :  *
    2605                 :            :  * In the CMA case: long term pins in a CMA region would unnecessarily fragment
    2606                 :            :  * that region.  And so, CMA attempts to migrate the page before pinning, when
    2607                 :            :  * FOLL_LONGTERM is specified.
    2608                 :            :  *
    2609                 :            :  * FOLL_PIN indicates that a special kind of tracking (not just page->_refcount,
    2610                 :            :  * but an additional pin counting system) will be invoked. This is intended for
    2611                 :            :  * anything that gets a page reference and then touches page data (for example,
    2612                 :            :  * Direct IO). This lets the filesystem know that some non-file-system entity is
    2613                 :            :  * potentially changing the pages' data. In contrast to FOLL_GET (whose pages
    2614                 :            :  * are released via put_page()), FOLL_PIN pages must be released, ultimately, by
    2615                 :            :  * a call to unpin_user_page().
    2616                 :            :  *
    2617                 :            :  * FOLL_PIN is similar to FOLL_GET: both of these pin pages. They use different
    2618                 :            :  * and separate refcounting mechanisms, however, and that means that each has
    2619                 :            :  * its own acquire and release mechanisms:
    2620                 :            :  *
    2621                 :            :  *     FOLL_GET: get_user_pages*() to acquire, and put_page() to release.
    2622                 :            :  *
    2623                 :            :  *     FOLL_PIN: pin_user_pages*() to acquire, and unpin_user_pages to release.
    2624                 :            :  *
    2625                 :            :  * FOLL_PIN and FOLL_GET are mutually exclusive for a given function call.
    2626                 :            :  * (The underlying pages may experience both FOLL_GET-based and FOLL_PIN-based
    2627                 :            :  * calls applied to them, and that's perfectly OK. This is a constraint on the
    2628                 :            :  * callers, not on the pages.)
    2629                 :            :  *
    2630                 :            :  * FOLL_PIN should be set internally by the pin_user_pages*() APIs, never
    2631                 :            :  * directly by the caller. That's in order to help avoid mismatches when
    2632                 :            :  * releasing pages: get_user_pages*() pages must be released via put_page(),
    2633                 :            :  * while pin_user_pages*() pages must be released via unpin_user_page().
    2634                 :            :  *
    2635                 :            :  * Please see Documentation/vm/pin_user_pages.rst for more information.
    2636                 :            :  */
    2637                 :            : 
    2638                 :          0 : static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags)
    2639                 :            : {
    2640   [ #  #  #  # ]:          0 :         if (vm_fault & VM_FAULT_OOM)
    2641                 :            :                 return -ENOMEM;
    2642   [ #  #  #  # ]:          0 :         if (vm_fault & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE))
    2643         [ #  # ]:          0 :                 return (foll_flags & FOLL_HWPOISON) ? -EHWPOISON : -EFAULT;
    2644   [ #  #  #  # ]:          0 :         if (vm_fault & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV))
    2645                 :          0 :                 return -EFAULT;
    2646                 :            :         return 0;
    2647                 :            : }
    2648                 :            : 
    2649                 :            : typedef int (*pte_fn_t)(pte_t *pte, unsigned long addr, void *data);
    2650                 :            : extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
    2651                 :            :                                unsigned long size, pte_fn_t fn, void *data);
    2652                 :            : extern int apply_to_existing_page_range(struct mm_struct *mm,
    2653                 :            :                                    unsigned long address, unsigned long size,
    2654                 :            :                                    pte_fn_t fn, void *data);
    2655                 :            : 
    2656                 :            : #ifdef CONFIG_PAGE_POISONING
    2657                 :            : extern bool page_poisoning_enabled(void);
    2658                 :            : extern void kernel_poison_pages(struct page *page, int numpages, int enable);
    2659                 :            : #else
    2660                 :            : static inline bool page_poisoning_enabled(void) { return false; }
    2661                 :    1632038 : static inline void kernel_poison_pages(struct page *page, int numpages,
    2662         [ +  - ]:    1632038 :                                         int enable) { }
    2663                 :            : #endif
    2664                 :            : 
    2665                 :            : #ifdef CONFIG_INIT_ON_ALLOC_DEFAULT_ON
    2666                 :            : DECLARE_STATIC_KEY_TRUE(init_on_alloc);
    2667                 :            : #else
    2668                 :            : DECLARE_STATIC_KEY_FALSE(init_on_alloc);
    2669                 :            : #endif
    2670                 :    1637108 : static inline bool want_init_on_alloc(gfp_t flags)
    2671                 :            : {
    2672   [ +  -  +  - ]:    3274216 :         if (static_branch_unlikely(&init_on_alloc) &&
    2673                 :            :             !page_poisoning_enabled())
    2674                 :            :                 return true;
    2675         [ +  + ]:    1637108 :         return flags & __GFP_ZERO;
    2676                 :            : }
    2677                 :            : 
    2678                 :            : #ifdef CONFIG_INIT_ON_FREE_DEFAULT_ON
    2679                 :            : DECLARE_STATIC_KEY_TRUE(init_on_free);
    2680                 :            : #else
    2681                 :            : DECLARE_STATIC_KEY_FALSE(init_on_free);
    2682                 :            : #endif
    2683                 :    2931207 : static inline bool want_init_on_free(void)
    2684                 :            : {
    2685   [ +  -  +  -  :    7641046 :         return static_branch_unlikely(&init_on_free) &&
          +  -  +  -  +  
          -  -  +  +  -  
                   +  - ]
    2686                 :            :                !page_poisoning_enabled();
    2687                 :            : }
    2688                 :            : 
    2689                 :            : #ifdef CONFIG_DEBUG_PAGEALLOC
    2690                 :            : extern void init_debug_pagealloc(void);
    2691                 :            : #else
    2692                 :         13 : static inline void init_debug_pagealloc(void) {}
    2693                 :            : #endif
    2694                 :            : extern bool _debug_pagealloc_enabled_early;
    2695                 :            : DECLARE_STATIC_KEY_FALSE(_debug_pagealloc_enabled);
    2696                 :            : 
    2697                 :     174875 : static inline bool debug_pagealloc_enabled(void)
    2698                 :            : {
    2699                 :     174875 :         return IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) &&
    2700                 :            :                 _debug_pagealloc_enabled_early;
    2701                 :            : }
    2702                 :            : 
    2703                 :            : /*
    2704                 :            :  * For use in fast paths after init_debug_pagealloc() has run, or when a
    2705                 :            :  * false negative result is not harmful when called too early.
    2706                 :            :  */
    2707                 :   14867934 : static inline bool debug_pagealloc_enabled_static(void)
    2708                 :            : {
    2709                 :   14867934 :         if (!IS_ENABLED(CONFIG_DEBUG_PAGEALLOC))
    2710                 :    5531979 :                 return false;
    2711                 :            : 
    2712                 :            :         return static_branch_unlikely(&_debug_pagealloc_enabled);
    2713                 :            : }
    2714                 :            : 
    2715                 :            : #if defined(CONFIG_DEBUG_PAGEALLOC) || defined(CONFIG_ARCH_HAS_SET_DIRECT_MAP)
    2716                 :            : extern void __kernel_map_pages(struct page *page, int numpages, int enable);
    2717                 :            : 
    2718                 :            : /*
    2719                 :            :  * When called in DEBUG_PAGEALLOC context, the call should most likely be
    2720                 :            :  * guarded by debug_pagealloc_enabled() or debug_pagealloc_enabled_static()
    2721                 :            :  */
    2722                 :            : static inline void
    2723                 :          0 : kernel_map_pages(struct page *page, int numpages, int enable)
    2724                 :            : {
    2725                 :          0 :         __kernel_map_pages(page, numpages, enable);
    2726                 :          0 : }
    2727                 :            : #ifdef CONFIG_HIBERNATION
    2728                 :            : extern bool kernel_page_present(struct page *page);
    2729                 :            : #endif  /* CONFIG_HIBERNATION */
    2730                 :            : #else   /* CONFIG_DEBUG_PAGEALLOC || CONFIG_ARCH_HAS_SET_DIRECT_MAP */
    2731                 :            : static inline void
    2732                 :            : kernel_map_pages(struct page *page, int numpages, int enable) {}
    2733                 :            : #ifdef CONFIG_HIBERNATION
    2734                 :            : static inline bool kernel_page_present(struct page *page) { return true; }
    2735                 :            : #endif  /* CONFIG_HIBERNATION */
    2736                 :            : #endif  /* CONFIG_DEBUG_PAGEALLOC || CONFIG_ARCH_HAS_SET_DIRECT_MAP */
    2737                 :            : 
    2738                 :            : #ifdef __HAVE_ARCH_GATE_AREA
    2739                 :            : extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm);
    2740                 :            : extern int in_gate_area_no_mm(unsigned long addr);
    2741                 :            : extern int in_gate_area(struct mm_struct *mm, unsigned long addr);
    2742                 :            : #else
    2743                 :            : static inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
    2744                 :            : {
    2745                 :            :         return NULL;
    2746                 :            : }
    2747                 :            : static inline int in_gate_area_no_mm(unsigned long addr) { return 0; }
    2748                 :            : static inline int in_gate_area(struct mm_struct *mm, unsigned long addr)
    2749                 :            : {
    2750                 :            :         return 0;
    2751                 :            : }
    2752                 :            : #endif  /* __HAVE_ARCH_GATE_AREA */
    2753                 :            : 
    2754                 :            : extern bool process_shares_mm(struct task_struct *p, struct mm_struct *mm);
    2755                 :            : 
    2756                 :            : #ifdef CONFIG_SYSCTL
    2757                 :            : extern int sysctl_drop_caches;
    2758                 :            : int drop_caches_sysctl_handler(struct ctl_table *, int,
    2759                 :            :                                         void __user *, size_t *, loff_t *);
    2760                 :            : #endif
    2761                 :            : 
    2762                 :            : void drop_slab(void);
    2763                 :            : void drop_slab_node(int nid);
    2764                 :            : 
    2765                 :            : #ifndef CONFIG_MMU
    2766                 :            : #define randomize_va_space 0
    2767                 :            : #else
    2768                 :            : extern int randomize_va_space;
    2769                 :            : #endif
    2770                 :            : 
    2771                 :            : const char * arch_vma_name(struct vm_area_struct *vma);
    2772                 :            : #ifdef CONFIG_MMU
    2773                 :            : void print_vma_addr(char *prefix, unsigned long rip);
    2774                 :            : #else
    2775                 :            : static inline void print_vma_addr(char *prefix, unsigned long rip)
    2776                 :            : {
    2777                 :            : }
    2778                 :            : #endif
    2779                 :            : 
    2780                 :            : void *sparse_buffer_alloc(unsigned long size);
    2781                 :            : struct page * __populate_section_memmap(unsigned long pfn,
    2782                 :            :                 unsigned long nr_pages, int nid, struct vmem_altmap *altmap);
    2783                 :            : pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
    2784                 :            : p4d_t *vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node);
    2785                 :            : pud_t *vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node);
    2786                 :            : pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
    2787                 :            : pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
    2788                 :            : void *vmemmap_alloc_block(unsigned long size, int node);
    2789                 :            : struct vmem_altmap;
    2790                 :            : void *vmemmap_alloc_block_buf(unsigned long size, int node);
    2791                 :            : void *altmap_alloc_block_buf(unsigned long size, struct vmem_altmap *altmap);
    2792                 :            : void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
    2793                 :            : int vmemmap_populate_basepages(unsigned long start, unsigned long end,
    2794                 :            :                                int node);
    2795                 :            : int vmemmap_populate(unsigned long start, unsigned long end, int node,
    2796                 :            :                 struct vmem_altmap *altmap);
    2797                 :            : void vmemmap_populate_print_last(void);
    2798                 :            : #ifdef CONFIG_MEMORY_HOTPLUG
    2799                 :            : void vmemmap_free(unsigned long start, unsigned long end,
    2800                 :            :                 struct vmem_altmap *altmap);
    2801                 :            : #endif
    2802                 :            : void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
    2803                 :            :                                   unsigned long nr_pages);
    2804                 :            : 
    2805                 :            : enum mf_flags {
    2806                 :            :         MF_COUNT_INCREASED = 1 << 0,
    2807                 :            :         MF_ACTION_REQUIRED = 1 << 1,
    2808                 :            :         MF_MUST_KILL = 1 << 2,
    2809                 :            :         MF_SOFT_OFFLINE = 1 << 3,
    2810                 :            : };
    2811                 :            : extern int memory_failure(unsigned long pfn, int flags);
    2812                 :            : extern void memory_failure_queue(unsigned long pfn, int flags);
    2813                 :            : extern int unpoison_memory(unsigned long pfn);
    2814                 :            : extern int get_hwpoison_page(struct page *page);
    2815                 :            : #define put_hwpoison_page(page) put_page(page)
    2816                 :            : extern int sysctl_memory_failure_early_kill;
    2817                 :            : extern int sysctl_memory_failure_recovery;
    2818                 :            : extern void shake_page(struct page *p, int access);
    2819                 :            : extern atomic_long_t num_poisoned_pages __read_mostly;
    2820                 :            : extern int soft_offline_page(unsigned long pfn, int flags);
    2821                 :            : 
    2822                 :            : 
    2823                 :            : /*
    2824                 :            :  * Error handlers for various types of pages.
    2825                 :            :  */
    2826                 :            : enum mf_result {
    2827                 :            :         MF_IGNORED,     /* Error: cannot be handled */
    2828                 :            :         MF_FAILED,      /* Error: handling failed */
    2829                 :            :         MF_DELAYED,     /* Will be handled later */
    2830                 :            :         MF_RECOVERED,   /* Successfully recovered */
    2831                 :            : };
    2832                 :            : 
    2833                 :            : enum mf_action_page_type {
    2834                 :            :         MF_MSG_KERNEL,
    2835                 :            :         MF_MSG_KERNEL_HIGH_ORDER,
    2836                 :            :         MF_MSG_SLAB,
    2837                 :            :         MF_MSG_DIFFERENT_COMPOUND,
    2838                 :            :         MF_MSG_POISONED_HUGE,
    2839                 :            :         MF_MSG_HUGE,
    2840                 :            :         MF_MSG_FREE_HUGE,
    2841                 :            :         MF_MSG_NON_PMD_HUGE,
    2842                 :            :         MF_MSG_UNMAP_FAILED,
    2843                 :            :         MF_MSG_DIRTY_SWAPCACHE,
    2844                 :            :         MF_MSG_CLEAN_SWAPCACHE,
    2845                 :            :         MF_MSG_DIRTY_MLOCKED_LRU,
    2846                 :            :         MF_MSG_CLEAN_MLOCKED_LRU,
    2847                 :            :         MF_MSG_DIRTY_UNEVICTABLE_LRU,
    2848                 :            :         MF_MSG_CLEAN_UNEVICTABLE_LRU,
    2849                 :            :         MF_MSG_DIRTY_LRU,
    2850                 :            :         MF_MSG_CLEAN_LRU,
    2851                 :            :         MF_MSG_TRUNCATED_LRU,
    2852                 :            :         MF_MSG_BUDDY,
    2853                 :            :         MF_MSG_BUDDY_2ND,
    2854                 :            :         MF_MSG_DAX,
    2855                 :            :         MF_MSG_UNKNOWN,
    2856                 :            : };
    2857                 :            : 
    2858                 :            : #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
    2859                 :            : extern void clear_huge_page(struct page *page,
    2860                 :            :                             unsigned long addr_hint,
    2861                 :            :                             unsigned int pages_per_huge_page);
    2862                 :            : extern void copy_user_huge_page(struct page *dst, struct page *src,
    2863                 :            :                                 unsigned long addr_hint,
    2864                 :            :                                 struct vm_area_struct *vma,
    2865                 :            :                                 unsigned int pages_per_huge_page);
    2866                 :            : extern long copy_huge_page_from_user(struct page *dst_page,
    2867                 :            :                                 const void __user *usr_src,
    2868                 :            :                                 unsigned int pages_per_huge_page,
    2869                 :            :                                 bool allow_pagefault);
    2870                 :            : #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
    2871                 :            : 
    2872                 :            : #ifdef CONFIG_DEBUG_PAGEALLOC
    2873                 :            : extern unsigned int _debug_guardpage_minorder;
    2874                 :            : DECLARE_STATIC_KEY_FALSE(_debug_guardpage_enabled);
    2875                 :            : 
    2876                 :            : static inline unsigned int debug_guardpage_minorder(void)
    2877                 :            : {
    2878                 :            :         return _debug_guardpage_minorder;
    2879                 :            : }
    2880                 :            : 
    2881                 :            : static inline bool debug_guardpage_enabled(void)
    2882                 :            : {
    2883                 :            :         return static_branch_unlikely(&_debug_guardpage_enabled);
    2884                 :            : }
    2885                 :            : 
    2886                 :            : static inline bool page_is_guard(struct page *page)
    2887                 :            : {
    2888                 :            :         if (!debug_guardpage_enabled())
    2889                 :            :                 return false;
    2890                 :            : 
    2891                 :            :         return PageGuard(page);
    2892                 :            : }
    2893                 :            : #else
    2894                 :         13 : static inline unsigned int debug_guardpage_minorder(void) { return 0; }
    2895                 :            : static inline bool debug_guardpage_enabled(void) { return false; }
    2896   [ +  +  +  + ]:    1207442 : static inline bool page_is_guard(struct page *page) { return false; }
    2897                 :            : #endif /* CONFIG_DEBUG_PAGEALLOC */
    2898                 :            : 
    2899                 :            : #if MAX_NUMNODES > 1
    2900                 :            : void __init setup_nr_node_ids(void);
    2901                 :            : #else
    2902                 :            : static inline void setup_nr_node_ids(void) {}
    2903                 :            : #endif
    2904                 :            : 
    2905                 :            : extern int memcmp_pages(struct page *page1, struct page *page2);
    2906                 :            : 
    2907                 :          0 : static inline int pages_identical(struct page *page1, struct page *page2)
    2908                 :            : {
    2909                 :          0 :         return !memcmp_pages(page1, page2);
    2910                 :            : }
    2911                 :            : 
    2912                 :            : #ifdef CONFIG_MAPPING_DIRTY_HELPERS
    2913                 :            : unsigned long clean_record_shared_mapping_range(struct address_space *mapping,
    2914                 :            :                                                 pgoff_t first_index, pgoff_t nr,
    2915                 :            :                                                 pgoff_t bitmap_pgoff,
    2916                 :            :                                                 unsigned long *bitmap,
    2917                 :            :                                                 pgoff_t *start,
    2918                 :            :                                                 pgoff_t *end);
    2919                 :            : 
    2920                 :            : unsigned long wp_shared_mapping_range(struct address_space *mapping,
    2921                 :            :                                       pgoff_t first_index, pgoff_t nr);
    2922                 :            : #endif
    2923                 :            : 
    2924                 :            : #endif /* __KERNEL__ */
    2925                 :            : #endif /* _LINUX_MM_H */

Generated by: LCOV version 1.14