Branch data     Line data    Source code

       1                 :            : /* SPDX-License-Identifier: GPL-2.0-or-later */
       2                 :            : /* internal.h: mm/ internal definitions
       3                 :            :  *
       4                 :            :  * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
       5                 :            :  * Written by David Howells (dhowells@redhat.com)
       6                 :            :  */
       7                 :            : #ifndef __MM_INTERNAL_H
       8                 :            : #define __MM_INTERNAL_H
       9                 :            : 
      10                 :            : #include <linux/fs.h>
      11                 :            : #include <linux/mm.h>
      12                 :            : #include <linux/pagemap.h>
      13                 :            : #include <linux/tracepoint-defs.h>
      14                 :            : 
      15                 :            : /*
      16                 :            :  * The set of flags that only affect watermark checking and reclaim
      17                 :            :  * behaviour. This is used by the MM to obey the caller constraints
      18                 :            :  * about IO, FS and watermark checking while ignoring placement
      19                 :            :  * hints such as HIGHMEM usage.
      20                 :            :  */
      21                 :            : #define GFP_RECLAIM_MASK (__GFP_RECLAIM|__GFP_HIGH|__GFP_IO|__GFP_FS|\
      22                 :            :                         __GFP_NOWARN|__GFP_RETRY_MAYFAIL|__GFP_NOFAIL|\
      23                 :            :                         __GFP_NORETRY|__GFP_MEMALLOC|__GFP_NOMEMALLOC|\
      24                 :            :                         __GFP_ATOMIC)
      25                 :            : 
      26                 :            : /* The GFP flags allowed during early boot */
      27                 :            : #define GFP_BOOT_MASK (__GFP_BITS_MASK & ~(__GFP_RECLAIM|__GFP_IO|__GFP_FS))
      28                 :            : 
      29                 :            : /* Control allocation cpuset and node placement constraints */
      30                 :            : #define GFP_CONSTRAINT_MASK (__GFP_HARDWALL|__GFP_THISNODE)
      31                 :            : 
      32                 :            : /* Do not use these with a slab allocator */
      33                 :            : #define GFP_SLAB_BUG_MASK (__GFP_DMA32|__GFP_HIGHMEM|~__GFP_BITS_MASK)
      34                 :            : 
      35                 :            : void page_writeback_init(void);
      36                 :            : 
      37                 :            : vm_fault_t do_swap_page(struct vm_fault *vmf);
      38                 :            : 
      39                 :            : void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
      40                 :            :                 unsigned long floor, unsigned long ceiling);
      41                 :            : 
      42                 :            : static inline bool can_madv_lru_vma(struct vm_area_struct *vma)
      43                 :            : {
      44                 :          3 :         return !(vma->vm_flags & (VM_LOCKED|VM_HUGETLB|VM_PFNMAP));
      45                 :            : }
      46                 :            : 
      47                 :            : void unmap_page_range(struct mmu_gather *tlb,
      48                 :            :                              struct vm_area_struct *vma,
      49                 :            :                              unsigned long addr, unsigned long end,
      50                 :            :                              struct zap_details *details);
      51                 :            : 
      52                 :            : extern unsigned int __do_page_cache_readahead(struct address_space *mapping,
      53                 :            :                 struct file *filp, pgoff_t offset, unsigned long nr_to_read,
      54                 :            :                 unsigned long lookahead_size);
      55                 :            : 
      56                 :            : /*
      57                 :            :  * Submit IO for the read-ahead request in file_ra_state.
      58                 :            :  */
      59                 :            : static inline unsigned long ra_submit(struct file_ra_state *ra,
      60                 :            :                 struct address_space *mapping, struct file *filp)
      61                 :            : {
      62                 :          3 :         return __do_page_cache_readahead(mapping, filp,
      63                 :          3 :                                         ra->start, ra->size, ra->async_size);
      64                 :            : }
      65                 :            : 
      66                 :            : /*
      67                 :            :  * Turn a non-refcounted page (->_refcount == 0) into refcounted with
      68                 :            :  * a count of one.
      69                 :            :  */
      70                 :            : static inline void set_page_refcounted(struct page *page)
      71                 :            : {
      72                 :            :         VM_BUG_ON_PAGE(PageTail(page), page);
      73                 :            :         VM_BUG_ON_PAGE(page_ref_count(page), page);
      74                 :            :         set_page_count(page, 1);
      75                 :            : }
      76                 :            : 
      77                 :            : extern unsigned long highest_memmap_pfn;
      78                 :            : 
      79                 :            : /*
      80                 :            :  * Maximum number of reclaim retries without progress before the OOM
      81                 :            :  * killer is consider the only way forward.
      82                 :            :  */
      83                 :            : #define MAX_RECLAIM_RETRIES 16
      84                 :            : 
      85                 :            : /*
      86                 :            :  * in mm/vmscan.c:
      87                 :            :  */
      88                 :            : extern int isolate_lru_page(struct page *page);
      89                 :            : extern void putback_lru_page(struct page *page);
      90                 :            : 
      91                 :            : /*
      92                 :            :  * in mm/rmap.c:
      93                 :            :  */
      94                 :            : extern pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address);
      95                 :            : 
      96                 :            : /*
      97                 :            :  * in mm/page_alloc.c
      98                 :            :  */
      99                 :            : 
     100                 :            : /*
     101                 :            :  * Structure for holding the mostly immutable allocation parameters passed
     102                 :            :  * between functions involved in allocations, including the alloc_pages*
     103                 :            :  * family of functions.
     104                 :            :  *
     105                 :            :  * nodemask, migratetype and high_zoneidx are initialized only once in
     106                 :            :  * __alloc_pages_nodemask() and then never change.
     107                 :            :  *
     108                 :            :  * zonelist, preferred_zone and classzone_idx are set first in
     109                 :            :  * __alloc_pages_nodemask() for the fast path, and might be later changed
     110                 :            :  * in __alloc_pages_slowpath(). All other functions pass the whole strucure
     111                 :            :  * by a const pointer.
     112                 :            :  */
     113                 :            : struct alloc_context {
     114                 :            :         struct zonelist *zonelist;
     115                 :            :         nodemask_t *nodemask;
     116                 :            :         struct zoneref *preferred_zoneref;
     117                 :            :         int migratetype;
     118                 :            :         enum zone_type high_zoneidx;
     119                 :            :         bool spread_dirty_pages;
     120                 :            : };
     121                 :            : 
     122                 :            : #define ac_classzone_idx(ac) zonelist_zone_idx(ac->preferred_zoneref)
     123                 :            : 
     124                 :            : /*
     125                 :            :  * Locate the struct page for both the matching buddy in our
     126                 :            :  * pair (buddy1) and the combined O(n+1) page they form (page).
     127                 :            :  *
     128                 :            :  * 1) Any buddy B1 will have an order O twin B2 which satisfies
     129                 :            :  * the following equation:
     130                 :            :  *     B2 = B1 ^ (1 << O)
     131                 :            :  * For example, if the starting buddy (buddy2) is #8 its order
     132                 :            :  * 1 buddy is #10:
     133                 :            :  *     B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
     134                 :            :  *
     135                 :            :  * 2) Any buddy B will have an order O+1 parent P which
     136                 :            :  * satisfies the following equation:
     137                 :            :  *     P = B & ~(1 << O)
     138                 :            :  *
     139                 :            :  * Assumption: *_mem_map is contiguous at least up to MAX_ORDER
     140                 :            :  */
     141                 :            : static inline unsigned long
     142                 :            : __find_buddy_pfn(unsigned long page_pfn, unsigned int order)
     143                 :            : {
     144                 :          3 :         return page_pfn ^ (1 << order);
     145                 :            : }
     146                 :            : 
     147                 :            : extern struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
     148                 :            :                                 unsigned long end_pfn, struct zone *zone);
     149                 :            : 
     150                 :          3 : static inline struct page *pageblock_pfn_to_page(unsigned long start_pfn,
     151                 :            :                                 unsigned long end_pfn, struct zone *zone)
     152                 :            : {
     153                 :          3 :         if (zone->contiguous)
     154                 :          3 :                 return pfn_to_page(start_pfn);
     155                 :            : 
     156                 :          0 :         return __pageblock_pfn_to_page(start_pfn, end_pfn, zone);
     157                 :            : }
     158                 :            : 
     159                 :            : extern int __isolate_free_page(struct page *page, unsigned int order);
     160                 :            : extern void memblock_free_pages(struct page *page, unsigned long pfn,
     161                 :            :                                         unsigned int order);
     162                 :            : extern void __free_pages_core(struct page *page, unsigned int order);
     163                 :            : extern void prep_compound_page(struct page *page, unsigned int order);
     164                 :            : extern void post_alloc_hook(struct page *page, unsigned int order,
     165                 :            :                                         gfp_t gfp_flags);
     166                 :            : extern int user_min_free_kbytes;
     167                 :            : 
     168                 :            : #if defined CONFIG_COMPACTION || defined CONFIG_CMA
     169                 :            : 
     170                 :            : /*
     171                 :            :  * in mm/compaction.c
     172                 :            :  */
     173                 :            : /*
     174                 :            :  * compact_control is used to track pages being migrated and the free pages
     175                 :            :  * they are being migrated to during memory compaction. The free_pfn starts
     176                 :            :  * at the end of a zone and migrate_pfn begins at the start. Movable pages
     177                 :            :  * are moved to the end of a zone during a compaction run and the run
     178                 :            :  * completes when free_pfn <= migrate_pfn
     179                 :            :  */
     180                 :            : struct compact_control {
     181                 :            :         struct list_head freepages;     /* List of free pages to migrate to */
     182                 :            :         struct list_head migratepages;  /* List of pages being migrated */
     183                 :            :         unsigned int nr_freepages;      /* Number of isolated free pages */
     184                 :            :         unsigned int nr_migratepages;   /* Number of pages to migrate */
     185                 :            :         unsigned long free_pfn;         /* isolate_freepages search base */
     186                 :            :         unsigned long migrate_pfn;      /* isolate_migratepages search base */
     187                 :            :         unsigned long fast_start_pfn;   /* a pfn to start linear scan from */
     188                 :            :         struct zone *zone;
     189                 :            :         unsigned long total_migrate_scanned;
     190                 :            :         unsigned long total_free_scanned;
     191                 :            :         unsigned short fast_search_fail;/* failures to use free list searches */
     192                 :            :         short search_order;             /* order to start a fast search at */
     193                 :            :         const gfp_t gfp_mask;           /* gfp mask of a direct compactor */
     194                 :            :         int order;                      /* order a direct compactor needs */
     195                 :            :         int migratetype;                /* migratetype of direct compactor */
     196                 :            :         const unsigned int alloc_flags; /* alloc flags of a direct compactor */
     197                 :            :         const int classzone_idx;        /* zone index of a direct compactor */
     198                 :            :         enum migrate_mode mode;         /* Async or sync migration mode */
     199                 :            :         bool ignore_skip_hint;          /* Scan blocks even if marked skip */
     200                 :            :         bool no_set_skip_hint;          /* Don't mark blocks for skipping */
     201                 :            :         bool ignore_block_suitable;     /* Scan blocks considered unsuitable */
     202                 :            :         bool direct_compaction;         /* False from kcompactd or /proc/... */
     203                 :            :         bool whole_zone;                /* Whole zone should/has been scanned */
     204                 :            :         bool contended;                 /* Signal lock or sched contention */
     205                 :            :         bool rescan;                    /* Rescanning the same pageblock */
     206                 :            : };
     207                 :            : 
     208                 :            : /*
     209                 :            :  * Used in direct compaction when a page should be taken from the freelists
     210                 :            :  * immediately when one is created during the free path.
     211                 :            :  */
     212                 :            : struct capture_control {
     213                 :            :         struct compact_control *cc;
     214                 :            :         struct page *page;
     215                 :            : };
     216                 :            : 
     217                 :            : unsigned long
     218                 :            : isolate_freepages_range(struct compact_control *cc,
     219                 :            :                         unsigned long start_pfn, unsigned long end_pfn);
     220                 :            : unsigned long
     221                 :            : isolate_migratepages_range(struct compact_control *cc,
     222                 :            :                            unsigned long low_pfn, unsigned long end_pfn);
     223                 :            : int find_suitable_fallback(struct free_area *area, unsigned int order,
     224                 :            :                         int migratetype, bool only_stealable, bool *can_steal);
     225                 :            : 
     226                 :            : #endif
     227                 :            : 
     228                 :            : /*
     229                 :            :  * This function returns the order of a free page in the buddy system. In
     230                 :            :  * general, page_zone(page)->lock must be held by the caller to prevent the
     231                 :            :  * page from being allocated in parallel and returning garbage as the order.
     232                 :            :  * If a caller does not hold page_zone(page)->lock, it must guarantee that the
     233                 :            :  * page cannot be allocated or merged in parallel. Alternatively, it must
     234                 :            :  * handle invalid values gracefully, and use page_order_unsafe() below.
     235                 :            :  */
     236                 :            : static inline unsigned int page_order(struct page *page)
     237                 :            : {
     238                 :            :         /* PageBuddy() must be checked by the caller */
     239                 :          3 :         return page_private(page);
     240                 :            : }
     241                 :            : 
     242                 :            : /*
     243                 :            :  * Like page_order(), but for callers who cannot afford to hold the zone lock.
     244                 :            :  * PageBuddy() should be checked first by the caller to minimize race window,
     245                 :            :  * and invalid values must be handled gracefully.
     246                 :            :  *
     247                 :            :  * READ_ONCE is used so that if the caller assigns the result into a local
     248                 :            :  * variable and e.g. tests it for valid range before using, the compiler cannot
     249                 :            :  * decide to remove the variable and inline the page_private(page) multiple
     250                 :            :  * times, potentially observing different values in the tests and the actual
     251                 :            :  * use of the result.
     252                 :            :  */
     253                 :            : #define page_order_unsafe(page)         READ_ONCE(page_private(page))
     254                 :            : 
     255                 :            : static inline bool is_cow_mapping(vm_flags_t flags)
     256                 :            : {
     257                 :          3 :         return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE;
     258                 :            : }
     259                 :            : 
     260                 :            : /*
     261                 :            :  * These three helpers classifies VMAs for virtual memory accounting.
     262                 :            :  */
     263                 :            : 
     264                 :            : /*
     265                 :            :  * Executable code area - executable, not writable, not stack
     266                 :            :  */
     267                 :            : static inline bool is_exec_mapping(vm_flags_t flags)
     268                 :            : {
     269                 :          3 :         return (flags & (VM_EXEC | VM_WRITE | VM_STACK)) == VM_EXEC;
     270                 :            : }
     271                 :            : 
     272                 :            : /*
     273                 :            :  * Stack area - atomatically grows in one direction
     274                 :            :  *
     275                 :            :  * VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous:
     276                 :            :  * do_mmap() forbids all other combinations.
     277                 :            :  */
     278                 :            : static inline bool is_stack_mapping(vm_flags_t flags)
     279                 :            : {
     280                 :          3 :         return (flags & VM_STACK) == VM_STACK;
     281                 :            : }
     282                 :            : 
     283                 :            : /*
     284                 :            :  * Data area - private, writable, not stack
     285                 :            :  */
     286                 :            : static inline bool is_data_mapping(vm_flags_t flags)
     287                 :            : {
     288                 :          3 :         return (flags & (VM_WRITE | VM_SHARED | VM_STACK)) == VM_WRITE;
     289                 :            : }
     290                 :            : 
     291                 :            : /* mm/util.c */
     292                 :            : void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
     293                 :            :                 struct vm_area_struct *prev, struct rb_node *rb_parent);
     294                 :            : 
     295                 :            : #ifdef CONFIG_MMU
     296                 :            : extern long populate_vma_page_range(struct vm_area_struct *vma,
     297                 :            :                 unsigned long start, unsigned long end, int *nonblocking);
     298                 :            : extern void munlock_vma_pages_range(struct vm_area_struct *vma,
     299                 :            :                         unsigned long start, unsigned long end);
     300                 :            : static inline void munlock_vma_pages_all(struct vm_area_struct *vma)
     301                 :            : {
     302                 :          3 :         munlock_vma_pages_range(vma, vma->vm_start, vma->vm_end);
     303                 :            : }
     304                 :            : 
     305                 :            : /*
     306                 :            :  * must be called with vma's mmap_sem held for read or write, and page locked.
     307                 :            :  */
     308                 :            : extern void mlock_vma_page(struct page *page);
     309                 :            : extern unsigned int munlock_vma_page(struct page *page);
     310                 :            : 
     311                 :            : /*
     312                 :            :  * Clear the page's PageMlocked().  This can be useful in a situation where
     313                 :            :  * we want to unconditionally remove a page from the pagecache -- e.g.,
     314                 :            :  * on truncation or freeing.
     315                 :            :  *
     316                 :            :  * It is legal to call this function for any page, mlocked or not.
     317                 :            :  * If called for a page that is still mapped by mlocked vmas, all we do
     318                 :            :  * is revert to lazy LRU behaviour -- semantics are not broken.
     319                 :            :  */
     320                 :            : extern void clear_page_mlock(struct page *page);
     321                 :            : 
     322                 :            : /*
     323                 :            :  * mlock_migrate_page - called only from migrate_misplaced_transhuge_page()
     324                 :            :  * (because that does not go through the full procedure of migration ptes):
     325                 :            :  * to migrate the Mlocked page flag; update statistics.
     326                 :            :  */
     327                 :            : static inline void mlock_migrate_page(struct page *newpage, struct page *page)
     328                 :            : {
     329                 :            :         if (TestClearPageMlocked(page)) {
     330                 :            :                 int nr_pages = hpage_nr_pages(page);
     331                 :            : 
     332                 :            :                 /* Holding pmd lock, no change in irq context: __mod is safe */
     333                 :            :                 __mod_zone_page_state(page_zone(page), NR_MLOCK, -nr_pages);
     334                 :            :                 SetPageMlocked(newpage);
     335                 :            :                 __mod_zone_page_state(page_zone(newpage), NR_MLOCK, nr_pages);
     336                 :            :         }
     337                 :            : }
     338                 :            : 
     339                 :            : extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);
     340                 :            : 
     341                 :            : /*
     342                 :            :  * At what user virtual address is page expected in @vma?
     343                 :            :  */
     344                 :            : static inline unsigned long
     345                 :            : __vma_address(struct page *page, struct vm_area_struct *vma)
     346                 :            : {
     347                 :            :         pgoff_t pgoff = page_to_pgoff(page);
     348                 :          2 :         return vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
     349                 :            : }
     350                 :            : 
     351                 :            : static inline unsigned long
     352                 :            : vma_address(struct page *page, struct vm_area_struct *vma)
     353                 :            : {
     354                 :            :         unsigned long start, end;
     355                 :            : 
     356                 :            :         start = __vma_address(page, vma);
     357                 :            :         end = start + PAGE_SIZE * (hpage_nr_pages(page) - 1);
     358                 :            : 
     359                 :            :         /* page should be within @vma mapping range */
     360                 :            :         VM_BUG_ON_VMA(end < vma->vm_start || start >= vma->vm_end, vma);
     361                 :            : 
     362                 :          2 :         return max(start, vma->vm_start);
     363                 :            : }
     364                 :            : 
     365                 :          3 : static inline struct file *maybe_unlock_mmap_for_io(struct vm_fault *vmf,
     366                 :            :                                                     struct file *fpin)
     367                 :            : {
     368                 :          3 :         int flags = vmf->flags;
     369                 :            : 
     370                 :          3 :         if (fpin)
     371                 :            :                 return fpin;
     372                 :            : 
     373                 :            :         /*
     374                 :            :          * FAULT_FLAG_RETRY_NOWAIT means we don't want to wait on page locks or
     375                 :            :          * anything, so we only pin the file and drop the mmap_sem if only
     376                 :            :          * FAULT_FLAG_ALLOW_RETRY is set.
     377                 :            :          */
     378                 :          3 :         if ((flags & (FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT)) ==
     379                 :            :             FAULT_FLAG_ALLOW_RETRY) {
     380                 :          3 :                 fpin = get_file(vmf->vma->vm_file);
     381                 :          3 :                 up_read(&vmf->vma->vm_mm->mmap_sem);
     382                 :            :         }
     383                 :          3 :         return fpin;
     384                 :            : }
     385                 :            : 
     386                 :            : #else /* !CONFIG_MMU */
     387                 :            : static inline void clear_page_mlock(struct page *page) { }
     388                 :            : static inline void mlock_vma_page(struct page *page) { }
     389                 :            : static inline void mlock_migrate_page(struct page *new, struct page *old) { }
     390                 :            : 
     391                 :            : #endif /* !CONFIG_MMU */
     392                 :            : 
     393                 :            : /*
     394                 :            :  * Return the mem_map entry representing the 'offset' subpage within
     395                 :            :  * the maximally aligned gigantic page 'base'.  Handle any discontiguity
     396                 :            :  * in the mem_map at MAX_ORDER_NR_PAGES boundaries.
     397                 :            :  */
     398                 :            : static inline struct page *mem_map_offset(struct page *base, int offset)
     399                 :            : {
     400                 :            :         if (unlikely(offset >= MAX_ORDER_NR_PAGES))
     401                 :            :                 return nth_page(base, offset);
     402                 :            :         return base + offset;
     403                 :            : }
     404                 :            : 
     405                 :            : /*
     406                 :            :  * Iterator over all subpages within the maximally aligned gigantic
     407                 :            :  * page 'base'.  Handle any discontiguity in the mem_map.
     408                 :            :  */
     409                 :            : static inline struct page *mem_map_next(struct page *iter,
     410                 :            :                                                 struct page *base, int offset)
     411                 :            : {
     412                 :            :         if (unlikely((offset & (MAX_ORDER_NR_PAGES - 1)) == 0)) {
     413                 :            :                 unsigned long pfn = page_to_pfn(base) + offset;
     414                 :            :                 if (!pfn_valid(pfn))
     415                 :            :                         return NULL;
     416                 :            :                 return pfn_to_page(pfn);
     417                 :            :         }
     418                 :            :         return iter + 1;
     419                 :            : }
     420                 :            : 
     421                 :            : /* Memory initialisation debug and verification */
     422                 :            : enum mminit_level {
     423                 :            :         MMINIT_WARNING,
     424                 :            :         MMINIT_VERIFY,
     425                 :            :         MMINIT_TRACE
     426                 :            : };
     427                 :            : 
     428                 :            : #ifdef CONFIG_DEBUG_MEMORY_INIT
     429                 :            : 
     430                 :            : extern int mminit_loglevel;
     431                 :            : 
     432                 :            : #define mminit_dprintk(level, prefix, fmt, arg...) \
     433                 :            : do { \
     434                 :            :         if (level < mminit_loglevel) { \
     435                 :            :                 if (level <= MMINIT_WARNING) \
     436                 :            :                         pr_warn("mminit::" prefix " " fmt, ##arg);  \
     437                 :            :                 else \
     438                 :            :                         printk(KERN_DEBUG "mminit::" prefix " " fmt, ##arg); \
     439                 :            :         } \
     440                 :            : } while (0)
     441                 :            : 
     442                 :            : extern void mminit_verify_pageflags_layout(void);
     443                 :            : extern void mminit_verify_zonelist(void);
     444                 :            : #else
     445                 :            : 
     446                 :            : static inline void mminit_dprintk(enum mminit_level level,
     447                 :            :                                 const char *prefix, const char *fmt, ...)
     448                 :            : {
     449                 :            : }
     450                 :            : 
     451                 :            : static inline void mminit_verify_pageflags_layout(void)
     452                 :            : {
     453                 :            : }
     454                 :            : 
     455                 :            : static inline void mminit_verify_zonelist(void)
     456                 :            : {
     457                 :            : }
     458                 :            : #endif /* CONFIG_DEBUG_MEMORY_INIT */
     459                 :            : 
     460                 :            : /* mminit_validate_memmodel_limits is independent of CONFIG_DEBUG_MEMORY_INIT */
     461                 :            : #if defined(CONFIG_SPARSEMEM)
     462                 :            : extern void mminit_validate_memmodel_limits(unsigned long *start_pfn,
     463                 :            :                                 unsigned long *end_pfn);
     464                 :            : #else
     465                 :            : static inline void mminit_validate_memmodel_limits(unsigned long *start_pfn,
     466                 :            :                                 unsigned long *end_pfn)
     467                 :            : {
     468                 :            : }
     469                 :            : #endif /* CONFIG_SPARSEMEM */
     470                 :            : 
     471                 :            : #define NODE_RECLAIM_NOSCAN     -2
     472                 :            : #define NODE_RECLAIM_FULL       -1
     473                 :            : #define NODE_RECLAIM_SOME       0
     474                 :            : #define NODE_RECLAIM_SUCCESS    1
     475                 :            : 
     476                 :            : #ifdef CONFIG_NUMA
     477                 :            : extern int node_reclaim(struct pglist_data *, gfp_t, unsigned int);
     478                 :            : #else
     479                 :            : static inline int node_reclaim(struct pglist_data *pgdat, gfp_t mask,
     480                 :            :                                 unsigned int order)
     481                 :            : {
     482                 :            :         return NODE_RECLAIM_NOSCAN;
     483                 :            : }
     484                 :            : #endif
     485                 :            : 
     486                 :            : extern int hwpoison_filter(struct page *p);
     487                 :            : 
     488                 :            : extern u32 hwpoison_filter_dev_major;
     489                 :            : extern u32 hwpoison_filter_dev_minor;
     490                 :            : extern u64 hwpoison_filter_flags_mask;
     491                 :            : extern u64 hwpoison_filter_flags_value;
     492                 :            : extern u64 hwpoison_filter_memcg;
     493                 :            : extern u32 hwpoison_filter_enable;
     494                 :            : 
     495                 :            : extern unsigned long  __must_check vm_mmap_pgoff(struct file *, unsigned long,
     496                 :            :         unsigned long, unsigned long,
     497                 :            :         unsigned long, unsigned long);
     498                 :            : 
     499                 :            : extern void set_pageblock_order(void);
     500                 :            : unsigned long reclaim_clean_pages_from_list(struct zone *zone,
     501                 :            :                                             struct list_head *page_list);
     502                 :            : /* The ALLOC_WMARK bits are used as an index to zone->watermark */
     503                 :            : #define ALLOC_WMARK_MIN         WMARK_MIN
     504                 :            : #define ALLOC_WMARK_LOW         WMARK_LOW
     505                 :            : #define ALLOC_WMARK_HIGH        WMARK_HIGH
     506                 :            : #define ALLOC_NO_WATERMARKS     0x04 /* don't check watermarks at all */
     507                 :            : 
     508                 :            : /* Mask to get the watermark bits */
     509                 :            : #define ALLOC_WMARK_MASK        (ALLOC_NO_WATERMARKS-1)
     510                 :            : 
     511                 :            : /*
     512                 :            :  * Only MMU archs have async oom victim reclaim - aka oom_reaper so we
     513                 :            :  * cannot assume a reduced access to memory reserves is sufficient for
     514                 :            :  * !MMU
     515                 :            :  */
     516                 :            : #ifdef CONFIG_MMU
     517                 :            : #define ALLOC_OOM               0x08
     518                 :            : #else
     519                 :            : #define ALLOC_OOM               ALLOC_NO_WATERMARKS
     520                 :            : #endif
     521                 :            : 
     522                 :            : #define ALLOC_HARDER             0x10 /* try to alloc harder */
     523                 :            : #define ALLOC_HIGH               0x20 /* __GFP_HIGH set */
     524                 :            : #define ALLOC_CPUSET             0x40 /* check for correct cpuset */
     525                 :            : #define ALLOC_CMA                0x80 /* allow allocations from CMA areas */
     526                 :            : #ifdef CONFIG_ZONE_DMA32
     527                 :            : #define ALLOC_NOFRAGMENT        0x100 /* avoid mixing pageblock types */
     528                 :            : #else
     529                 :            : #define ALLOC_NOFRAGMENT          0x0
     530                 :            : #endif
     531                 :            : #define ALLOC_KSWAPD            0x200 /* allow waking of kswapd */
     532                 :            : 
     533                 :            : enum ttu_flags;
     534                 :            : struct tlbflush_unmap_batch;
     535                 :            : 
     536                 :            : 
     537                 :            : /*
     538                 :            :  * only for MM internal work items which do not depend on
     539                 :            :  * any allocations or locks which might depend on allocations
     540                 :            :  */
     541                 :            : extern struct workqueue_struct *mm_percpu_wq;
     542                 :            : 
     543                 :            : #ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
     544                 :            : void try_to_unmap_flush(void);
     545                 :            : void try_to_unmap_flush_dirty(void);
     546                 :            : void flush_tlb_batched_pending(struct mm_struct *mm);
     547                 :            : #else
     548                 :            : static inline void try_to_unmap_flush(void)
     549                 :            : {
     550                 :            : }
     551                 :            : static inline void try_to_unmap_flush_dirty(void)
     552                 :            : {
     553                 :            : }
     554                 :            : static inline void flush_tlb_batched_pending(struct mm_struct *mm)
     555                 :            : {
     556                 :            : }
     557                 :            : #endif /* CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH */
     558                 :            : 
     559                 :            : extern const struct trace_print_flags pageflag_names[];
     560                 :            : extern const struct trace_print_flags vmaflag_names[];
     561                 :            : extern const struct trace_print_flags gfpflag_names[];
     562                 :            : 
     563                 :            : static inline bool is_migrate_highatomic(enum migratetype migratetype)
     564                 :            : {
     565                 :            :         return migratetype == MIGRATE_HIGHATOMIC;
     566                 :            : }
     567                 :            : 
     568                 :          0 : static inline bool is_migrate_highatomic_page(struct page *page)
     569                 :            : {
     570                 :          0 :         return get_pageblock_migratetype(page) == MIGRATE_HIGHATOMIC;
     571                 :            : }
     572                 :            : 
     573                 :            : void setup_zone_pageset(struct zone *zone);
     574                 :            : extern struct page *alloc_new_node_page(struct page *page, unsigned long node);
     575                 :            : #endif  /* __MM_INTERNAL_H */