LCOV - code coverage report
Current view: top level - mm - page-writeback.c (source / functions) Hit Total Coverage
Test: combined.info Lines: 328 869 37.7 %
Date: 2022-04-01 14:35:51 Functions: 27 57 47.4 %
Branches: 94 554 17.0 %

           Branch data     Line data    Source code
       1                 :            : // SPDX-License-Identifier: GPL-2.0-only
       2                 :            : /*
       3                 :            :  * mm/page-writeback.c
       4                 :            :  *
       5                 :            :  * Copyright (C) 2002, Linus Torvalds.
       6                 :            :  * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
       7                 :            :  *
       8                 :            :  * Contains functions related to writing back dirty pages at the
       9                 :            :  * address_space level.
      10                 :            :  *
      11                 :            :  * 10Apr2002    Andrew Morton
      12                 :            :  *              Initial version
      13                 :            :  */
      14                 :            : 
      15                 :            : #include <linux/kernel.h>
      16                 :            : #include <linux/export.h>
      17                 :            : #include <linux/spinlock.h>
      18                 :            : #include <linux/fs.h>
      19                 :            : #include <linux/mm.h>
      20                 :            : #include <linux/swap.h>
      21                 :            : #include <linux/slab.h>
      22                 :            : #include <linux/pagemap.h>
      23                 :            : #include <linux/writeback.h>
      24                 :            : #include <linux/init.h>
      25                 :            : #include <linux/backing-dev.h>
      26                 :            : #include <linux/task_io_accounting_ops.h>
      27                 :            : #include <linux/blkdev.h>
      28                 :            : #include <linux/mpage.h>
      29                 :            : #include <linux/rmap.h>
      30                 :            : #include <linux/percpu.h>
      31                 :            : #include <linux/smp.h>
      32                 :            : #include <linux/sysctl.h>
      33                 :            : #include <linux/cpu.h>
      34                 :            : #include <linux/syscalls.h>
      35                 :            : #include <linux/buffer_head.h> /* __set_page_dirty_buffers */
      36                 :            : #include <linux/pagevec.h>
      37                 :            : #include <linux/timer.h>
      38                 :            : #include <linux/sched/rt.h>
      39                 :            : #include <linux/sched/signal.h>
      40                 :            : #include <linux/mm_inline.h>
      41                 :            : #include <trace/events/writeback.h>
      42                 :            : 
      43                 :            : #include "internal.h"
      44                 :            : 
      45                 :            : /*
      46                 :            :  * Sleep at most 200ms at a time in balance_dirty_pages().
      47                 :            :  */
      48                 :            : #define MAX_PAUSE               max(HZ/5, 1)
      49                 :            : 
      50                 :            : /*
      51                 :            :  * Try to keep balance_dirty_pages() call intervals higher than this many pages
      52                 :            :  * by raising pause time to max_pause when falls below it.
      53                 :            :  */
      54                 :            : #define DIRTY_POLL_THRESH       (128 >> (PAGE_SHIFT - 10))
      55                 :            : 
      56                 :            : /*
      57                 :            :  * Estimate write bandwidth at 200ms intervals.
      58                 :            :  */
      59                 :            : #define BANDWIDTH_INTERVAL      max(HZ/5, 1)
      60                 :            : 
      61                 :            : #define RATELIMIT_CALC_SHIFT    10
      62                 :            : 
      63                 :            : /*
      64                 :            :  * After a CPU has dirtied this many pages, balance_dirty_pages_ratelimited
      65                 :            :  * will look to see if it needs to force writeback or throttling.
      66                 :            :  */
      67                 :            : static long ratelimit_pages = 32;
      68                 :            : 
      69                 :            : /* The following parameters are exported via /proc/sys/vm */
      70                 :            : 
      71                 :            : /*
      72                 :            :  * Start background writeback (via writeback threads) at this percentage
      73                 :            :  */
      74                 :            : int dirty_background_ratio = 10;
      75                 :            : 
      76                 :            : /*
      77                 :            :  * dirty_background_bytes starts at 0 (disabled) so that it is a function of
      78                 :            :  * dirty_background_ratio * the amount of dirtyable memory
      79                 :            :  */
      80                 :            : unsigned long dirty_background_bytes;
      81                 :            : 
      82                 :            : /*
      83                 :            :  * free highmem will not be subtracted from the total free memory
      84                 :            :  * for calculating free ratios if vm_highmem_is_dirtyable is true
      85                 :            :  */
      86                 :            : int vm_highmem_is_dirtyable;
      87                 :            : 
      88                 :            : /*
      89                 :            :  * The generator of dirty data starts writeback at this percentage
      90                 :            :  */
      91                 :            : int vm_dirty_ratio = 20;
      92                 :            : 
      93                 :            : /*
      94                 :            :  * vm_dirty_bytes starts at 0 (disabled) so that it is a function of
      95                 :            :  * vm_dirty_ratio * the amount of dirtyable memory
      96                 :            :  */
      97                 :            : unsigned long vm_dirty_bytes;
      98                 :            : 
      99                 :            : /*
     100                 :            :  * The interval between `kupdate'-style writebacks
     101                 :            :  */
     102                 :            : unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */
     103                 :            : 
     104                 :            : EXPORT_SYMBOL_GPL(dirty_writeback_interval);
     105                 :            : 
     106                 :            : /*
     107                 :            :  * The longest time for which data is allowed to remain dirty
     108                 :            :  */
     109                 :            : unsigned int dirty_expire_interval = 30 * 100; /* centiseconds */
     110                 :            : 
     111                 :            : /*
     112                 :            :  * Flag that makes the machine dump writes/reads and block dirtyings.
     113                 :            :  */
     114                 :            : int block_dump;
     115                 :            : 
     116                 :            : /*
     117                 :            :  * Flag that puts the machine in "laptop mode". Doubles as a timeout in jiffies:
     118                 :            :  * a full sync is triggered after this time elapses without any disk activity.
     119                 :            :  */
     120                 :            : int laptop_mode;
     121                 :            : 
     122                 :            : EXPORT_SYMBOL(laptop_mode);
     123                 :            : 
     124                 :            : /* End of sysctl-exported parameters */
     125                 :            : 
     126                 :            : struct wb_domain global_wb_domain;
     127                 :            : 
     128                 :            : /* consolidated parameters for balance_dirty_pages() and its subroutines */
     129                 :            : struct dirty_throttle_control {
     130                 :            : #ifdef CONFIG_CGROUP_WRITEBACK
     131                 :            :         struct wb_domain        *dom;
     132                 :            :         struct dirty_throttle_control *gdtc;    /* only set in memcg dtc's */
     133                 :            : #endif
     134                 :            :         struct bdi_writeback    *wb;
     135                 :            :         struct fprop_local_percpu *wb_completions;
     136                 :            : 
     137                 :            :         unsigned long           avail;          /* dirtyable */
     138                 :            :         unsigned long           dirty;          /* file_dirty + write + nfs */
     139                 :            :         unsigned long           thresh;         /* dirty threshold */
     140                 :            :         unsigned long           bg_thresh;      /* dirty background threshold */
     141                 :            : 
     142                 :            :         unsigned long           wb_dirty;       /* per-wb counterparts */
     143                 :            :         unsigned long           wb_thresh;
     144                 :            :         unsigned long           wb_bg_thresh;
     145                 :            : 
     146                 :            :         unsigned long           pos_ratio;
     147                 :            : };
     148                 :            : 
     149                 :            : /*
     150                 :            :  * Length of period for aging writeout fractions of bdis. This is an
     151                 :            :  * arbitrarily chosen number. The longer the period, the slower fractions will
     152                 :            :  * reflect changes in current writeout rate.
     153                 :            :  */
     154                 :            : #define VM_COMPLETIONS_PERIOD_LEN (3*HZ)
     155                 :            : 
     156                 :            : #ifdef CONFIG_CGROUP_WRITEBACK
     157                 :            : 
     158                 :            : #define GDTC_INIT(__wb)         .wb = (__wb),                           \
     159                 :            :                                 .dom = &global_wb_domain,           \
     160                 :            :                                 .wb_completions = &(__wb)->completions
     161                 :            : 
     162                 :            : #define GDTC_INIT_NO_WB         .dom = &global_wb_domain
     163                 :            : 
     164                 :            : #define MDTC_INIT(__wb, __gdtc) .wb = (__wb),                           \
     165                 :            :                                 .dom = mem_cgroup_wb_domain(__wb),      \
     166                 :            :                                 .wb_completions = &(__wb)->memcg_completions, \
     167                 :            :                                 .gdtc = __gdtc
     168                 :            : 
     169                 :            : static bool mdtc_valid(struct dirty_throttle_control *dtc)
     170                 :            : {
     171                 :            :         return dtc->dom;
     172                 :            : }
     173                 :            : 
     174                 :            : static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc)
     175                 :            : {
     176                 :            :         return dtc->dom;
     177                 :            : }
     178                 :            : 
     179                 :            : static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc)
     180                 :            : {
     181                 :            :         return mdtc->gdtc;
     182                 :            : }
     183                 :            : 
     184                 :            : static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb)
     185                 :            : {
     186                 :            :         return &wb->memcg_completions;
     187                 :            : }
     188                 :            : 
     189                 :            : static void wb_min_max_ratio(struct bdi_writeback *wb,
     190                 :            :                              unsigned long *minp, unsigned long *maxp)
     191                 :            : {
     192                 :            :         unsigned long this_bw = wb->avg_write_bandwidth;
     193                 :            :         unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth);
     194                 :            :         unsigned long long min = wb->bdi->min_ratio;
     195                 :            :         unsigned long long max = wb->bdi->max_ratio;
     196                 :            : 
     197                 :            :         /*
     198                 :            :          * @wb may already be clean by the time control reaches here and
     199                 :            :          * the total may not include its bw.
     200                 :            :          */
     201                 :            :         if (this_bw < tot_bw) {
     202                 :            :                 if (min) {
     203                 :            :                         min *= this_bw;
     204                 :            :                         min = div64_ul(min, tot_bw);
     205                 :            :                 }
     206                 :            :                 if (max < 100) {
     207                 :            :                         max *= this_bw;
     208                 :            :                         max = div64_ul(max, tot_bw);
     209                 :            :                 }
     210                 :            :         }
     211                 :            : 
     212                 :            :         *minp = min;
     213                 :            :         *maxp = max;
     214                 :            : }
     215                 :            : 
     216                 :            : #else   /* CONFIG_CGROUP_WRITEBACK */
     217                 :            : 
     218                 :            : #define GDTC_INIT(__wb)         .wb = (__wb),                           \
     219                 :            :                                 .wb_completions = &(__wb)->completions
     220                 :            : #define GDTC_INIT_NO_WB
     221                 :            : #define MDTC_INIT(__wb, __gdtc)
     222                 :            : 
     223                 :        231 : static bool mdtc_valid(struct dirty_throttle_control *dtc)
     224                 :            : {
     225                 :        231 :         return false;
     226                 :            : }
     227                 :            : 
     228                 :          0 : static struct wb_domain *dtc_dom(struct dirty_throttle_control *dtc)
     229                 :            : {
     230                 :          0 :         return &global_wb_domain;
     231                 :            : }
     232                 :            : 
     233                 :        252 : static struct dirty_throttle_control *mdtc_gdtc(struct dirty_throttle_control *mdtc)
     234                 :            : {
     235                 :        252 :         return NULL;
     236                 :            : }
     237                 :            : 
     238                 :            : static struct fprop_local_percpu *wb_memcg_completions(struct bdi_writeback *wb)
     239                 :            : {
     240                 :            :         return NULL;
     241                 :            : }
     242                 :            : 
     243                 :          0 : static void wb_min_max_ratio(struct bdi_writeback *wb,
     244                 :            :                              unsigned long *minp, unsigned long *maxp)
     245                 :            : {
     246                 :          0 :         *minp = wb->bdi->min_ratio;
     247                 :          0 :         *maxp = wb->bdi->max_ratio;
     248                 :            : }
     249                 :            : 
     250                 :            : #endif  /* CONFIG_CGROUP_WRITEBACK */
     251                 :            : 
     252                 :            : /*
     253                 :            :  * In a memory zone, there is a certain amount of pages we consider
     254                 :            :  * available for the page cache, which is essentially the number of
     255                 :            :  * free and reclaimable pages, minus some zone reserves to protect
     256                 :            :  * lowmem and the ability to uphold the zone's watermarks without
     257                 :            :  * requiring writeback.
     258                 :            :  *
     259                 :            :  * This number of dirtyable pages is the base value of which the
     260                 :            :  * user-configurable dirty ratio is the effictive number of pages that
     261                 :            :  * are allowed to be actually dirtied.  Per individual zone, or
     262                 :            :  * globally by using the sum of dirtyable pages over all zones.
     263                 :            :  *
     264                 :            :  * Because the user is allowed to specify the dirty limit globally as
     265                 :            :  * absolute number of bytes, calculating the per-zone dirty limit can
     266                 :            :  * require translating the configured limit into a percentage of
     267                 :            :  * global dirtyable memory first.
     268                 :            :  */
     269                 :            : 
     270                 :            : /**
     271                 :            :  * node_dirtyable_memory - number of dirtyable pages in a node
     272                 :            :  * @pgdat: the node
     273                 :            :  *
     274                 :            :  * Return: the node's number of pages potentially available for dirty
     275                 :            :  * page cache.  This is the base value for the per-node dirty limits.
     276                 :            :  */
     277                 :       8449 : static unsigned long node_dirtyable_memory(struct pglist_data *pgdat)
     278                 :            : {
     279                 :       8449 :         unsigned long nr_pages = 0;
     280                 :       8449 :         int z;
     281                 :            : 
     282         [ +  + ]:      42245 :         for (z = 0; z < MAX_NR_ZONES; z++) {
     283                 :      33796 :                 struct zone *zone = pgdat->node_zones + z;
     284                 :            : 
     285         [ +  + ]:      33796 :                 if (!populated_zone(zone))
     286                 :      16898 :                         continue;
     287                 :            : 
     288                 :      16898 :                 nr_pages += zone_page_state(zone, NR_FREE_PAGES);
     289                 :            :         }
     290                 :            : 
     291                 :            :         /*
     292                 :            :          * Pages reserved for the kernel should not be considered
     293                 :            :          * dirtyable, to prevent a situation where reclaim has to
     294                 :            :          * clean pages in order to balance the zones.
     295                 :            :          */
     296                 :       8449 :         nr_pages -= min(nr_pages, pgdat->totalreserve_pages);
     297                 :            : 
     298                 :       8449 :         nr_pages += node_page_state(pgdat, NR_INACTIVE_FILE);
     299                 :       8449 :         nr_pages += node_page_state(pgdat, NR_ACTIVE_FILE);
     300                 :            : 
     301                 :       8449 :         return nr_pages;
     302                 :            : }
     303                 :            : 
     304                 :            : static unsigned long highmem_dirtyable_memory(unsigned long total)
     305                 :            : {
     306                 :            : #ifdef CONFIG_HIGHMEM
     307                 :            :         int node;
     308                 :            :         unsigned long x = 0;
     309                 :            :         int i;
     310                 :            : 
     311                 :            :         for_each_node_state(node, N_HIGH_MEMORY) {
     312                 :            :                 for (i = ZONE_NORMAL + 1; i < MAX_NR_ZONES; i++) {
     313                 :            :                         struct zone *z;
     314                 :            :                         unsigned long nr_pages;
     315                 :            : 
     316                 :            :                         if (!is_highmem_idx(i))
     317                 :            :                                 continue;
     318                 :            : 
     319                 :            :                         z = &NODE_DATA(node)->node_zones[i];
     320                 :            :                         if (!populated_zone(z))
     321                 :            :                                 continue;
     322                 :            : 
     323                 :            :                         nr_pages = zone_page_state(z, NR_FREE_PAGES);
     324                 :            :                         /* watch for underflows */
     325                 :            :                         nr_pages -= min(nr_pages, high_wmark_pages(z));
     326                 :            :                         nr_pages += zone_page_state(z, NR_ZONE_INACTIVE_FILE);
     327                 :            :                         nr_pages += zone_page_state(z, NR_ZONE_ACTIVE_FILE);
     328                 :            :                         x += nr_pages;
     329                 :            :                 }
     330                 :            :         }
     331                 :            : 
     332                 :            :         /*
     333                 :            :          * Unreclaimable memory (kernel memory or anonymous memory
     334                 :            :          * without swap) can bring down the dirtyable pages below
     335                 :            :          * the zone's dirty balance reserve and the above calculation
     336                 :            :          * will underflow.  However we still want to add in nodes
     337                 :            :          * which are below threshold (negative values) to get a more
     338                 :            :          * accurate calculation but make sure that the total never
     339                 :            :          * underflows.
     340                 :            :          */
     341                 :            :         if ((long)x < 0)
     342                 :            :                 x = 0;
     343                 :            : 
     344                 :            :         /*
     345                 :            :          * Make sure that the number of highmem pages is never larger
     346                 :            :          * than the number of the total dirtyable memory. This can only
     347                 :            :          * occur in very strange VM situations but we want to make sure
     348                 :            :          * that this does not occur.
     349                 :            :          */
     350                 :            :         return min(x, total);
     351                 :            : #else
     352                 :            :         return 0;
     353                 :            : #endif
     354                 :            : }
     355                 :            : 
     356                 :            : /**
     357                 :            :  * global_dirtyable_memory - number of globally dirtyable pages
     358                 :            :  *
     359                 :            :  * Return: the global number of pages potentially available for dirty
     360                 :            :  * page cache.  This is the base value for the global dirty limits.
     361                 :            :  */
     362                 :        252 : static unsigned long global_dirtyable_memory(void)
     363                 :            : {
     364                 :        252 :         unsigned long x;
     365                 :            : 
     366                 :        252 :         x = global_zone_page_state(NR_FREE_PAGES);
     367                 :            :         /*
     368                 :            :          * Pages reserved for the kernel should not be considered
     369                 :            :          * dirtyable, to prevent a situation where reclaim has to
     370                 :            :          * clean pages in order to balance the zones.
     371                 :            :          */
     372                 :        252 :         x -= min(x, totalreserve_pages);
     373                 :            : 
     374                 :        252 :         x += global_node_page_state(NR_INACTIVE_FILE);
     375                 :        252 :         x += global_node_page_state(NR_ACTIVE_FILE);
     376                 :            : 
     377                 :        252 :         if (!vm_highmem_is_dirtyable)
     378                 :            :                 x -= highmem_dirtyable_memory(x);
     379                 :            : 
     380                 :        252 :         return x + 1;   /* Ensure that we never return 0 */
     381                 :            : }
     382                 :            : 
     383                 :            : /**
     384                 :            :  * domain_dirty_limits - calculate thresh and bg_thresh for a wb_domain
     385                 :            :  * @dtc: dirty_throttle_control of interest
     386                 :            :  *
     387                 :            :  * Calculate @dtc->thresh and ->bg_thresh considering
     388                 :            :  * vm_dirty_{bytes|ratio} and dirty_background_{bytes|ratio}.  The caller
     389                 :            :  * must ensure that @dtc->avail is set before calling this function.  The
     390                 :            :  * dirty limits will be lifted by 1/4 for PF_LESS_THROTTLE (ie. nfsd) and
     391                 :            :  * real-time tasks.
     392                 :            :  */
     393                 :        252 : static void domain_dirty_limits(struct dirty_throttle_control *dtc)
     394                 :            : {
     395                 :        252 :         const unsigned long available_memory = dtc->avail;
     396                 :        252 :         struct dirty_throttle_control *gdtc = mdtc_gdtc(dtc);
     397                 :        252 :         unsigned long bytes = vm_dirty_bytes;
     398                 :        252 :         unsigned long bg_bytes = dirty_background_bytes;
     399                 :            :         /* convert ratios to per-PAGE_SIZE for higher precision */
     400                 :        252 :         unsigned long ratio = (vm_dirty_ratio * PAGE_SIZE) / 100;
     401                 :        252 :         unsigned long bg_ratio = (dirty_background_ratio * PAGE_SIZE) / 100;
     402                 :        252 :         unsigned long thresh;
     403                 :        252 :         unsigned long bg_thresh;
     404                 :        252 :         struct task_struct *tsk;
     405                 :            : 
     406                 :            :         /* gdtc is !NULL iff @dtc is for memcg domain */
     407                 :        252 :         if (gdtc) {
     408                 :            :                 unsigned long global_avail = gdtc->avail;
     409                 :            : 
     410                 :            :                 /*
     411                 :            :                  * The byte settings can't be applied directly to memcg
     412                 :            :                  * domains.  Convert them to ratios by scaling against
     413                 :            :                  * globally available memory.  As the ratios are in
     414                 :            :                  * per-PAGE_SIZE, they can be obtained by dividing bytes by
     415                 :            :                  * number of pages.
     416                 :            :                  */
     417                 :            :                 if (bytes)
     418                 :            :                         ratio = min(DIV_ROUND_UP(bytes, global_avail),
     419                 :            :                                     PAGE_SIZE);
     420                 :            :                 if (bg_bytes)
     421                 :            :                         bg_ratio = min(DIV_ROUND_UP(bg_bytes, global_avail),
     422                 :            :                                        PAGE_SIZE);
     423                 :            :                 bytes = bg_bytes = 0;
     424                 :            :         }
     425                 :            : 
     426         [ -  + ]:        252 :         if (bytes)
     427                 :          0 :                 thresh = DIV_ROUND_UP(bytes, PAGE_SIZE);
     428                 :            :         else
     429                 :        252 :                 thresh = (ratio * available_memory) / PAGE_SIZE;
     430                 :            : 
     431         [ -  + ]:        252 :         if (bg_bytes)
     432                 :          0 :                 bg_thresh = DIV_ROUND_UP(bg_bytes, PAGE_SIZE);
     433                 :            :         else
     434                 :        252 :                 bg_thresh = (bg_ratio * available_memory) / PAGE_SIZE;
     435                 :            : 
     436         [ -  + ]:        252 :         if (bg_thresh >= thresh)
     437                 :          0 :                 bg_thresh = thresh / 2;
     438         [ +  - ]:        252 :         tsk = current;
     439         [ +  - ]:        252 :         if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk)) {
     440                 :          0 :                 bg_thresh += bg_thresh / 4 + global_wb_domain.dirty_limit / 32;
     441                 :          0 :                 thresh += thresh / 4 + global_wb_domain.dirty_limit / 32;
     442                 :            :         }
     443                 :        252 :         dtc->thresh = thresh;
     444                 :        252 :         dtc->bg_thresh = bg_thresh;
     445                 :            : 
     446                 :            :         /* we should eventually report the domain in the TP */
     447                 :        252 :         if (!gdtc)
     448                 :        252 :                 trace_global_dirty_state(bg_thresh, thresh);
     449                 :        252 : }
     450                 :            : 
     451                 :            : /**
     452                 :            :  * global_dirty_limits - background-writeback and dirty-throttling thresholds
     453                 :            :  * @pbackground: out parameter for bg_thresh
     454                 :            :  * @pdirty: out parameter for thresh
     455                 :            :  *
     456                 :            :  * Calculate bg_thresh and thresh for global_wb_domain.  See
     457                 :            :  * domain_dirty_limits() for details.
     458                 :            :  */
     459                 :         21 : void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty)
     460                 :            : {
     461                 :         21 :         struct dirty_throttle_control gdtc = { GDTC_INIT_NO_WB };
     462                 :            : 
     463                 :         21 :         gdtc.avail = global_dirtyable_memory();
     464                 :         21 :         domain_dirty_limits(&gdtc);
     465                 :            : 
     466                 :         21 :         *pbackground = gdtc.bg_thresh;
     467                 :         21 :         *pdirty = gdtc.thresh;
     468                 :         21 : }
     469                 :            : 
     470                 :            : /**
     471                 :            :  * node_dirty_limit - maximum number of dirty pages allowed in a node
     472                 :            :  * @pgdat: the node
     473                 :            :  *
     474                 :            :  * Return: the maximum number of dirty pages allowed in a node, based
     475                 :            :  * on the node's dirtyable memory.
     476                 :            :  */
     477                 :       8449 : static unsigned long node_dirty_limit(struct pglist_data *pgdat)
     478                 :            : {
     479                 :       8449 :         unsigned long node_memory = node_dirtyable_memory(pgdat);
     480         [ -  + ]:       8449 :         struct task_struct *tsk = current;
     481                 :       8449 :         unsigned long dirty;
     482                 :            : 
     483         [ -  + ]:       8449 :         if (vm_dirty_bytes)
     484                 :          0 :                 dirty = DIV_ROUND_UP(vm_dirty_bytes, PAGE_SIZE) *
     485                 :          0 :                         node_memory / global_dirtyable_memory();
     486                 :            :         else
     487                 :       8449 :                 dirty = vm_dirty_ratio * node_memory / 100;
     488                 :            : 
     489         [ +  - ]:       8449 :         if (tsk->flags & PF_LESS_THROTTLE || rt_task(tsk))
     490                 :          0 :                 dirty += dirty / 4;
     491                 :            : 
     492                 :       8449 :         return dirty;
     493                 :            : }
     494                 :            : 
     495                 :            : /**
     496                 :            :  * node_dirty_ok - tells whether a node is within its dirty limits
     497                 :            :  * @pgdat: the node to check
     498                 :            :  *
     499                 :            :  * Return: %true when the dirty pages in @pgdat are within the node's
     500                 :            :  * dirty limit, %false if the limit is exceeded.
     501                 :            :  */
     502                 :       8449 : bool node_dirty_ok(struct pglist_data *pgdat)
     503                 :            : {
     504                 :       8449 :         unsigned long limit = node_dirty_limit(pgdat);
     505                 :       8449 :         unsigned long nr_pages = 0;
     506                 :            : 
     507                 :       8449 :         nr_pages += node_page_state(pgdat, NR_FILE_DIRTY);
     508                 :       8449 :         nr_pages += node_page_state(pgdat, NR_UNSTABLE_NFS);
     509                 :       8449 :         nr_pages += node_page_state(pgdat, NR_WRITEBACK);
     510                 :            : 
     511                 :       8449 :         return nr_pages <= limit;
     512                 :            : }
     513                 :            : 
     514                 :          0 : int dirty_background_ratio_handler(struct ctl_table *table, int write,
     515                 :            :                 void __user *buffer, size_t *lenp,
     516                 :            :                 loff_t *ppos)
     517                 :            : {
     518                 :          0 :         int ret;
     519                 :            : 
     520                 :          0 :         ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
     521         [ #  # ]:          0 :         if (ret == 0 && write)
     522                 :          0 :                 dirty_background_bytes = 0;
     523                 :          0 :         return ret;
     524                 :            : }
     525                 :            : 
     526                 :          0 : int dirty_background_bytes_handler(struct ctl_table *table, int write,
     527                 :            :                 void __user *buffer, size_t *lenp,
     528                 :            :                 loff_t *ppos)
     529                 :            : {
     530                 :          0 :         int ret;
     531                 :            : 
     532                 :          0 :         ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
     533         [ #  # ]:          0 :         if (ret == 0 && write)
     534                 :          0 :                 dirty_background_ratio = 0;
     535                 :          0 :         return ret;
     536                 :            : }
     537                 :            : 
     538                 :          0 : int dirty_ratio_handler(struct ctl_table *table, int write,
     539                 :            :                 void __user *buffer, size_t *lenp,
     540                 :            :                 loff_t *ppos)
     541                 :            : {
     542                 :          0 :         int old_ratio = vm_dirty_ratio;
     543                 :          0 :         int ret;
     544                 :            : 
     545                 :          0 :         ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
     546   [ #  #  #  # ]:          0 :         if (ret == 0 && write && vm_dirty_ratio != old_ratio) {
     547                 :          0 :                 writeback_set_ratelimit();
     548                 :          0 :                 vm_dirty_bytes = 0;
     549                 :            :         }
     550                 :          0 :         return ret;
     551                 :            : }
     552                 :            : 
     553                 :          0 : int dirty_bytes_handler(struct ctl_table *table, int write,
     554                 :            :                 void __user *buffer, size_t *lenp,
     555                 :            :                 loff_t *ppos)
     556                 :            : {
     557                 :          0 :         unsigned long old_bytes = vm_dirty_bytes;
     558                 :          0 :         int ret;
     559                 :            : 
     560                 :          0 :         ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
     561   [ #  #  #  # ]:          0 :         if (ret == 0 && write && vm_dirty_bytes != old_bytes) {
     562                 :          0 :                 writeback_set_ratelimit();
     563                 :          0 :                 vm_dirty_ratio = 0;
     564                 :            :         }
     565                 :          0 :         return ret;
     566                 :            : }
     567                 :            : 
     568                 :         21 : static unsigned long wp_next_time(unsigned long cur_time)
     569                 :            : {
     570                 :         21 :         cur_time += VM_COMPLETIONS_PERIOD_LEN;
     571                 :            :         /* 0 has a special meaning... */
     572                 :         21 :         if (!cur_time)
     573                 :            :                 return 1;
     574                 :            :         return cur_time;
     575                 :            : }
     576                 :            : 
     577                 :        168 : static void wb_domain_writeout_inc(struct wb_domain *dom,
     578                 :            :                                    struct fprop_local_percpu *completions,
     579                 :            :                                    unsigned int max_prop_frac)
     580                 :            : {
     581                 :        168 :         __fprop_inc_percpu_max(&dom->completions, completions,
     582                 :            :                                max_prop_frac);
     583                 :            :         /* First event after period switching was turned off? */
     584         [ +  + ]:        168 :         if (unlikely(!dom->period_time)) {
     585                 :            :                 /*
     586                 :            :                  * We can race with other __bdi_writeout_inc calls here but
     587                 :            :                  * it does not cause any harm since the resulting time when
     588                 :            :                  * timer will fire and what is in writeout_period_time will be
     589                 :            :                  * roughly the same.
     590                 :            :                  */
     591                 :         21 :                 dom->period_time = wp_next_time(jiffies);
     592                 :         21 :                 mod_timer(&dom->period_timer, dom->period_time);
     593                 :            :         }
     594                 :        168 : }
     595                 :            : 
     596                 :            : /*
     597                 :            :  * Increment @wb's writeout completion count and the global writeout
     598                 :            :  * completion count. Called from test_clear_page_writeback().
     599                 :            :  */
     600                 :        168 : static inline void __wb_writeout_inc(struct bdi_writeback *wb)
     601                 :            : {
     602                 :        168 :         struct wb_domain *cgdom;
     603                 :            : 
     604                 :        168 :         inc_wb_stat(wb, WB_WRITTEN);
     605                 :        168 :         wb_domain_writeout_inc(&global_wb_domain, &wb->completions,
     606                 :        168 :                                wb->bdi->max_prop_frac);
     607                 :            : 
     608                 :        168 :         cgdom = mem_cgroup_wb_domain(wb);
     609                 :        168 :         if (cgdom)
     610                 :            :                 wb_domain_writeout_inc(cgdom, wb_memcg_completions(wb),
     611                 :            :                                        wb->bdi->max_prop_frac);
     612                 :        168 : }
     613                 :            : 
     614                 :          0 : void wb_writeout_inc(struct bdi_writeback *wb)
     615                 :            : {
     616                 :          0 :         unsigned long flags;
     617                 :            : 
     618                 :          0 :         local_irq_save(flags);
     619                 :          0 :         __wb_writeout_inc(wb);
     620                 :          0 :         local_irq_restore(flags);
     621                 :          0 : }
     622                 :            : EXPORT_SYMBOL_GPL(wb_writeout_inc);
     623                 :            : 
     624                 :            : /*
     625                 :            :  * On idle system, we can be called long after we scheduled because we use
     626                 :            :  * deferred timers so count with missed periods.
     627                 :            :  */
     628                 :          0 : static void writeout_period(struct timer_list *t)
     629                 :            : {
     630                 :          0 :         struct wb_domain *dom = from_timer(dom, t, period_timer);
     631                 :          0 :         int miss_periods = (jiffies - dom->period_time) /
     632                 :            :                                                  VM_COMPLETIONS_PERIOD_LEN;
     633                 :            : 
     634         [ #  # ]:          0 :         if (fprop_new_period(&dom->completions, miss_periods + 1)) {
     635                 :          0 :                 dom->period_time = wp_next_time(dom->period_time +
     636                 :          0 :                                 miss_periods * VM_COMPLETIONS_PERIOD_LEN);
     637                 :          0 :                 mod_timer(&dom->period_timer, dom->period_time);
     638                 :            :         } else {
     639                 :            :                 /*
     640                 :            :                  * Aging has zeroed all fractions. Stop wasting CPU on period
     641                 :            :                  * updates.
     642                 :            :                  */
     643                 :          0 :                 dom->period_time = 0;
     644                 :            :         }
     645                 :          0 : }
     646                 :            : 
     647                 :         21 : int wb_domain_init(struct wb_domain *dom, gfp_t gfp)
     648                 :            : {
     649                 :         21 :         memset(dom, 0, sizeof(*dom));
     650                 :            : 
     651                 :         21 :         spin_lock_init(&dom->lock);
     652                 :            : 
     653                 :         21 :         timer_setup(&dom->period_timer, writeout_period, TIMER_DEFERRABLE);
     654                 :            : 
     655                 :         21 :         dom->dirty_limit_tstamp = jiffies;
     656                 :            : 
     657                 :         21 :         return fprop_global_init(&dom->completions, gfp);
     658                 :            : }
     659                 :            : 
     660                 :            : #ifdef CONFIG_CGROUP_WRITEBACK
     661                 :            : void wb_domain_exit(struct wb_domain *dom)
     662                 :            : {
     663                 :            :         del_timer_sync(&dom->period_timer);
     664                 :            :         fprop_global_destroy(&dom->completions);
     665                 :            : }
     666                 :            : #endif
     667                 :            : 
     668                 :            : /*
     669                 :            :  * bdi_min_ratio keeps the sum of the minimum dirty shares of all
     670                 :            :  * registered backing devices, which, for obvious reasons, can not
     671                 :            :  * exceed 100%.
     672                 :            :  */
     673                 :            : static unsigned int bdi_min_ratio;
     674                 :            : 
     675                 :          0 : int bdi_set_min_ratio(struct backing_dev_info *bdi, unsigned int min_ratio)
     676                 :            : {
     677                 :          0 :         int ret = 0;
     678                 :            : 
     679                 :          0 :         spin_lock_bh(&bdi_lock);
     680         [ #  # ]:          0 :         if (min_ratio > bdi->max_ratio) {
     681                 :            :                 ret = -EINVAL;
     682                 :            :         } else {
     683                 :          0 :                 min_ratio -= bdi->min_ratio;
     684         [ #  # ]:          0 :                 if (bdi_min_ratio + min_ratio < 100) {
     685                 :          0 :                         bdi_min_ratio += min_ratio;
     686                 :          0 :                         bdi->min_ratio += min_ratio;
     687                 :            :                 } else {
     688                 :            :                         ret = -EINVAL;
     689                 :            :                 }
     690                 :            :         }
     691                 :          0 :         spin_unlock_bh(&bdi_lock);
     692                 :            : 
     693                 :          0 :         return ret;
     694                 :            : }
     695                 :            : 
     696                 :          0 : int bdi_set_max_ratio(struct backing_dev_info *bdi, unsigned max_ratio)
     697                 :            : {
     698                 :          0 :         int ret = 0;
     699                 :            : 
     700         [ #  # ]:          0 :         if (max_ratio > 100)
     701                 :            :                 return -EINVAL;
     702                 :            : 
     703                 :          0 :         spin_lock_bh(&bdi_lock);
     704         [ #  # ]:          0 :         if (bdi->min_ratio > max_ratio) {
     705                 :            :                 ret = -EINVAL;
     706                 :            :         } else {
     707                 :          0 :                 bdi->max_ratio = max_ratio;
     708                 :          0 :                 bdi->max_prop_frac = (FPROP_FRAC_BASE * max_ratio) / 100;
     709                 :            :         }
     710                 :          0 :         spin_unlock_bh(&bdi_lock);
     711                 :            : 
     712                 :          0 :         return ret;
     713                 :            : }
     714                 :            : EXPORT_SYMBOL(bdi_set_max_ratio);
     715                 :            : 
     716                 :        231 : static unsigned long dirty_freerun_ceiling(unsigned long thresh,
     717                 :            :                                            unsigned long bg_thresh)
     718                 :            : {
     719                 :        231 :         return (thresh + bg_thresh) / 2;
     720                 :            : }
     721                 :            : 
     722                 :          0 : static unsigned long hard_dirty_limit(struct wb_domain *dom,
     723                 :            :                                       unsigned long thresh)
     724                 :            : {
     725                 :          0 :         return max(thresh, dom->dirty_limit);
     726                 :            : }
     727                 :            : 
     728                 :            : /*
     729                 :            :  * Memory which can be further allocated to a memcg domain is capped by
     730                 :            :  * system-wide clean memory excluding the amount being used in the domain.
     731                 :            :  */
     732                 :            : static void mdtc_calc_avail(struct dirty_throttle_control *mdtc,
     733                 :            :                             unsigned long filepages, unsigned long headroom)
     734                 :            : {
     735                 :            :         struct dirty_throttle_control *gdtc = mdtc_gdtc(mdtc);
     736                 :            :         unsigned long clean = filepages - min(filepages, mdtc->dirty);
     737                 :            :         unsigned long global_clean = gdtc->avail - min(gdtc->avail, gdtc->dirty);
     738                 :            :         unsigned long other_clean = global_clean - min(global_clean, clean);
     739                 :            : 
     740                 :            :         mdtc->avail = filepages + min(headroom, other_clean);
     741                 :            : }
     742                 :            : 
     743                 :            : /**
     744                 :            :  * __wb_calc_thresh - @wb's share of dirty throttling threshold
     745                 :            :  * @dtc: dirty_throttle_context of interest
     746                 :            :  *
     747                 :            :  * Note that balance_dirty_pages() will only seriously take it as a hard limit
     748                 :            :  * when sleeping max_pause per page is not enough to keep the dirty pages under
     749                 :            :  * control. For example, when the device is completely stalled due to some error
     750                 :            :  * conditions, or when there are 1000 dd tasks writing to a slow 10MB/s USB key.
     751                 :            :  * In the other normal situations, it acts more gently by throttling the tasks
     752                 :            :  * more (rather than completely block them) when the wb dirty pages go high.
     753                 :            :  *
     754                 :            :  * It allocates high/low dirty limits to fast/slow devices, in order to prevent
     755                 :            :  * - starving fast devices
     756                 :            :  * - piling up dirty pages (that will take long time to sync) on slow devices
     757                 :            :  *
     758                 :            :  * The wb's share of dirty limit will be adapting to its throughput and
     759                 :            :  * bounded by the bdi->min_ratio and/or bdi->max_ratio parameters, if set.
     760                 :            :  *
     761                 :            :  * Return: @wb's dirty limit in pages. The term "dirty" in the context of
     762                 :            :  * dirty balancing includes all PG_dirty, PG_writeback and NFS unstable pages.
     763                 :            :  */
     764                 :          0 : static unsigned long __wb_calc_thresh(struct dirty_throttle_control *dtc)
     765                 :            : {
     766                 :          0 :         struct wb_domain *dom = dtc_dom(dtc);
     767                 :          0 :         unsigned long thresh = dtc->thresh;
     768                 :          0 :         u64 wb_thresh;
     769                 :          0 :         unsigned long numerator, denominator;
     770                 :          0 :         unsigned long wb_min_ratio, wb_max_ratio;
     771                 :            : 
     772                 :            :         /*
     773                 :            :          * Calculate this BDI's share of the thresh ratio.
     774                 :            :          */
     775                 :          0 :         fprop_fraction_percpu(&dom->completions, dtc->wb_completions,
     776                 :            :                               &numerator, &denominator);
     777                 :            : 
     778                 :          0 :         wb_thresh = (thresh * (100 - bdi_min_ratio)) / 100;
     779                 :          0 :         wb_thresh *= numerator;
     780                 :          0 :         wb_thresh = div64_ul(wb_thresh, denominator);
     781                 :            : 
     782                 :          0 :         wb_min_max_ratio(dtc->wb, &wb_min_ratio, &wb_max_ratio);
     783                 :            : 
     784                 :          0 :         wb_thresh += (thresh * wb_min_ratio) / 100;
     785                 :          0 :         if (wb_thresh > (thresh * wb_max_ratio) / 100)
     786                 :            :                 wb_thresh = thresh * wb_max_ratio / 100;
     787                 :            : 
     788                 :          0 :         return wb_thresh;
     789                 :            : }
     790                 :            : 
     791                 :          0 : unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh)
     792                 :            : {
     793                 :          0 :         struct dirty_throttle_control gdtc = { GDTC_INIT(wb),
     794                 :            :                                                .thresh = thresh };
     795                 :          0 :         return __wb_calc_thresh(&gdtc);
     796                 :            : }
     797                 :            : 
     798                 :            : /*
     799                 :            :  *                           setpoint - dirty 3
     800                 :            :  *        f(dirty) := 1.0 + (----------------)
     801                 :            :  *                           limit - setpoint
     802                 :            :  *
     803                 :            :  * it's a 3rd order polynomial that subjects to
     804                 :            :  *
     805                 :            :  * (1) f(freerun)  = 2.0 => rampup dirty_ratelimit reasonably fast
     806                 :            :  * (2) f(setpoint) = 1.0 => the balance point
     807                 :            :  * (3) f(limit)    = 0   => the hard limit
     808                 :            :  * (4) df/dx      <= 0        => negative feedback control
     809                 :            :  * (5) the closer to setpoint, the smaller |df/dx| (and the reverse)
     810                 :            :  *     => fast response on large errors; small oscillation near setpoint
     811                 :            :  */
     812                 :          0 : static long long pos_ratio_polynom(unsigned long setpoint,
     813                 :            :                                           unsigned long dirty,
     814                 :            :                                           unsigned long limit)
     815                 :            : {
     816                 :          0 :         long long pos_ratio;
     817                 :          0 :         long x;
     818                 :            : 
     819                 :          0 :         x = div64_s64(((s64)setpoint - (s64)dirty) << RATELIMIT_CALC_SHIFT,
     820                 :          0 :                       (limit - setpoint) | 1);
     821                 :          0 :         pos_ratio = x;
     822                 :          0 :         pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
     823                 :          0 :         pos_ratio = pos_ratio * x >> RATELIMIT_CALC_SHIFT;
     824                 :          0 :         pos_ratio += 1 << RATELIMIT_CALC_SHIFT;
     825                 :            : 
     826                 :          0 :         return clamp(pos_ratio, 0LL, 2LL << RATELIMIT_CALC_SHIFT);
     827                 :            : }
     828                 :            : 
     829                 :            : /*
     830                 :            :  * Dirty position control.
     831                 :            :  *
     832                 :            :  * (o) global/bdi setpoints
     833                 :            :  *
     834                 :            :  * We want the dirty pages be balanced around the global/wb setpoints.
     835                 :            :  * When the number of dirty pages is higher/lower than the setpoint, the
     836                 :            :  * dirty position control ratio (and hence task dirty ratelimit) will be
     837                 :            :  * decreased/increased to bring the dirty pages back to the setpoint.
     838                 :            :  *
     839                 :            :  *     pos_ratio = 1 << RATELIMIT_CALC_SHIFT
     840                 :            :  *
     841                 :            :  *     if (dirty < setpoint) scale up   pos_ratio
     842                 :            :  *     if (dirty > setpoint) scale down pos_ratio
     843                 :            :  *
     844                 :            :  *     if (wb_dirty < wb_setpoint) scale up   pos_ratio
     845                 :            :  *     if (wb_dirty > wb_setpoint) scale down pos_ratio
     846                 :            :  *
     847                 :            :  *     task_ratelimit = dirty_ratelimit * pos_ratio >> RATELIMIT_CALC_SHIFT
     848                 :            :  *
     849                 :            :  * (o) global control line
     850                 :            :  *
     851                 :            :  *     ^ pos_ratio
     852                 :            :  *     |
     853                 :            :  *     |            |<===== global dirty control scope ======>|
     854                 :            :  * 2.0 .............*
     855                 :            :  *     |            .*
     856                 :            :  *     |            . *
     857                 :            :  *     |            .   *
     858                 :            :  *     |            .     *
     859                 :            :  *     |            .        *
     860                 :            :  *     |            .            *
     861                 :            :  * 1.0 ................................*
     862                 :            :  *     |            .                  .     *
     863                 :            :  *     |            .                  .          *
     864                 :            :  *     |            .                  .              *
     865                 :            :  *     |            .                  .                 *
     866                 :            :  *     |            .                  .                    *
     867                 :            :  *   0 +------------.------------------.----------------------*------------->
     868                 :            :  *           freerun^          setpoint^                 limit^   dirty pages
     869                 :            :  *
     870                 :            :  * (o) wb control line
     871                 :            :  *
     872                 :            :  *     ^ pos_ratio
     873                 :            :  *     |
     874                 :            :  *     |            *
     875                 :            :  *     |              *
     876                 :            :  *     |                *
     877                 :            :  *     |                  *
     878                 :            :  *     |                    * |<=========== span ============>|
     879                 :            :  * 1.0 .......................*
     880                 :            :  *     |                      . *
     881                 :            :  *     |                      .   *
     882                 :            :  *     |                      .     *
     883                 :            :  *     |                      .       *
     884                 :            :  *     |                      .         *
     885                 :            :  *     |                      .           *
     886                 :            :  *     |                      .             *
     887                 :            :  *     |                      .               *
     888                 :            :  *     |                      .                 *
     889                 :            :  *     |                      .                   *
     890                 :            :  *     |                      .                     *
     891                 :            :  * 1/4 ...............................................* * * * * * * * * * * *
     892                 :            :  *     |                      .                         .
     893                 :            :  *     |                      .                           .
     894                 :            :  *     |                      .                             .
     895                 :            :  *   0 +----------------------.-------------------------------.------------->
     896                 :            :  *                wb_setpoint^                    x_intercept^
     897                 :            :  *
     898                 :            :  * The wb control line won't drop below pos_ratio=1/4, so that wb_dirty can
     899                 :            :  * be smoothly throttled down to normal if it starts high in situations like
     900                 :            :  * - start writing to a slow SD card and a fast disk at the same time. The SD
     901                 :            :  *   card's wb_dirty may rush to many times higher than wb_setpoint.
     902                 :            :  * - the wb dirty thresh drops quickly due to change of JBOD workload
     903                 :            :  */
     904                 :          0 : static void wb_position_ratio(struct dirty_throttle_control *dtc)
     905                 :            : {
     906                 :          0 :         struct bdi_writeback *wb = dtc->wb;
     907                 :          0 :         unsigned long write_bw = wb->avg_write_bandwidth;
     908                 :          0 :         unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh);
     909                 :          0 :         unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh);
     910                 :          0 :         unsigned long wb_thresh = dtc->wb_thresh;
     911                 :          0 :         unsigned long x_intercept;
     912                 :          0 :         unsigned long setpoint;         /* dirty pages' target balance point */
     913                 :          0 :         unsigned long wb_setpoint;
     914                 :          0 :         unsigned long span;
     915                 :          0 :         long long pos_ratio;            /* for scaling up/down the rate limit */
     916                 :          0 :         long x;
     917                 :            : 
     918                 :          0 :         dtc->pos_ratio = 0;
     919                 :            : 
     920         [ #  # ]:          0 :         if (unlikely(dtc->dirty >= limit))
     921                 :            :                 return;
     922                 :            : 
     923                 :            :         /*
     924                 :            :          * global setpoint
     925                 :            :          *
     926                 :            :          * See comment for pos_ratio_polynom().
     927                 :            :          */
     928                 :          0 :         setpoint = (freerun + limit) / 2;
     929         [ #  # ]:          0 :         pos_ratio = pos_ratio_polynom(setpoint, dtc->dirty, limit);
     930                 :            : 
     931                 :            :         /*
     932                 :            :          * The strictlimit feature is a tool preventing mistrusted filesystems
     933                 :            :          * from growing a large number of dirty pages before throttling. For
     934                 :            :          * such filesystems balance_dirty_pages always checks wb counters
     935                 :            :          * against wb limits. Even if global "nr_dirty" is under "freerun".
     936                 :            :          * This is especially important for fuse which sets bdi->max_ratio to
     937                 :            :          * 1% by default. Without strictlimit feature, fuse writeback may
     938                 :            :          * consume arbitrary amount of RAM because it is accounted in
     939                 :            :          * NR_WRITEBACK_TEMP which is not involved in calculating "nr_dirty".
     940                 :            :          *
     941                 :            :          * Here, in wb_position_ratio(), we calculate pos_ratio based on
     942                 :            :          * two values: wb_dirty and wb_thresh. Let's consider an example:
     943                 :            :          * total amount of RAM is 16GB, bdi->max_ratio is equal to 1%, global
     944                 :            :          * limits are set by default to 10% and 20% (background and throttle).
     945                 :            :          * Then wb_thresh is 1% of 20% of 16GB. This amounts to ~8K pages.
     946                 :            :          * wb_calc_thresh(wb, bg_thresh) is about ~4K pages. wb_setpoint is
     947                 :            :          * about ~6K pages (as the average of background and throttle wb
     948                 :            :          * limits). The 3rd order polynomial will provide positive feedback if
     949                 :            :          * wb_dirty is under wb_setpoint and vice versa.
     950                 :            :          *
     951                 :            :          * Note, that we cannot use global counters in these calculations
     952                 :            :          * because we want to throttle process writing to a strictlimit wb
     953                 :            :          * much earlier than global "freerun" is reached (~23MB vs. ~2.3GB
     954                 :            :          * in the example above).
     955                 :            :          */
     956         [ #  # ]:          0 :         if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) {
     957                 :          0 :                 long long wb_pos_ratio;
     958                 :            : 
     959         [ #  # ]:          0 :                 if (dtc->wb_dirty < 8) {
     960                 :          0 :                         dtc->pos_ratio = min_t(long long, pos_ratio * 2,
     961                 :            :                                            2 << RATELIMIT_CALC_SHIFT);
     962                 :          0 :                         return;
     963                 :            :                 }
     964                 :            : 
     965         [ #  # ]:          0 :                 if (dtc->wb_dirty >= wb_thresh)
     966                 :            :                         return;
     967                 :            : 
     968                 :          0 :                 wb_setpoint = dirty_freerun_ceiling(wb_thresh,
     969                 :            :                                                     dtc->wb_bg_thresh);
     970                 :            : 
     971         [ #  # ]:          0 :                 if (wb_setpoint == 0 || wb_setpoint == wb_thresh)
     972                 :            :                         return;
     973                 :            : 
     974                 :          0 :                 wb_pos_ratio = pos_ratio_polynom(wb_setpoint, dtc->wb_dirty,
     975                 :            :                                                  wb_thresh);
     976                 :            : 
     977                 :            :                 /*
     978                 :            :                  * Typically, for strictlimit case, wb_setpoint << setpoint
     979                 :            :                  * and pos_ratio >> wb_pos_ratio. In the other words global
     980                 :            :                  * state ("dirty") is not limiting factor and we have to
     981                 :            :                  * make decision based on wb counters. But there is an
     982                 :            :                  * important case when global pos_ratio should get precedence:
     983                 :            :                  * global limits are exceeded (e.g. due to activities on other
     984                 :            :                  * wb's) while given strictlimit wb is below limit.
     985                 :            :                  *
     986                 :            :                  * "pos_ratio * wb_pos_ratio" would work for the case above,
     987                 :            :                  * but it would look too non-natural for the case of all
     988                 :            :                  * activity in the system coming from a single strictlimit wb
     989                 :            :                  * with bdi->max_ratio == 100%.
     990                 :            :                  *
     991                 :            :                  * Note that min() below somewhat changes the dynamics of the
     992                 :            :                  * control system. Normally, pos_ratio value can be well over 3
     993                 :            :                  * (when globally we are at freerun and wb is well below wb
     994                 :            :                  * setpoint). Now the maximum pos_ratio in the same situation
     995                 :            :                  * is 2. We might want to tweak this if we observe the control
     996                 :            :                  * system is too slow to adapt.
     997                 :            :                  */
     998                 :          0 :                 dtc->pos_ratio = min(pos_ratio, wb_pos_ratio);
     999                 :          0 :                 return;
    1000                 :            :         }
    1001                 :            : 
    1002                 :            :         /*
    1003                 :            :          * We have computed basic pos_ratio above based on global situation. If
    1004                 :            :          * the wb is over/under its share of dirty pages, we want to scale
    1005                 :            :          * pos_ratio further down/up. That is done by the following mechanism.
    1006                 :            :          */
    1007                 :            : 
    1008                 :            :         /*
    1009                 :            :          * wb setpoint
    1010                 :            :          *
    1011                 :            :          *        f(wb_dirty) := 1.0 + k * (wb_dirty - wb_setpoint)
    1012                 :            :          *
    1013                 :            :          *                        x_intercept - wb_dirty
    1014                 :            :          *                     := --------------------------
    1015                 :            :          *                        x_intercept - wb_setpoint
    1016                 :            :          *
    1017                 :            :          * The main wb control line is a linear function that subjects to
    1018                 :            :          *
    1019                 :            :          * (1) f(wb_setpoint) = 1.0
    1020                 :            :          * (2) k = - 1 / (8 * write_bw)  (in single wb case)
    1021                 :            :          *     or equally: x_intercept = wb_setpoint + 8 * write_bw
    1022                 :            :          *
    1023                 :            :          * For single wb case, the dirty pages are observed to fluctuate
    1024                 :            :          * regularly within range
    1025                 :            :          *        [wb_setpoint - write_bw/2, wb_setpoint + write_bw/2]
    1026                 :            :          * for various filesystems, where (2) can yield in a reasonable 12.5%
    1027                 :            :          * fluctuation range for pos_ratio.
    1028                 :            :          *
    1029                 :            :          * For JBOD case, wb_thresh (not wb_dirty!) could fluctuate up to its
    1030                 :            :          * own size, so move the slope over accordingly and choose a slope that
    1031                 :            :          * yields 100% pos_ratio fluctuation on suddenly doubled wb_thresh.
    1032                 :            :          */
    1033         [ #  # ]:          0 :         if (unlikely(wb_thresh > dtc->thresh))
    1034                 :          0 :                 wb_thresh = dtc->thresh;
    1035                 :            :         /*
    1036                 :            :          * It's very possible that wb_thresh is close to 0 not because the
    1037                 :            :          * device is slow, but that it has remained inactive for long time.
    1038                 :            :          * Honour such devices a reasonable good (hopefully IO efficient)
    1039                 :            :          * threshold, so that the occasional writes won't be blocked and active
    1040                 :            :          * writes can rampup the threshold quickly.
    1041                 :            :          */
    1042                 :          0 :         wb_thresh = max(wb_thresh, (limit - dtc->dirty) / 8);
    1043                 :            :         /*
    1044                 :            :          * scale global setpoint to wb's:
    1045                 :            :          *      wb_setpoint = setpoint * wb_thresh / thresh
    1046                 :            :          */
    1047         [ #  # ]:          0 :         x = div_u64((u64)wb_thresh << 16, dtc->thresh | 1);
    1048                 :          0 :         wb_setpoint = setpoint * (u64)x >> 16;
    1049                 :            :         /*
    1050                 :            :          * Use span=(8*write_bw) in single wb case as indicated by
    1051                 :            :          * (thresh - wb_thresh ~= 0) and transit to wb_thresh in JBOD case.
    1052                 :            :          *
    1053                 :            :          *        wb_thresh                    thresh - wb_thresh
    1054                 :            :          * span = --------- * (8 * write_bw) + ------------------ * wb_thresh
    1055                 :            :          *         thresh                           thresh
    1056                 :            :          */
    1057                 :          0 :         span = (dtc->thresh - wb_thresh + 8 * write_bw) * (u64)x >> 16;
    1058                 :          0 :         x_intercept = wb_setpoint + span;
    1059                 :            : 
    1060         [ #  # ]:          0 :         if (dtc->wb_dirty < x_intercept - span / 4) {
    1061                 :          0 :                 pos_ratio = div64_u64(pos_ratio * (x_intercept - dtc->wb_dirty),
    1062                 :          0 :                                       (x_intercept - wb_setpoint) | 1);
    1063                 :            :         } else
    1064                 :          0 :                 pos_ratio /= 4;
    1065                 :            : 
    1066                 :            :         /*
    1067                 :            :          * wb reserve area, safeguard against dirty pool underrun and disk idle
    1068                 :            :          * It may push the desired control point of global dirty pages higher
    1069                 :            :          * than setpoint.
    1070                 :            :          */
    1071                 :          0 :         x_intercept = wb_thresh / 2;
    1072         [ #  # ]:          0 :         if (dtc->wb_dirty < x_intercept) {
    1073         [ #  # ]:          0 :                 if (dtc->wb_dirty > x_intercept / 8)
    1074                 :          0 :                         pos_ratio = div_u64(pos_ratio * x_intercept,
    1075                 :            :                                             dtc->wb_dirty);
    1076                 :            :                 else
    1077                 :          0 :                         pos_ratio *= 8;
    1078                 :            :         }
    1079                 :            : 
    1080                 :          0 :         dtc->pos_ratio = pos_ratio;
    1081                 :            : }
    1082                 :            : 
    1083                 :          0 : static void wb_update_write_bandwidth(struct bdi_writeback *wb,
    1084                 :            :                                       unsigned long elapsed,
    1085                 :            :                                       unsigned long written)
    1086                 :            : {
    1087                 :          0 :         const unsigned long period = roundup_pow_of_two(3 * HZ);
    1088                 :          0 :         unsigned long avg = wb->avg_write_bandwidth;
    1089                 :          0 :         unsigned long old = wb->write_bandwidth;
    1090                 :          0 :         u64 bw;
    1091                 :            : 
    1092                 :            :         /*
    1093                 :            :          * bw = written * HZ / elapsed
    1094                 :            :          *
    1095                 :            :          *                   bw * elapsed + write_bandwidth * (period - elapsed)
    1096                 :            :          * write_bandwidth = ---------------------------------------------------
    1097                 :            :          *                                          period
    1098                 :            :          *
    1099                 :            :          * @written may have decreased due to account_page_redirty().
    1100                 :            :          * Avoid underflowing @bw calculation.
    1101                 :            :          */
    1102                 :          0 :         bw = written - min(written, wb->written_stamp);
    1103                 :          0 :         bw *= HZ;
    1104         [ #  # ]:          0 :         if (unlikely(elapsed > period)) {
    1105                 :          0 :                 bw = div64_ul(bw, elapsed);
    1106                 :          0 :                 avg = bw;
    1107                 :          0 :                 goto out;
    1108                 :            :         }
    1109                 :          0 :         bw += (u64)wb->write_bandwidth * (period - elapsed);
    1110                 :          0 :         bw >>= ilog2(period);
    1111                 :            : 
    1112                 :            :         /*
    1113                 :            :          * one more level of smoothing, for filtering out sudden spikes
    1114                 :            :          */
    1115         [ #  # ]:          0 :         if (avg > old && old >= (unsigned long)bw)
    1116                 :          0 :                 avg -= (avg - old) >> 3;
    1117                 :            : 
    1118         [ #  # ]:          0 :         if (avg < old && old <= (unsigned long)bw)
    1119                 :          0 :                 avg += (old - avg) >> 3;
    1120                 :            : 
    1121                 :          0 : out:
    1122                 :            :         /* keep avg > 0 to guarantee that tot > 0 if there are dirty wbs */
    1123                 :          0 :         avg = max(avg, 1LU);
    1124         [ #  # ]:          0 :         if (wb_has_dirty_io(wb)) {
    1125                 :          0 :                 long delta = avg - wb->avg_write_bandwidth;
    1126         [ #  # ]:          0 :                 WARN_ON_ONCE(atomic_long_add_return(delta,
    1127                 :            :                                         &wb->bdi->tot_write_bandwidth) <= 0);
    1128                 :            :         }
    1129                 :          0 :         wb->write_bandwidth = bw;
    1130                 :          0 :         wb->avg_write_bandwidth = avg;
    1131                 :          0 : }
    1132                 :            : 
    1133                 :          0 : static void update_dirty_limit(struct dirty_throttle_control *dtc)
    1134                 :            : {
    1135                 :          0 :         struct wb_domain *dom = dtc_dom(dtc);
    1136                 :          0 :         unsigned long thresh = dtc->thresh;
    1137                 :          0 :         unsigned long limit = dom->dirty_limit;
    1138                 :            : 
    1139                 :            :         /*
    1140                 :            :          * Follow up in one step.
    1141                 :            :          */
    1142                 :          0 :         if (limit < thresh) {
    1143                 :          0 :                 limit = thresh;
    1144                 :          0 :                 goto update;
    1145                 :            :         }
    1146                 :            : 
    1147                 :            :         /*
    1148                 :            :          * Follow down slowly. Use the higher one as the target, because thresh
    1149                 :            :          * may drop below dirty. This is exactly the reason to introduce
    1150                 :            :          * dom->dirty_limit which is guaranteed to lie above the dirty pages.
    1151                 :            :          */
    1152                 :          0 :         thresh = max(thresh, dtc->dirty);
    1153         [ #  # ]:          0 :         if (limit > thresh) {
    1154                 :          0 :                 limit -= (limit - thresh) >> 5;
    1155                 :          0 :                 goto update;
    1156                 :            :         }
    1157                 :            :         return;
    1158                 :          0 : update:
    1159                 :          0 :         dom->dirty_limit = limit;
    1160                 :            : }
    1161                 :            : 
    1162                 :          0 : static void domain_update_bandwidth(struct dirty_throttle_control *dtc,
    1163                 :            :                                     unsigned long now)
    1164                 :            : {
    1165                 :          0 :         struct wb_domain *dom = dtc_dom(dtc);
    1166                 :            : 
    1167                 :            :         /*
    1168                 :            :          * check locklessly first to optimize away locking for the most time
    1169                 :            :          */
    1170         [ #  # ]:          0 :         if (time_before(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL))
    1171                 :            :                 return;
    1172                 :            : 
    1173                 :          0 :         spin_lock(&dom->lock);
    1174         [ #  # ]:          0 :         if (time_after_eq(now, dom->dirty_limit_tstamp + BANDWIDTH_INTERVAL)) {
    1175         [ #  # ]:          0 :                 update_dirty_limit(dtc);
    1176                 :          0 :                 dom->dirty_limit_tstamp = now;
    1177                 :            :         }
    1178                 :          0 :         spin_unlock(&dom->lock);
    1179                 :            : }
    1180                 :            : 
    1181                 :            : /*
    1182                 :            :  * Maintain wb->dirty_ratelimit, the base dirty throttle rate.
    1183                 :            :  *
    1184                 :            :  * Normal wb tasks will be curbed at or below it in long term.
    1185                 :            :  * Obviously it should be around (write_bw / N) when there are N dd tasks.
    1186                 :            :  */
    1187                 :          0 : static void wb_update_dirty_ratelimit(struct dirty_throttle_control *dtc,
    1188                 :            :                                       unsigned long dirtied,
    1189                 :            :                                       unsigned long elapsed)
    1190                 :            : {
    1191                 :          0 :         struct bdi_writeback *wb = dtc->wb;
    1192                 :          0 :         unsigned long dirty = dtc->dirty;
    1193                 :          0 :         unsigned long freerun = dirty_freerun_ceiling(dtc->thresh, dtc->bg_thresh);
    1194                 :          0 :         unsigned long limit = hard_dirty_limit(dtc_dom(dtc), dtc->thresh);
    1195                 :          0 :         unsigned long setpoint = (freerun + limit) / 2;
    1196                 :          0 :         unsigned long write_bw = wb->avg_write_bandwidth;
    1197                 :          0 :         unsigned long dirty_ratelimit = wb->dirty_ratelimit;
    1198                 :          0 :         unsigned long dirty_rate;
    1199                 :          0 :         unsigned long task_ratelimit;
    1200                 :          0 :         unsigned long balanced_dirty_ratelimit;
    1201                 :          0 :         unsigned long step;
    1202                 :          0 :         unsigned long x;
    1203                 :          0 :         unsigned long shift;
    1204                 :            : 
    1205                 :            :         /*
    1206                 :            :          * The dirty rate will match the writeout rate in long term, except
    1207                 :            :          * when dirty pages are truncated by userspace or re-dirtied by FS.
    1208                 :            :          */
    1209                 :          0 :         dirty_rate = (dirtied - wb->dirtied_stamp) * HZ / elapsed;
    1210                 :            : 
    1211                 :            :         /*
    1212                 :            :          * task_ratelimit reflects each dd's dirty rate for the past 200ms.
    1213                 :            :          */
    1214                 :          0 :         task_ratelimit = (u64)dirty_ratelimit *
    1215                 :          0 :                                         dtc->pos_ratio >> RATELIMIT_CALC_SHIFT;
    1216                 :          0 :         task_ratelimit++; /* it helps rampup dirty_ratelimit from tiny values */
    1217                 :            : 
    1218                 :            :         /*
    1219                 :            :          * A linear estimation of the "balanced" throttle rate. The theory is,
    1220                 :            :          * if there are N dd tasks, each throttled at task_ratelimit, the wb's
    1221                 :            :          * dirty_rate will be measured to be (N * task_ratelimit). So the below
    1222                 :            :          * formula will yield the balanced rate limit (write_bw / N).
    1223                 :            :          *
    1224                 :            :          * Note that the expanded form is not a pure rate feedback:
    1225                 :            :          *      rate_(i+1) = rate_(i) * (write_bw / dirty_rate)              (1)
    1226                 :            :          * but also takes pos_ratio into account:
    1227                 :            :          *      rate_(i+1) = rate_(i) * (write_bw / dirty_rate) * pos_ratio  (2)
    1228                 :            :          *
    1229                 :            :          * (1) is not realistic because pos_ratio also takes part in balancing
    1230                 :            :          * the dirty rate.  Consider the state
    1231                 :            :          *      pos_ratio = 0.5                                              (3)
    1232                 :            :          *      rate = 2 * (write_bw / N)                                    (4)
    1233                 :            :          * If (1) is used, it will stuck in that state! Because each dd will
    1234                 :            :          * be throttled at
    1235                 :            :          *      task_ratelimit = pos_ratio * rate = (write_bw / N)           (5)
    1236                 :            :          * yielding
    1237                 :            :          *      dirty_rate = N * task_ratelimit = write_bw                   (6)
    1238                 :            :          * put (6) into (1) we get
    1239                 :            :          *      rate_(i+1) = rate_(i)                                        (7)
    1240                 :            :          *
    1241                 :            :          * So we end up using (2) to always keep
    1242                 :            :          *      rate_(i+1) ~= (write_bw / N)                                 (8)
    1243                 :            :          * regardless of the value of pos_ratio. As long as (8) is satisfied,
    1244                 :            :          * pos_ratio is able to drive itself to 1.0, which is not only where
    1245                 :            :          * the dirty count meet the setpoint, but also where the slope of
    1246                 :            :          * pos_ratio is most flat and hence task_ratelimit is least fluctuated.
    1247                 :            :          */
    1248         [ #  # ]:          0 :         balanced_dirty_ratelimit = div_u64((u64)task_ratelimit * write_bw,
    1249                 :            :                                            dirty_rate | 1);
    1250                 :            :         /*
    1251                 :            :          * balanced_dirty_ratelimit ~= (write_bw / N) <= write_bw
    1252                 :            :          */
    1253         [ #  # ]:          0 :         if (unlikely(balanced_dirty_ratelimit > write_bw))
    1254                 :          0 :                 balanced_dirty_ratelimit = write_bw;
    1255                 :            : 
    1256                 :            :         /*
    1257                 :            :          * We could safely do this and return immediately:
    1258                 :            :          *
    1259                 :            :          *      wb->dirty_ratelimit = balanced_dirty_ratelimit;
    1260                 :            :          *
    1261                 :            :          * However to get a more stable dirty_ratelimit, the below elaborated
    1262                 :            :          * code makes use of task_ratelimit to filter out singular points and
    1263                 :            :          * limit the step size.
    1264                 :            :          *
    1265                 :            :          * The below code essentially only uses the relative value of
    1266                 :            :          *
    1267                 :            :          *      task_ratelimit - dirty_ratelimit
    1268                 :            :          *      = (pos_ratio - 1) * dirty_ratelimit
    1269                 :            :          *
    1270                 :            :          * which reflects the direction and size of dirty position error.
    1271                 :            :          */
    1272                 :            : 
    1273                 :            :         /*
    1274                 :            :          * dirty_ratelimit will follow balanced_dirty_ratelimit iff
    1275                 :            :          * task_ratelimit is on the same side of dirty_ratelimit, too.
    1276                 :            :          * For example, when
    1277                 :            :          * - dirty_ratelimit > balanced_dirty_ratelimit
    1278                 :            :          * - dirty_ratelimit > task_ratelimit (dirty pages are above setpoint)
    1279                 :            :          * lowering dirty_ratelimit will help meet both the position and rate
    1280                 :            :          * control targets. Otherwise, don't update dirty_ratelimit if it will
    1281                 :            :          * only help meet the rate target. After all, what the users ultimately
    1282                 :            :          * feel and care are stable dirty rate and small position error.
    1283                 :            :          *
    1284                 :            :          * |task_ratelimit - dirty_ratelimit| is used to limit the step size
    1285                 :            :          * and filter out the singular points of balanced_dirty_ratelimit. Which
    1286                 :            :          * keeps jumping around randomly and can even leap far away at times
    1287                 :            :          * due to the small 200ms estimation period of dirty_rate (we want to
    1288                 :            :          * keep that period small to reduce time lags).
    1289                 :            :          */
    1290                 :          0 :         step = 0;
    1291                 :            : 
    1292                 :            :         /*
    1293                 :            :          * For strictlimit case, calculations above were based on wb counters
    1294                 :            :          * and limits (starting from pos_ratio = wb_position_ratio() and up to
    1295                 :            :          * balanced_dirty_ratelimit = task_ratelimit * write_bw / dirty_rate).
    1296                 :            :          * Hence, to calculate "step" properly, we have to use wb_dirty as
    1297                 :            :          * "dirty" and wb_setpoint as "setpoint".
    1298                 :            :          *
    1299                 :            :          * We rampup dirty_ratelimit forcibly if wb_dirty is low because
    1300                 :            :          * it's possible that wb_thresh is close to zero due to inactivity
    1301                 :            :          * of backing device.
    1302                 :            :          */
    1303         [ #  # ]:          0 :         if (unlikely(wb->bdi->capabilities & BDI_CAP_STRICTLIMIT)) {
    1304                 :          0 :                 dirty = dtc->wb_dirty;
    1305         [ #  # ]:          0 :                 if (dtc->wb_dirty < 8)
    1306                 :          0 :                         setpoint = dtc->wb_dirty + 1;
    1307                 :            :                 else
    1308                 :          0 :                         setpoint = (dtc->wb_thresh + dtc->wb_bg_thresh) / 2;
    1309                 :            :         }
    1310                 :            : 
    1311         [ #  # ]:          0 :         if (dirty < setpoint) {
    1312                 :          0 :                 x = min3(wb->balanced_dirty_ratelimit,
    1313                 :            :                          balanced_dirty_ratelimit, task_ratelimit);
    1314         [ #  # ]:          0 :                 if (dirty_ratelimit < x)
    1315                 :          0 :                         step = x - dirty_ratelimit;
    1316                 :            :         } else {
    1317                 :          0 :                 x = max3(wb->balanced_dirty_ratelimit,
    1318                 :            :                          balanced_dirty_ratelimit, task_ratelimit);
    1319         [ #  # ]:          0 :                 if (dirty_ratelimit > x)
    1320                 :          0 :                         step = dirty_ratelimit - x;
    1321                 :            :         }
    1322                 :            : 
    1323                 :            :         /*
    1324                 :            :          * Don't pursue 100% rate matching. It's impossible since the balanced
    1325                 :            :          * rate itself is constantly fluctuating. So decrease the track speed
    1326                 :            :          * when it gets close to the target. Helps eliminate pointless tremors.
    1327                 :            :          */
    1328                 :          0 :         shift = dirty_ratelimit / (2 * step + 1);
    1329         [ #  # ]:          0 :         if (shift < BITS_PER_LONG)
    1330                 :          0 :                 step = DIV_ROUND_UP(step >> shift, 8);
    1331                 :            :         else
    1332                 :            :                 step = 0;
    1333                 :            : 
    1334         [ #  # ]:          0 :         if (dirty_ratelimit < balanced_dirty_ratelimit)
    1335                 :          0 :                 dirty_ratelimit += step;
    1336                 :            :         else
    1337                 :          0 :                 dirty_ratelimit -= step;
    1338                 :            : 
    1339                 :          0 :         wb->dirty_ratelimit = max(dirty_ratelimit, 1UL);
    1340                 :          0 :         wb->balanced_dirty_ratelimit = balanced_dirty_ratelimit;
    1341                 :            : 
    1342                 :          0 :         trace_bdi_dirty_ratelimit(wb, dirty_rate, task_ratelimit);
    1343                 :          0 : }
    1344                 :            : 
    1345                 :          0 : static void __wb_update_bandwidth(struct dirty_throttle_control *gdtc,
    1346                 :            :                                   struct dirty_throttle_control *mdtc,
    1347                 :            :                                   unsigned long start_time,
    1348                 :            :                                   bool update_ratelimit)
    1349                 :            : {
    1350                 :          0 :         struct bdi_writeback *wb = gdtc->wb;
    1351                 :          0 :         unsigned long now = jiffies;
    1352                 :          0 :         unsigned long elapsed = now - wb->bw_time_stamp;
    1353                 :          0 :         unsigned long dirtied;
    1354                 :          0 :         unsigned long written;
    1355                 :            : 
    1356                 :          0 :         lockdep_assert_held(&wb->list_lock);
    1357                 :            : 
    1358                 :            :         /*
    1359                 :            :          * rate-limit, only update once every 200ms.
    1360                 :            :          */
    1361         [ #  # ]:          0 :         if (elapsed < BANDWIDTH_INTERVAL)
    1362                 :            :                 return;
    1363                 :            : 
    1364         [ #  # ]:          0 :         dirtied = percpu_counter_read(&wb->stat[WB_DIRTIED]);
    1365                 :          0 :         written = percpu_counter_read(&wb->stat[WB_WRITTEN]);
    1366                 :            : 
    1367                 :            :         /*
    1368                 :            :          * Skip quiet periods when disk bandwidth is under-utilized.
    1369                 :            :          * (at least 1s idle time between two flusher runs)
    1370                 :            :          */
    1371   [ #  #  #  # ]:          0 :         if (elapsed > HZ && time_before(wb->bw_time_stamp, start_time))
    1372                 :          0 :                 goto snapshot;
    1373                 :            : 
    1374         [ #  # ]:          0 :         if (update_ratelimit) {
    1375                 :          0 :                 domain_update_bandwidth(gdtc, now);
    1376                 :          0 :                 wb_update_dirty_ratelimit(gdtc, dirtied, elapsed);
    1377                 :            : 
    1378                 :            :                 /*
    1379                 :            :                  * @mdtc is always NULL if !CGROUP_WRITEBACK but the
    1380                 :            :                  * compiler has no way to figure that out.  Help it.
    1381                 :            :                  */
    1382                 :          0 :                 if (IS_ENABLED(CONFIG_CGROUP_WRITEBACK) && mdtc) {
    1383                 :            :                         domain_update_bandwidth(mdtc, now);
    1384                 :            :                         wb_update_dirty_ratelimit(mdtc, dirtied, elapsed);
    1385                 :            :                 }
    1386                 :            :         }
    1387                 :          0 :         wb_update_write_bandwidth(wb, elapsed, written);
    1388                 :            : 
    1389                 :          0 : snapshot:
    1390                 :          0 :         wb->dirtied_stamp = dirtied;
    1391                 :          0 :         wb->written_stamp = written;
    1392                 :          0 :         wb->bw_time_stamp = now;
    1393                 :            : }
    1394                 :            : 
    1395                 :          0 : void wb_update_bandwidth(struct bdi_writeback *wb, unsigned long start_time)
    1396                 :            : {
    1397                 :          0 :         struct dirty_throttle_control gdtc = { GDTC_INIT(wb) };
    1398                 :            : 
    1399                 :          0 :         __wb_update_bandwidth(&gdtc, NULL, start_time, false);
    1400                 :          0 : }
    1401                 :            : 
    1402                 :            : /*
    1403                 :            :  * After a task dirtied this many pages, balance_dirty_pages_ratelimited()
    1404                 :            :  * will look to see if it needs to start dirty throttling.
    1405                 :            :  *
    1406                 :            :  * If dirty_poll_interval is too low, big NUMA machines will call the expensive
    1407                 :            :  * global_zone_page_state() too often. So scale it near-sqrt to the safety margin
    1408                 :            :  * (the number of pages we may dirty without exceeding the dirty limits).
    1409                 :            :  */
    1410                 :        231 : static unsigned long dirty_poll_interval(unsigned long dirty,
    1411                 :            :                                          unsigned long thresh)
    1412                 :            : {
    1413         [ +  - ]:        231 :         if (thresh > dirty)
    1414   [ -  +  -  -  :        231 :                 return 1UL << (ilog2(thresh - dirty) >> 1);
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
          -  -  -  -  -  
                   -  - ]
    1415                 :            : 
    1416                 :            :         return 1;
    1417                 :            : }
    1418                 :            : 
    1419                 :          0 : static unsigned long wb_max_pause(struct bdi_writeback *wb,
    1420                 :            :                                   unsigned long wb_dirty)
    1421                 :            : {
    1422                 :          0 :         unsigned long bw = wb->avg_write_bandwidth;
    1423                 :          0 :         unsigned long t;
    1424                 :            : 
    1425                 :            :         /*
    1426                 :            :          * Limit pause time for small memory systems. If sleeping for too long
    1427                 :            :          * time, a small pool of dirty/writeback pages may go empty and disk go
    1428                 :            :          * idle.
    1429                 :            :          *
    1430                 :            :          * 8 serves as the safety ratio.
    1431                 :            :          */
    1432                 :          0 :         t = wb_dirty / (1 + bw / roundup_pow_of_two(1 + HZ / 8));
    1433                 :          0 :         t++;
    1434                 :            : 
    1435                 :          0 :         return min_t(unsigned long, t, MAX_PAUSE);
    1436                 :            : }
    1437                 :            : 
    1438                 :            : static long wb_min_pause(struct bdi_writeback *wb,
    1439                 :            :                          long max_pause,
    1440                 :            :                          unsigned long task_ratelimit,
    1441                 :            :                          unsigned long dirty_ratelimit,
    1442                 :            :                          int *nr_dirtied_pause)
    1443                 :            : {
    1444                 :            :         long hi = ilog2(wb->avg_write_bandwidth);
    1445                 :            :         long lo = ilog2(wb->dirty_ratelimit);
    1446                 :            :         long t;         /* target pause */
    1447                 :            :         long pause;     /* estimated next pause */
    1448                 :            :         int pages;      /* target nr_dirtied_pause */
    1449                 :            : 
    1450                 :            :         /* target for 10ms pause on 1-dd case */
    1451                 :            :         t = max(1, HZ / 100);
    1452                 :            : 
    1453                 :            :         /*
    1454                 :            :          * Scale up pause time for concurrent dirtiers in order to reduce CPU
    1455                 :            :          * overheads.
    1456                 :            :          *
    1457                 :            :          * (N * 10ms) on 2^N concurrent tasks.
    1458                 :            :          */
    1459                 :            :         if (hi > lo)
    1460                 :            :                 t += (hi - lo) * (10 * HZ) / 1024;
    1461                 :            : 
    1462                 :            :         /*
    1463                 :            :          * This is a bit convoluted. We try to base the next nr_dirtied_pause
    1464                 :            :          * on the much more stable dirty_ratelimit. However the next pause time
    1465                 :            :          * will be computed based on task_ratelimit and the two rate limits may
    1466                 :            :          * depart considerably at some time. Especially if task_ratelimit goes
    1467                 :            :          * below dirty_ratelimit/2 and the target pause is max_pause, the next
    1468                 :            :          * pause time will be max_pause*2 _trimmed down_ to max_pause.  As a
    1469                 :            :          * result task_ratelimit won't be executed faithfully, which could
    1470                 :            :          * eventually bring down dirty_ratelimit.
    1471                 :            :          *
    1472                 :            :          * We apply two rules to fix it up:
    1473                 :            :          * 1) try to estimate the next pause time and if necessary, use a lower
    1474                 :            :          *    nr_dirtied_pause so as not to exceed max_pause. When this happens,
    1475                 :            :          *    nr_dirtied_pause will be "dancing" with task_ratelimit.
    1476                 :            :          * 2) limit the target pause time to max_pause/2, so that the normal
    1477                 :            :          *    small fluctuations of task_ratelimit won't trigger rule (1) and
    1478                 :            :          *    nr_dirtied_pause will remain as stable as dirty_ratelimit.
    1479                 :            :          */
    1480                 :            :         t = min(t, 1 + max_pause / 2);
    1481                 :            :         pages = dirty_ratelimit * t / roundup_pow_of_two(HZ);
    1482                 :            : 
    1483                 :            :         /*
    1484                 :            :          * Tiny nr_dirtied_pause is found to hurt I/O performance in the test
    1485                 :            :          * case fio-mmap-randwrite-64k, which does 16*{sync read, async write}.
    1486                 :            :          * When the 16 consecutive reads are often interrupted by some dirty
    1487                 :            :          * throttling pause during the async writes, cfq will go into idles
    1488                 :            :          * (deadline is fine). So push nr_dirtied_pause as high as possible
    1489                 :            :          * until reaches DIRTY_POLL_THRESH=32 pages.
    1490                 :            :          */
    1491                 :            :         if (pages < DIRTY_POLL_THRESH) {
    1492                 :            :                 t = max_pause;
    1493                 :            :                 pages = dirty_ratelimit * t / roundup_pow_of_two(HZ);
    1494                 :            :                 if (pages > DIRTY_POLL_THRESH) {
    1495                 :            :                         pages = DIRTY_POLL_THRESH;
    1496                 :            :                         t = HZ * DIRTY_POLL_THRESH / dirty_ratelimit;
    1497                 :            :                 }
    1498                 :            :         }
    1499                 :            : 
    1500                 :            :         pause = HZ * pages / (task_ratelimit + 1);
    1501                 :            :         if (pause > max_pause) {
    1502                 :            :                 t = max_pause;
    1503                 :            :                 pages = task_ratelimit * t / roundup_pow_of_two(HZ);
    1504                 :            :         }
    1505                 :            : 
    1506                 :            :         *nr_dirtied_pause = pages;
    1507                 :            :         /*
    1508                 :            :          * The minimal pause time will normally be half the target pause time.
    1509                 :            :          */
    1510                 :            :         return pages >= DIRTY_POLL_THRESH ? 1 + t / 2 : t;
    1511                 :            : }
    1512                 :            : 
    1513                 :          0 : static inline void wb_dirty_limits(struct dirty_throttle_control *dtc)
    1514                 :            : {
    1515                 :          0 :         struct bdi_writeback *wb = dtc->wb;
    1516                 :          0 :         unsigned long wb_reclaimable;
    1517                 :            : 
    1518                 :            :         /*
    1519                 :            :          * wb_thresh is not treated as some limiting factor as
    1520                 :            :          * dirty_thresh, due to reasons
    1521                 :            :          * - in JBOD setup, wb_thresh can fluctuate a lot
    1522                 :            :          * - in a system with HDD and USB key, the USB key may somehow
    1523                 :            :          *   go into state (wb_dirty >> wb_thresh) either because
    1524                 :            :          *   wb_dirty starts high, or because wb_thresh drops low.
    1525                 :            :          *   In this case we don't want to hard throttle the USB key
    1526                 :            :          *   dirtiers for 100 seconds until wb_dirty drops under
    1527                 :            :          *   wb_thresh. Instead the auxiliary wb control line in
    1528                 :            :          *   wb_position_ratio() will let the dirtier task progress
    1529                 :            :          *   at some rate <= (write_bw / 2) for bringing down wb_dirty.
    1530                 :            :          */
    1531                 :          0 :         dtc->wb_thresh = __wb_calc_thresh(dtc);
    1532                 :          0 :         dtc->wb_bg_thresh = dtc->thresh ?
    1533         [ #  # ]:          0 :                 div_u64((u64)dtc->wb_thresh * dtc->bg_thresh, dtc->thresh) : 0;
    1534                 :            : 
    1535                 :            :         /*
    1536                 :            :          * In order to avoid the stacked BDI deadlock we need
    1537                 :            :          * to ensure we accurately count the 'dirty' pages when
    1538                 :            :          * the threshold is low.
    1539                 :            :          *
    1540                 :            :          * Otherwise it would be possible to get thresh+n pages
    1541                 :            :          * reported dirty, even though there are thresh-m pages
    1542                 :            :          * actually dirty; with m+n sitting in the percpu
    1543                 :            :          * deltas.
    1544                 :            :          */
    1545         [ #  # ]:          0 :         if (dtc->wb_thresh < 2 * wb_stat_error()) {
    1546                 :          0 :                 wb_reclaimable = wb_stat_sum(wb, WB_RECLAIMABLE);
    1547                 :          0 :                 dtc->wb_dirty = wb_reclaimable + wb_stat_sum(wb, WB_WRITEBACK);
    1548                 :            :         } else {
    1549                 :          0 :                 wb_reclaimable = wb_stat(wb, WB_RECLAIMABLE);
    1550                 :          0 :                 dtc->wb_dirty = wb_reclaimable + wb_stat(wb, WB_WRITEBACK);
    1551                 :            :         }
    1552                 :          0 : }
    1553                 :            : 
    1554                 :            : /*
    1555                 :            :  * balance_dirty_pages() must be called by processes which are generating dirty
    1556                 :            :  * data.  It looks at the number of dirty pages in the machine and will force
    1557                 :            :  * the caller to wait once crossing the (background_thresh + dirty_thresh) / 2.
    1558                 :            :  * If we're over `background_thresh' then the writeback threads are woken to
    1559                 :            :  * perform some writeout.
    1560                 :            :  */
    1561                 :        231 : static void balance_dirty_pages(struct bdi_writeback *wb,
    1562                 :            :                                 unsigned long pages_dirtied)
    1563                 :            : {
    1564                 :        231 :         struct dirty_throttle_control gdtc_stor = { GDTC_INIT(wb) };
    1565                 :        231 :         struct dirty_throttle_control mdtc_stor = { MDTC_INIT(wb, &gdtc_stor) };
    1566                 :        231 :         struct dirty_throttle_control * const gdtc = &gdtc_stor;
    1567                 :        231 :         struct dirty_throttle_control * const mdtc = mdtc_valid(&mdtc_stor) ?
    1568                 :            :                                                      &mdtc_stor : NULL;
    1569                 :        231 :         struct dirty_throttle_control *sdtc;
    1570                 :        231 :         unsigned long nr_reclaimable;   /* = file_dirty + unstable_nfs */
    1571                 :        231 :         long period;
    1572                 :        231 :         long pause;
    1573                 :        231 :         long max_pause;
    1574                 :        231 :         long min_pause;
    1575                 :        231 :         int nr_dirtied_pause;
    1576                 :        231 :         bool dirty_exceeded = false;
    1577                 :        231 :         unsigned long task_ratelimit;
    1578                 :        231 :         unsigned long dirty_ratelimit;
    1579                 :        231 :         struct backing_dev_info *bdi = wb->bdi;
    1580                 :        231 :         bool strictlimit = bdi->capabilities & BDI_CAP_STRICTLIMIT;
    1581                 :        231 :         unsigned long start_time = jiffies;
    1582                 :            : 
    1583                 :        231 :         for (;;) {
    1584                 :        231 :                 unsigned long now = jiffies;
    1585                 :        231 :                 unsigned long dirty, thresh, bg_thresh;
    1586                 :        231 :                 unsigned long m_dirty = 0;      /* stop bogus uninit warnings */
    1587                 :        231 :                 unsigned long m_thresh = 0;
    1588                 :        231 :                 unsigned long m_bg_thresh = 0;
    1589                 :            : 
    1590                 :            :                 /*
    1591                 :            :                  * Unstable writes are a feature of certain networked
    1592                 :            :                  * filesystems (i.e. NFS) in which data may have been
    1593                 :            :                  * written to the server's write cache, but has not yet
    1594                 :            :                  * been flushed to permanent storage.
    1595                 :            :                  */
    1596                 :        231 :                 nr_reclaimable = global_node_page_state(NR_FILE_DIRTY) +
    1597                 :            :                                         global_node_page_state(NR_UNSTABLE_NFS);
    1598                 :        231 :                 gdtc->avail = global_dirtyable_memory();
    1599                 :        231 :                 gdtc->dirty = nr_reclaimable + global_node_page_state(NR_WRITEBACK);
    1600                 :            : 
    1601                 :        231 :                 domain_dirty_limits(gdtc);
    1602                 :            : 
    1603         [ -  + ]:        231 :                 if (unlikely(strictlimit)) {
    1604                 :          0 :                         wb_dirty_limits(gdtc);
    1605                 :            : 
    1606                 :          0 :                         dirty = gdtc->wb_dirty;
    1607                 :          0 :                         thresh = gdtc->wb_thresh;
    1608                 :          0 :                         bg_thresh = gdtc->wb_bg_thresh;
    1609                 :            :                 } else {
    1610                 :        231 :                         dirty = gdtc->dirty;
    1611                 :        231 :                         thresh = gdtc->thresh;
    1612                 :        231 :                         bg_thresh = gdtc->bg_thresh;
    1613                 :            :                 }
    1614                 :            : 
    1615                 :        231 :                 if (mdtc) {
    1616                 :            :                         unsigned long filepages, headroom, writeback;
    1617                 :            : 
    1618                 :            :                         /*
    1619                 :            :                          * If @wb belongs to !root memcg, repeat the same
    1620                 :            :                          * basic calculations for the memcg domain.
    1621                 :            :                          */
    1622                 :            :                         mem_cgroup_wb_stats(wb, &filepages, &headroom,
    1623                 :            :                                             &mdtc->dirty, &writeback);
    1624                 :            :                         mdtc->dirty += writeback;
    1625                 :            :                         mdtc_calc_avail(mdtc, filepages, headroom);
    1626                 :            : 
    1627                 :            :                         domain_dirty_limits(mdtc);
    1628                 :            : 
    1629                 :            :                         if (unlikely(strictlimit)) {
    1630                 :            :                                 wb_dirty_limits(mdtc);
    1631                 :            :                                 m_dirty = mdtc->wb_dirty;
    1632                 :            :                                 m_thresh = mdtc->wb_thresh;
    1633                 :            :                                 m_bg_thresh = mdtc->wb_bg_thresh;
    1634                 :            :                         } else {
    1635                 :            :                                 m_dirty = mdtc->dirty;
    1636                 :            :                                 m_thresh = mdtc->thresh;
    1637                 :            :                                 m_bg_thresh = mdtc->bg_thresh;
    1638                 :            :                         }
    1639                 :            :                 }
    1640                 :            : 
    1641                 :            :                 /*
    1642                 :            :                  * Throttle it only when the background writeback cannot
    1643                 :            :                  * catch-up. This avoids (excessively) small writeouts
    1644                 :            :                  * when the wb limits are ramping up in case of !strictlimit.
    1645                 :            :                  *
    1646                 :            :                  * In strictlimit case make decision based on the wb counters
    1647                 :            :                  * and limits. Small writeouts when the wb limits are ramping
    1648                 :            :                  * up are the price we consciously pay for strictlimit-ing.
    1649                 :            :                  *
    1650                 :            :                  * If memcg domain is in effect, @dirty should be under
    1651                 :            :                  * both global and memcg freerun ceilings.
    1652                 :            :                  */
    1653         [ +  - ]:        231 :                 if (dirty <= dirty_freerun_ceiling(thresh, bg_thresh) &&
    1654                 :            :                     (!mdtc ||
    1655                 :            :                      m_dirty <= dirty_freerun_ceiling(m_thresh, m_bg_thresh))) {
    1656                 :        231 :                         unsigned long intv = dirty_poll_interval(dirty, thresh);
    1657                 :        231 :                         unsigned long m_intv = ULONG_MAX;
    1658                 :            : 
    1659                 :        231 :                         current->dirty_paused_when = now;
    1660                 :        231 :                         current->nr_dirtied = 0;
    1661                 :        231 :                         if (mdtc)
    1662                 :            :                                 m_intv = dirty_poll_interval(m_dirty, m_thresh);
    1663                 :        231 :                         current->nr_dirtied_pause = min(intv, m_intv);
    1664                 :        231 :                         break;
    1665                 :            :                 }
    1666                 :            : 
    1667         [ #  # ]:          0 :                 if (unlikely(!writeback_in_progress(wb)))
    1668                 :          0 :                         wb_start_background_writeback(wb);
    1669                 :            : 
    1670         [ #  # ]:          0 :                 mem_cgroup_flush_foreign(wb);
    1671                 :            : 
    1672                 :            :                 /*
    1673                 :            :                  * Calculate global domain's pos_ratio and select the
    1674                 :            :                  * global dtc by default.
    1675                 :            :                  */
    1676         [ #  # ]:          0 :                 if (!strictlimit)
    1677                 :          0 :                         wb_dirty_limits(gdtc);
    1678                 :            : 
    1679         [ #  # ]:          0 :                 dirty_exceeded = (gdtc->wb_dirty > gdtc->wb_thresh) &&
    1680   [ #  #  #  # ]:          0 :                         ((gdtc->dirty > gdtc->thresh) || strictlimit);
    1681                 :            : 
    1682                 :          0 :                 wb_position_ratio(gdtc);
    1683                 :          0 :                 sdtc = gdtc;
    1684                 :            : 
    1685                 :          0 :                 if (mdtc) {
    1686                 :            :                         /*
    1687                 :            :                          * If memcg domain is in effect, calculate its
    1688                 :            :                          * pos_ratio.  @wb should satisfy constraints from
    1689                 :            :                          * both global and memcg domains.  Choose the one
    1690                 :            :                          * w/ lower pos_ratio.
    1691                 :            :                          */
    1692                 :            :                         if (!strictlimit)
    1693                 :            :                                 wb_dirty_limits(mdtc);
    1694                 :            : 
    1695                 :            :                         dirty_exceeded |= (mdtc->wb_dirty > mdtc->wb_thresh) &&
    1696                 :            :                                 ((mdtc->dirty > mdtc->thresh) || strictlimit);
    1697                 :            : 
    1698                 :            :                         wb_position_ratio(mdtc);
    1699                 :            :                         if (mdtc->pos_ratio < gdtc->pos_ratio)
    1700                 :            :                                 sdtc = mdtc;
    1701                 :            :                 }
    1702                 :            : 
    1703   [ #  #  #  # ]:          0 :                 if (dirty_exceeded && !wb->dirty_exceeded)
    1704                 :          0 :                         wb->dirty_exceeded = 1;
    1705                 :            : 
    1706         [ #  # ]:          0 :                 if (time_is_before_jiffies(wb->bw_time_stamp +
    1707                 :            :                                            BANDWIDTH_INTERVAL)) {
    1708                 :          0 :                         spin_lock(&wb->list_lock);
    1709                 :          0 :                         __wb_update_bandwidth(gdtc, mdtc, start_time, true);
    1710                 :          0 :                         spin_unlock(&wb->list_lock);
    1711                 :            :                 }
    1712                 :            : 
    1713                 :            :                 /* throttle according to the chosen dtc */
    1714                 :          0 :                 dirty_ratelimit = wb->dirty_ratelimit;
    1715                 :          0 :                 task_ratelimit = ((u64)dirty_ratelimit * sdtc->pos_ratio) >>
    1716                 :            :                                                         RATELIMIT_CALC_SHIFT;
    1717                 :          0 :                 max_pause = wb_max_pause(wb, sdtc->wb_dirty);
    1718                 :          0 :                 min_pause = wb_min_pause(wb, max_pause,
    1719                 :            :                                          task_ratelimit, dirty_ratelimit,
    1720                 :            :                                          &nr_dirtied_pause);
    1721                 :            : 
    1722         [ #  # ]:          0 :                 if (unlikely(task_ratelimit == 0)) {
    1723                 :          0 :                         period = max_pause;
    1724                 :          0 :                         pause = max_pause;
    1725                 :          0 :                         goto pause;
    1726                 :            :                 }
    1727                 :          0 :                 period = HZ * pages_dirtied / task_ratelimit;
    1728                 :          0 :                 pause = period;
    1729         [ #  # ]:          0 :                 if (current->dirty_paused_when)
    1730                 :          0 :                         pause -= now - current->dirty_paused_when;
    1731                 :            :                 /*
    1732                 :            :                  * For less than 1s think time (ext3/4 may block the dirtier
    1733                 :            :                  * for up to 800ms from time to time on 1-HDD; so does xfs,
    1734                 :            :                  * however at much less frequency), try to compensate it in
    1735                 :            :                  * future periods by updating the virtual time; otherwise just
    1736                 :            :                  * do a reset, as it may be a light dirtier.
    1737                 :            :                  */
    1738         [ #  # ]:          0 :                 if (pause < min_pause) {
    1739                 :          0 :                         trace_balance_dirty_pages(wb,
    1740                 :            :                                                   sdtc->thresh,
    1741                 :            :                                                   sdtc->bg_thresh,
    1742                 :            :                                                   sdtc->dirty,
    1743                 :            :                                                   sdtc->wb_thresh,
    1744                 :            :                                                   sdtc->wb_dirty,
    1745                 :            :                                                   dirty_ratelimit,
    1746                 :            :                                                   task_ratelimit,
    1747                 :            :                                                   pages_dirtied,
    1748                 :            :                                                   period,
    1749                 :          0 :                                                   min(pause, 0L),
    1750                 :            :                                                   start_time);
    1751         [ #  # ]:          0 :                         if (pause < -HZ) {
    1752                 :          0 :                                 current->dirty_paused_when = now;
    1753                 :          0 :                                 current->nr_dirtied = 0;
    1754         [ #  # ]:          0 :                         } else if (period) {
    1755                 :          0 :                                 current->dirty_paused_when += period;
    1756                 :          0 :                                 current->nr_dirtied = 0;
    1757         [ #  # ]:          0 :                         } else if (current->nr_dirtied_pause <= pages_dirtied)
    1758                 :          0 :                                 current->nr_dirtied_pause += pages_dirtied;
    1759                 :            :                         break;
    1760                 :            :                 }
    1761         [ #  # ]:          0 :                 if (unlikely(pause > max_pause)) {
    1762                 :            :                         /* for occasional dropped task_ratelimit */
    1763                 :          0 :                         now += min(pause - max_pause, max_pause);
    1764                 :          0 :                         pause = max_pause;
    1765                 :            :                 }
    1766                 :            : 
    1767                 :          0 : pause:
    1768                 :          0 :                 trace_balance_dirty_pages(wb,
    1769                 :            :                                           sdtc->thresh,
    1770                 :            :                                           sdtc->bg_thresh,
    1771                 :            :                                           sdtc->dirty,
    1772                 :            :                                           sdtc->wb_thresh,
    1773                 :            :                                           sdtc->wb_dirty,
    1774                 :            :                                           dirty_ratelimit,
    1775                 :            :                                           task_ratelimit,
    1776                 :            :                                           pages_dirtied,
    1777                 :            :                                           period,
    1778                 :            :                                           pause,
    1779                 :            :                                           start_time);
    1780                 :          0 :                 __set_current_state(TASK_KILLABLE);
    1781                 :          0 :                 wb->dirty_sleep = now;
    1782                 :          0 :                 io_schedule_timeout(pause);
    1783                 :            : 
    1784         [ #  # ]:          0 :                 current->dirty_paused_when = now + pause;
    1785                 :          0 :                 current->nr_dirtied = 0;
    1786                 :          0 :                 current->nr_dirtied_pause = nr_dirtied_pause;
    1787                 :            : 
    1788                 :            :                 /*
    1789                 :            :                  * This is typically equal to (dirty < thresh) and can also
    1790                 :            :                  * keep "1000+ dd on a slow USB stick" under control.
    1791                 :            :                  */
    1792         [ #  # ]:          0 :                 if (task_ratelimit)
    1793                 :            :                         break;
    1794                 :            : 
    1795                 :            :                 /*
    1796                 :            :                  * In the case of an unresponding NFS server and the NFS dirty
    1797                 :            :                  * pages exceeds dirty_thresh, give the other good wb's a pipe
    1798                 :            :                  * to go through, so that tasks on them still remain responsive.
    1799                 :            :                  *
    1800                 :            :                  * In theory 1 page is enough to keep the consumer-producer
    1801                 :            :                  * pipe going: the flusher cleans 1 page => the task dirties 1
    1802                 :            :                  * more page. However wb_dirty has accounting errors.  So use
    1803                 :            :                  * the larger and more IO friendly wb_stat_error.
    1804                 :            :                  */
    1805         [ #  # ]:          0 :                 if (sdtc->wb_dirty <= wb_stat_error())
    1806                 :            :                         break;
    1807                 :            : 
    1808         [ #  # ]:          0 :                 if (fatal_signal_pending(current))
    1809                 :            :                         break;
    1810                 :            :         }
    1811                 :            : 
    1812   [ +  -  -  + ]:        231 :         if (!dirty_exceeded && wb->dirty_exceeded)
    1813                 :          0 :                 wb->dirty_exceeded = 0;
    1814                 :            : 
    1815         [ +  - ]:        231 :         if (writeback_in_progress(wb))
    1816                 :          0 :                 return;
    1817                 :            : 
    1818                 :            :         /*
    1819                 :            :          * In laptop mode, we wait until hitting the higher threshold before
    1820                 :            :          * starting background writeout, and then write out all the way down
    1821                 :            :          * to the lower threshold.  So slow writers cause minimal disk activity.
    1822                 :            :          *
    1823                 :            :          * In normal mode, we start background writeout at the lower
    1824                 :            :          * background_thresh, to keep the amount of dirty memory low.
    1825                 :            :          */
    1826         [ +  - ]:        231 :         if (laptop_mode)
    1827                 :            :                 return;
    1828                 :            : 
    1829         [ -  + ]:        231 :         if (nr_reclaimable > gdtc->bg_thresh)
    1830                 :          0 :                 wb_start_background_writeback(wb);
    1831                 :            : }
    1832                 :            : 
    1833                 :            : static DEFINE_PER_CPU(int, bdp_ratelimits);
    1834                 :            : 
    1835                 :            : /*
    1836                 :            :  * Normal tasks are throttled by
    1837                 :            :  *      loop {
    1838                 :            :  *              dirty tsk->nr_dirtied_pause pages;
    1839                 :            :  *              take a snap in balance_dirty_pages();
    1840                 :            :  *      }
    1841                 :            :  * However there is a worst case. If every task exit immediately when dirtied
    1842                 :            :  * (tsk->nr_dirtied_pause - 1) pages, balance_dirty_pages() will never be
    1843                 :            :  * called to throttle the page dirties. The solution is to save the not yet
    1844                 :            :  * throttled page dirties in dirty_throttle_leaks on task exit and charge them
    1845                 :            :  * randomly into the running tasks. This works well for the above worst case,
    1846                 :            :  * as the new task will pick up and accumulate the old task's leaked dirty
    1847                 :            :  * count and eventually get throttled.
    1848                 :            :  */
    1849                 :            : DEFINE_PER_CPU(int, dirty_throttle_leaks) = 0;
    1850                 :            : 
    1851                 :            : /**
    1852                 :            :  * balance_dirty_pages_ratelimited - balance dirty memory state
    1853                 :            :  * @mapping: address_space which was dirtied
    1854                 :            :  *
    1855                 :            :  * Processes which are dirtying memory should call in here once for each page
    1856                 :            :  * which was newly dirtied.  The function will periodically check the system's
    1857                 :            :  * dirty state and will initiate writeback if needed.
    1858                 :            :  *
    1859                 :            :  * On really big machines, get_writeback_state is expensive, so try to avoid
    1860                 :            :  * calling it too often (ratelimiting).  But once we're over the dirty memory
    1861                 :            :  * limit we decrease the ratelimiting by a lot, to prevent individual processes
    1862                 :            :  * from overshooting the limit by (ratelimit_pages) each.
    1863                 :            :  */
    1864                 :      13710 : void balance_dirty_pages_ratelimited(struct address_space *mapping)
    1865                 :            : {
    1866                 :      13710 :         struct inode *inode = mapping->host;
    1867                 :      13710 :         struct backing_dev_info *bdi = inode_to_bdi(inode);
    1868                 :      13710 :         struct bdi_writeback *wb = NULL;
    1869                 :      13710 :         int ratelimit;
    1870                 :      13710 :         int *p;
    1871                 :            : 
    1872         [ +  + ]:      13710 :         if (!bdi_cap_account_dirty(bdi))
    1873                 :            :                 return;
    1874                 :            : 
    1875         [ -  + ]:      11337 :         if (inode_cgwb_enabled(inode))
    1876                 :            :                 wb = wb_get_create_current(bdi, GFP_KERNEL);
    1877                 :      11337 :         if (!wb)
    1878                 :      11337 :                 wb = &bdi->wb;
    1879                 :            : 
    1880         [ -  + ]:      11337 :         ratelimit = current->nr_dirtied_pause;
    1881         [ -  + ]:      11337 :         if (wb->dirty_exceeded)
    1882                 :          0 :                 ratelimit = min(ratelimit, 32 >> (PAGE_SHIFT - 10));
    1883                 :            : 
    1884                 :      11337 :         preempt_disable();
    1885                 :            :         /*
    1886                 :            :          * This prevents one CPU to accumulate too many dirtied pages without
    1887                 :            :          * calling into balance_dirty_pages(), which can happen when there are
    1888                 :            :          * 1000+ tasks, all of them start dirtying pages at exactly the same
    1889                 :            :          * time, hence all honoured too large initial task->nr_dirtied_pause.
    1890                 :            :          */
    1891                 :      11337 :         p =  this_cpu_ptr(&bdp_ratelimits);
    1892         [ +  + ]:      11337 :         if (unlikely(current->nr_dirtied >= ratelimit))
    1893                 :        126 :                 *p = 0;
    1894         [ -  + ]:      11211 :         else if (unlikely(*p >= ratelimit_pages)) {
    1895                 :          0 :                 *p = 0;
    1896                 :          0 :                 ratelimit = 0;
    1897                 :            :         }
    1898                 :            :         /*
    1899                 :            :          * Pick up the dirtied pages by the exited tasks. This avoids lots of
    1900                 :            :          * short-lived tasks (eg. gcc invocations in a kernel build) escaping
    1901                 :            :          * the dirty throttling and livelock other long-run dirtiers.
    1902                 :            :          */
    1903                 :      11337 :         p = this_cpu_ptr(&dirty_throttle_leaks);
    1904   [ +  +  +  - ]:      11337 :         if (*p > 0 && current->nr_dirtied < ratelimit) {
    1905                 :       4515 :                 unsigned long nr_pages_dirtied;
    1906                 :       4515 :                 nr_pages_dirtied = min(*p, ratelimit - current->nr_dirtied);
    1907                 :       4515 :                 *p -= nr_pages_dirtied;
    1908                 :       4515 :                 current->nr_dirtied += nr_pages_dirtied;
    1909                 :            :         }
    1910                 :      11337 :         preempt_enable();
    1911                 :            : 
    1912         [ +  + ]:      11337 :         if (unlikely(current->nr_dirtied >= ratelimit))
    1913                 :        231 :                 balance_dirty_pages(wb, current->nr_dirtied);
    1914                 :            : 
    1915                 :            :         wb_put(wb);
    1916                 :            : }
    1917                 :            : EXPORT_SYMBOL(balance_dirty_pages_ratelimited);
    1918                 :            : 
    1919                 :            : /**
    1920                 :            :  * wb_over_bg_thresh - does @wb need to be written back?
    1921                 :            :  * @wb: bdi_writeback of interest
    1922                 :            :  *
    1923                 :            :  * Determines whether background writeback should keep writing @wb or it's
    1924                 :            :  * clean enough.
    1925                 :            :  *
    1926                 :            :  * Return: %true if writeback should continue.
    1927                 :            :  */
    1928                 :          0 : bool wb_over_bg_thresh(struct bdi_writeback *wb)
    1929                 :            : {
    1930                 :          0 :         struct dirty_throttle_control gdtc_stor = { GDTC_INIT(wb) };
    1931                 :          0 :         struct dirty_throttle_control mdtc_stor = { MDTC_INIT(wb, &gdtc_stor) };
    1932                 :          0 :         struct dirty_throttle_control * const gdtc = &gdtc_stor;
    1933                 :          0 :         struct dirty_throttle_control * const mdtc = mdtc_valid(&mdtc_stor) ?
    1934                 :            :                                                      &mdtc_stor : NULL;
    1935                 :            : 
    1936                 :            :         /*
    1937                 :            :          * Similar to balance_dirty_pages() but ignores pages being written
    1938                 :            :          * as we're trying to decide whether to put more under writeback.
    1939                 :            :          */
    1940                 :          0 :         gdtc->avail = global_dirtyable_memory();
    1941                 :          0 :         gdtc->dirty = global_node_page_state(NR_FILE_DIRTY) +
    1942                 :            :                       global_node_page_state(NR_UNSTABLE_NFS);
    1943                 :          0 :         domain_dirty_limits(gdtc);
    1944                 :            : 
    1945         [ #  # ]:          0 :         if (gdtc->dirty > gdtc->bg_thresh)
    1946                 :            :                 return true;
    1947                 :            : 
    1948         [ #  # ]:          0 :         if (wb_stat(wb, WB_RECLAIMABLE) >
    1949                 :          0 :             wb_calc_thresh(gdtc->wb, gdtc->bg_thresh))
    1950                 :          0 :                 return true;
    1951                 :            : 
    1952                 :            :         if (mdtc) {
    1953                 :            :                 unsigned long filepages, headroom, writeback;
    1954                 :            : 
    1955                 :            :                 mem_cgroup_wb_stats(wb, &filepages, &headroom, &mdtc->dirty,
    1956                 :            :                                     &writeback);
    1957                 :            :                 mdtc_calc_avail(mdtc, filepages, headroom);
    1958                 :            :                 domain_dirty_limits(mdtc);      /* ditto, ignore writeback */
    1959                 :            : 
    1960                 :            :                 if (mdtc->dirty > mdtc->bg_thresh)
    1961                 :            :                         return true;
    1962                 :            : 
    1963                 :            :                 if (wb_stat(wb, WB_RECLAIMABLE) >
    1964                 :            :                     wb_calc_thresh(mdtc->wb, mdtc->bg_thresh))
    1965                 :            :                         return true;
    1966                 :            :         }
    1967                 :            : 
    1968                 :            :         return false;
    1969                 :            : }
    1970                 :            : 
    1971                 :            : /*
    1972                 :            :  * sysctl handler for /proc/sys/vm/dirty_writeback_centisecs
    1973                 :            :  */
    1974                 :          0 : int dirty_writeback_centisecs_handler(struct ctl_table *table, int write,
    1975                 :            :         void __user *buffer, size_t *length, loff_t *ppos)
    1976                 :            : {
    1977                 :          0 :         unsigned int old_interval = dirty_writeback_interval;
    1978                 :          0 :         int ret;
    1979                 :            : 
    1980                 :          0 :         ret = proc_dointvec(table, write, buffer, length, ppos);
    1981                 :            : 
    1982                 :            :         /*
    1983                 :            :          * Writing 0 to dirty_writeback_interval will disable periodic writeback
    1984                 :            :          * and a different non-zero value will wakeup the writeback threads.
    1985                 :            :          * wb_wakeup_delayed() would be more appropriate, but it's a pain to
    1986                 :            :          * iterate over all bdis and wbs.
    1987                 :            :          * The reason we do this is to make the change take effect immediately.
    1988                 :            :          */
    1989   [ #  #  #  #  :          0 :         if (!ret && write && dirty_writeback_interval &&
                   #  # ]
    1990                 :            :                 dirty_writeback_interval != old_interval)
    1991                 :          0 :                 wakeup_flusher_threads(WB_REASON_PERIODIC);
    1992                 :            : 
    1993                 :          0 :         return ret;
    1994                 :            : }
    1995                 :            : 
    1996                 :            : #ifdef CONFIG_BLOCK
    1997                 :          0 : void laptop_mode_timer_fn(struct timer_list *t)
    1998                 :            : {
    1999                 :          0 :         struct backing_dev_info *backing_dev_info =
    2000                 :          0 :                 from_timer(backing_dev_info, t, laptop_mode_wb_timer);
    2001                 :            : 
    2002                 :          0 :         wakeup_flusher_threads_bdi(backing_dev_info, WB_REASON_LAPTOP_TIMER);
    2003                 :          0 : }
    2004                 :            : 
    2005                 :            : /*
    2006                 :            :  * We've spun up the disk and we're in laptop mode: schedule writeback
    2007                 :            :  * of all dirty data a few seconds from now.  If the flush is already scheduled
    2008                 :            :  * then push it back - the user is still using the disk.
    2009                 :            :  */
    2010                 :          0 : void laptop_io_completion(struct backing_dev_info *info)
    2011                 :            : {
    2012                 :          0 :         mod_timer(&info->laptop_mode_wb_timer, jiffies + laptop_mode);
    2013                 :          0 : }
    2014                 :            : 
    2015                 :            : /*
    2016                 :            :  * We're in laptop mode and we've just synced. The sync's writes will have
    2017                 :            :  * caused another writeback to be scheduled by laptop_io_completion.
    2018                 :            :  * Nothing needs to be written back anymore, so we unschedule the writeback.
    2019                 :            :  */
    2020                 :          0 : void laptop_sync_completion(void)
    2021                 :            : {
    2022                 :          0 :         struct backing_dev_info *bdi;
    2023                 :            : 
    2024                 :          0 :         rcu_read_lock();
    2025                 :            : 
    2026         [ #  # ]:          0 :         list_for_each_entry_rcu(bdi, &bdi_list, bdi_list)
    2027                 :          0 :                 del_timer(&bdi->laptop_mode_wb_timer);
    2028                 :            : 
    2029                 :          0 :         rcu_read_unlock();
    2030                 :          0 : }
    2031                 :            : #endif
    2032                 :            : 
    2033                 :            : /*
    2034                 :            :  * If ratelimit_pages is too high then we can get into dirty-data overload
    2035                 :            :  * if a large number of processes all perform writes at the same time.
    2036                 :            :  * If it is too low then SMP machines will call the (expensive)
    2037                 :            :  * get_writeback_state too often.
    2038                 :            :  *
    2039                 :            :  * Here we set ratelimit_pages to a level which ensures that when all CPUs are
    2040                 :            :  * dirtying in parallel, we cannot go more than 3% (1/32) over the dirty memory
    2041                 :            :  * thresholds.
    2042                 :            :  */
    2043                 :            : 
    2044                 :         21 : void writeback_set_ratelimit(void)
    2045                 :            : {
    2046                 :         21 :         struct wb_domain *dom = &global_wb_domain;
    2047                 :         21 :         unsigned long background_thresh;
    2048                 :         21 :         unsigned long dirty_thresh;
    2049                 :            : 
    2050                 :         21 :         global_dirty_limits(&background_thresh, &dirty_thresh);
    2051                 :         21 :         dom->dirty_limit = dirty_thresh;
    2052                 :         21 :         ratelimit_pages = dirty_thresh / (num_online_cpus() * 32);
    2053         [ -  + ]:         21 :         if (ratelimit_pages < 16)
    2054                 :          0 :                 ratelimit_pages = 16;
    2055                 :         21 : }
    2056                 :            : 
    2057                 :         21 : static int page_writeback_cpu_online(unsigned int cpu)
    2058                 :            : {
    2059                 :         21 :         writeback_set_ratelimit();
    2060                 :         21 :         return 0;
    2061                 :            : }
    2062                 :            : 
    2063                 :            : /*
    2064                 :            :  * Called early on to tune the page writeback dirty limits.
    2065                 :            :  *
    2066                 :            :  * We used to scale dirty pages according to how total memory
    2067                 :            :  * related to pages that could be allocated for buffers (by
    2068                 :            :  * comparing nr_free_buffer_pages() to vm_total_pages.
    2069                 :            :  *
    2070                 :            :  * However, that was when we used "dirty_ratio" to scale with
    2071                 :            :  * all memory, and we don't do that any more. "dirty_ratio"
    2072                 :            :  * is now applied to total non-HIGHPAGE memory (by subtracting
    2073                 :            :  * totalhigh_pages from vm_total_pages), and as such we can't
    2074                 :            :  * get into the old insane situation any more where we had
    2075                 :            :  * large amounts of dirty pages compared to a small amount of
    2076                 :            :  * non-HIGHMEM memory.
    2077                 :            :  *
    2078                 :            :  * But we might still want to scale the dirty_ratio by how
    2079                 :            :  * much memory the box has..
    2080                 :            :  */
    2081                 :         21 : void __init page_writeback_init(void)
    2082                 :            : {
    2083         [ -  + ]:         21 :         BUG_ON(wb_domain_init(&global_wb_domain, GFP_KERNEL));
    2084                 :            : 
    2085                 :         21 :         cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mm/writeback:online",
    2086                 :            :                           page_writeback_cpu_online, NULL);
    2087                 :         21 :         cpuhp_setup_state(CPUHP_MM_WRITEBACK_DEAD, "mm/writeback:dead", NULL,
    2088                 :            :                           page_writeback_cpu_online);
    2089                 :         21 : }
    2090                 :            : 
    2091                 :            : /**
    2092                 :            :  * tag_pages_for_writeback - tag pages to be written by write_cache_pages
    2093                 :            :  * @mapping: address space structure to write
    2094                 :            :  * @start: starting page index
    2095                 :            :  * @end: ending page index (inclusive)
    2096                 :            :  *
    2097                 :            :  * This function scans the page range from @start to @end (inclusive) and tags
    2098                 :            :  * all pages that have DIRTY tag set with a special TOWRITE tag. The idea is
    2099                 :            :  * that write_cache_pages (or whoever calls this function) will then use
    2100                 :            :  * TOWRITE tag to identify pages eligible for writeback.  This mechanism is
    2101                 :            :  * used to avoid livelocking of writeback by a process steadily creating new
    2102                 :            :  * dirty pages in the file (thus it is important for this function to be quick
    2103                 :            :  * so that it can tag pages faster than a dirtying process can create them).
    2104                 :            :  */
    2105                 :        147 : void tag_pages_for_writeback(struct address_space *mapping,
    2106                 :            :                              pgoff_t start, pgoff_t end)
    2107                 :            : {
    2108                 :        147 :         XA_STATE(xas, &mapping->i_pages, start);
    2109                 :        147 :         unsigned int tagged = 0;
    2110                 :        147 :         void *page;
    2111                 :            : 
    2112                 :        147 :         xas_lock_irq(&xas);
    2113         [ +  + ]:        294 :         xas_for_each_marked(&xas, page, end, PAGECACHE_TAG_DIRTY) {
    2114                 :        147 :                 xas_set_mark(&xas, PAGECACHE_TAG_TOWRITE);
    2115         [ +  - ]:        147 :                 if (++tagged % XA_CHECK_SCHED)
    2116                 :        147 :                         continue;
    2117                 :            : 
    2118                 :          0 :                 xas_pause(&xas);
    2119                 :          0 :                 xas_unlock_irq(&xas);
    2120                 :          0 :                 cond_resched();
    2121                 :          0 :                 xas_lock_irq(&xas);
    2122                 :            :         }
    2123                 :        147 :         xas_unlock_irq(&xas);
    2124                 :        147 : }
    2125                 :            : EXPORT_SYMBOL(tag_pages_for_writeback);
    2126                 :            : 
    2127                 :            : /**
    2128                 :            :  * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
    2129                 :            :  * @mapping: address space structure to write
    2130                 :            :  * @wbc: subtract the number of written pages from *@wbc->nr_to_write
    2131                 :            :  * @writepage: function called for each page
    2132                 :            :  * @data: data passed to writepage function
    2133                 :            :  *
    2134                 :            :  * If a page is already under I/O, write_cache_pages() skips it, even
    2135                 :            :  * if it's dirty.  This is desirable behaviour for memory-cleaning writeback,
    2136                 :            :  * but it is INCORRECT for data-integrity system calls such as fsync().  fsync()
    2137                 :            :  * and msync() need to guarantee that all the data which was dirty at the time
    2138                 :            :  * the call was made get new I/O started against them.  If wbc->sync_mode is
    2139                 :            :  * WB_SYNC_ALL then we were called for data integrity and we must wait for
    2140                 :            :  * existing IO to complete.
    2141                 :            :  *
    2142                 :            :  * To avoid livelocks (when other process dirties new pages), we first tag
    2143                 :            :  * pages which should be written back with TOWRITE tag and only then start
    2144                 :            :  * writing them. For data-integrity sync we have to be careful so that we do
    2145                 :            :  * not miss some pages (e.g., because some other process has cleared TOWRITE
    2146                 :            :  * tag we set). The rule we follow is that TOWRITE tag can be cleared only
    2147                 :            :  * by the process clearing the DIRTY tag (and submitting the page for IO).
    2148                 :            :  *
    2149                 :            :  * To avoid deadlocks between range_cyclic writeback and callers that hold
    2150                 :            :  * pages in PageWriteback to aggregate IO until write_cache_pages() returns,
    2151                 :            :  * we do not loop back to the start of the file. Doing so causes a page
    2152                 :            :  * lock/page writeback access order inversion - we should only ever lock
    2153                 :            :  * multiple pages in ascending page->index order, and looping back to the start
    2154                 :            :  * of the file violates that rule and causes deadlocks.
    2155                 :            :  *
    2156                 :            :  * Return: %0 on success, negative error code otherwise
    2157                 :            :  */
    2158                 :          0 : int write_cache_pages(struct address_space *mapping,
    2159                 :            :                       struct writeback_control *wbc, writepage_t writepage,
    2160                 :            :                       void *data)
    2161                 :            : {
    2162                 :          0 :         int ret = 0;
    2163                 :          0 :         int done = 0;
    2164                 :          0 :         int error;
    2165                 :          0 :         struct pagevec pvec;
    2166                 :          0 :         int nr_pages;
    2167                 :          0 :         pgoff_t uninitialized_var(writeback_index);
    2168                 :          0 :         pgoff_t index;
    2169                 :          0 :         pgoff_t end;            /* Inclusive */
    2170                 :          0 :         pgoff_t done_index;
    2171                 :          0 :         int range_whole = 0;
    2172                 :          0 :         xa_mark_t tag;
    2173                 :            : 
    2174         [ #  # ]:          0 :         pagevec_init(&pvec);
    2175         [ #  # ]:          0 :         if (wbc->range_cyclic) {
    2176                 :          0 :                 writeback_index = mapping->writeback_index; /* prev offset */
    2177                 :          0 :                 index = writeback_index;
    2178                 :          0 :                 end = -1;
    2179                 :            :         } else {
    2180                 :          0 :                 index = wbc->range_start >> PAGE_SHIFT;
    2181                 :          0 :                 end = wbc->range_end >> PAGE_SHIFT;
    2182   [ #  #  #  # ]:          0 :                 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
    2183                 :          0 :                         range_whole = 1;
    2184                 :            :         }
    2185   [ #  #  #  # ]:          0 :         if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
    2186                 :            :                 tag = PAGECACHE_TAG_TOWRITE;
    2187                 :            :         else
    2188                 :          0 :                 tag = PAGECACHE_TAG_DIRTY;
    2189   [ #  #  #  # ]:          0 :         if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
    2190                 :          0 :                 tag_pages_for_writeback(mapping, index, end);
    2191                 :          0 :         done_index = index;
    2192         [ #  # ]:          0 :         while (!done && (index <= end)) {
    2193                 :          0 :                 int i;
    2194                 :            : 
    2195                 :          0 :                 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
    2196                 :            :                                 tag);
    2197         [ #  # ]:          0 :                 if (nr_pages == 0)
    2198                 :            :                         break;
    2199                 :            : 
    2200         [ #  # ]:          0 :                 for (i = 0; i < nr_pages; i++) {
    2201                 :          0 :                         struct page *page = pvec.pages[i];
    2202                 :            : 
    2203                 :          0 :                         done_index = page->index;
    2204                 :            : 
    2205                 :          0 :                         lock_page(page);
    2206                 :            : 
    2207                 :            :                         /*
    2208                 :            :                          * Page truncated or invalidated. We can freely skip it
    2209                 :            :                          * then, even for data integrity operations: the page
    2210                 :            :                          * has disappeared concurrently, so there could be no
    2211                 :            :                          * real expectation of this data interity operation
    2212                 :            :                          * even if there is now a new, dirty page at the same
    2213                 :            :                          * pagecache address.
    2214                 :            :                          */
    2215         [ #  # ]:          0 :                         if (unlikely(page->mapping != mapping)) {
    2216                 :          0 : continue_unlock:
    2217                 :          0 :                                 unlock_page(page);
    2218                 :          0 :                                 continue;
    2219                 :            :                         }
    2220                 :            : 
    2221   [ #  #  #  # ]:          0 :                         if (!PageDirty(page)) {
    2222                 :            :                                 /* someone wrote it for us */
    2223                 :          0 :                                 goto continue_unlock;
    2224                 :            :                         }
    2225                 :            : 
    2226   [ #  #  #  # ]:          0 :                         if (PageWriteback(page)) {
    2227         [ #  # ]:          0 :                                 if (wbc->sync_mode != WB_SYNC_NONE)
    2228                 :          0 :                                         wait_on_page_writeback(page);
    2229                 :            :                                 else
    2230                 :          0 :                                         goto continue_unlock;
    2231                 :            :                         }
    2232                 :            : 
    2233   [ #  #  #  # ]:          0 :                         BUG_ON(PageWriteback(page));
    2234         [ #  # ]:          0 :                         if (!clear_page_dirty_for_io(page))
    2235                 :          0 :                                 goto continue_unlock;
    2236                 :            : 
    2237                 :          0 :                         trace_wbc_writepage(wbc, inode_to_bdi(mapping->host));
    2238                 :          0 :                         error = (*writepage)(page, wbc, data);
    2239         [ #  # ]:          0 :                         if (unlikely(error)) {
    2240                 :            :                                 /*
    2241                 :            :                                  * Handle errors according to the type of
    2242                 :            :                                  * writeback. There's no need to continue for
    2243                 :            :                                  * background writeback. Just push done_index
    2244                 :            :                                  * past this page so media errors won't choke
    2245                 :            :                                  * writeout for the entire file. For integrity
    2246                 :            :                                  * writeback, we must process the entire dirty
    2247                 :            :                                  * set regardless of errors because the fs may
    2248                 :            :                                  * still have state to clear for each page. In
    2249                 :            :                                  * that case we continue processing and return
    2250                 :            :                                  * the first error.
    2251                 :            :                                  */
    2252         [ #  # ]:          0 :                                 if (error == AOP_WRITEPAGE_ACTIVATE) {
    2253                 :          0 :                                         unlock_page(page);
    2254                 :          0 :                                         error = 0;
    2255         [ #  # ]:          0 :                                 } else if (wbc->sync_mode != WB_SYNC_ALL) {
    2256                 :          0 :                                         ret = error;
    2257                 :          0 :                                         done_index = page->index + 1;
    2258                 :          0 :                                         done = 1;
    2259                 :          0 :                                         break;
    2260                 :            :                                 }
    2261         [ #  # ]:          0 :                                 if (!ret)
    2262                 :          0 :                                         ret = error;
    2263                 :            :                         }
    2264                 :            : 
    2265                 :            :                         /*
    2266                 :            :                          * We stop writing back only if we are not doing
    2267                 :            :                          * integrity sync. In case of integrity sync we have to
    2268                 :            :                          * keep going until we have written all the pages
    2269                 :            :                          * we tagged for writeback prior to entering this loop.
    2270                 :            :                          */
    2271         [ #  # ]:          0 :                         if (--wbc->nr_to_write <= 0 &&
    2272         [ #  # ]:          0 :                             wbc->sync_mode == WB_SYNC_NONE) {
    2273                 :            :                                 done = 1;
    2274                 :            :                                 break;
    2275                 :            :                         }
    2276                 :            :                 }
    2277         [ #  # ]:          0 :                 pagevec_release(&pvec);
    2278                 :          0 :                 cond_resched();
    2279                 :            :         }
    2280                 :            : 
    2281                 :            :         /*
    2282                 :            :          * If we hit the last page and there is more work to be done: wrap
    2283                 :            :          * back the index back to the start of the file for the next
    2284                 :            :          * time we are called.
    2285                 :            :          */
    2286   [ #  #  #  # ]:          0 :         if (wbc->range_cyclic && !done)
    2287                 :          0 :                 done_index = 0;
    2288   [ #  #  #  #  :          0 :         if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
                   #  # ]
    2289                 :          0 :                 mapping->writeback_index = done_index;
    2290                 :            : 
    2291                 :          0 :         return ret;
    2292                 :            : }
    2293                 :            : EXPORT_SYMBOL(write_cache_pages);
    2294                 :            : 
    2295                 :            : /*
    2296                 :            :  * Function used by generic_writepages to call the real writepage
    2297                 :            :  * function and set the mapping flags on error
    2298                 :            :  */
    2299                 :          0 : static int __writepage(struct page *page, struct writeback_control *wbc,
    2300                 :            :                        void *data)
    2301                 :            : {
    2302                 :          0 :         struct address_space *mapping = data;
    2303                 :          0 :         int ret = mapping->a_ops->writepage(page, wbc);
    2304                 :          0 :         mapping_set_error(mapping, ret);
    2305                 :          0 :         return ret;
    2306                 :            : }
    2307                 :            : 
    2308                 :            : /**
    2309                 :            :  * generic_writepages - walk the list of dirty pages of the given address space and writepage() all of them.
    2310                 :            :  * @mapping: address space structure to write
    2311                 :            :  * @wbc: subtract the number of written pages from *@wbc->nr_to_write
    2312                 :            :  *
    2313                 :            :  * This is a library function, which implements the writepages()
    2314                 :            :  * address_space_operation.
    2315                 :            :  *
    2316                 :            :  * Return: %0 on success, negative error code otherwise
    2317                 :            :  */
    2318                 :          0 : int generic_writepages(struct address_space *mapping,
    2319                 :            :                        struct writeback_control *wbc)
    2320                 :            : {
    2321                 :          0 :         struct blk_plug plug;
    2322                 :          0 :         int ret;
    2323                 :            : 
    2324                 :            :         /* deal with chardevs and other special file */
    2325         [ #  # ]:          0 :         if (!mapping->a_ops->writepage)
    2326                 :            :                 return 0;
    2327                 :            : 
    2328                 :          0 :         blk_start_plug(&plug);
    2329                 :          0 :         ret = write_cache_pages(mapping, wbc, __writepage, mapping);
    2330                 :          0 :         blk_finish_plug(&plug);
    2331                 :          0 :         return ret;
    2332                 :            : }
    2333                 :            : 
    2334                 :            : EXPORT_SYMBOL(generic_writepages);
    2335                 :            : 
    2336                 :        168 : int do_writepages(struct address_space *mapping, struct writeback_control *wbc)
    2337                 :            : {
    2338                 :        168 :         int ret;
    2339                 :            : 
    2340         [ +  - ]:        168 :         if (wbc->nr_to_write <= 0)
    2341                 :            :                 return 0;
    2342                 :        168 :         while (1) {
    2343         [ +  - ]:        168 :                 if (mapping->a_ops->writepages)
    2344                 :        168 :                         ret = mapping->a_ops->writepages(mapping, wbc);
    2345                 :            :                 else
    2346                 :          0 :                         ret = generic_writepages(mapping, wbc);
    2347   [ -  +  -  - ]:        168 :                 if ((ret != -ENOMEM) || (wbc->sync_mode != WB_SYNC_ALL))
    2348                 :            :                         break;
    2349                 :          0 :                 cond_resched();
    2350                 :          0 :                 congestion_wait(BLK_RW_ASYNC, HZ/50);
    2351                 :            :         }
    2352                 :            :         return ret;
    2353                 :            : }
    2354                 :            : 
    2355                 :            : /**
    2356                 :            :  * write_one_page - write out a single page and wait on I/O
    2357                 :            :  * @page: the page to write
    2358                 :            :  *
    2359                 :            :  * The page must be locked by the caller and will be unlocked upon return.
    2360                 :            :  *
    2361                 :            :  * Note that the mapping's AS_EIO/AS_ENOSPC flags will be cleared when this
    2362                 :            :  * function returns.
    2363                 :            :  *
    2364                 :            :  * Return: %0 on success, negative error code otherwise
    2365                 :            :  */
    2366                 :          0 : int write_one_page(struct page *page)
    2367                 :            : {
    2368                 :          0 :         struct address_space *mapping = page->mapping;
    2369                 :          0 :         int ret = 0;
    2370                 :          0 :         struct writeback_control wbc = {
    2371                 :            :                 .sync_mode = WB_SYNC_ALL,
    2372                 :            :                 .nr_to_write = 1,
    2373                 :            :         };
    2374                 :            : 
    2375   [ #  #  #  # ]:          0 :         BUG_ON(!PageLocked(page));
    2376                 :            : 
    2377                 :          0 :         wait_on_page_writeback(page);
    2378                 :            : 
    2379         [ #  # ]:          0 :         if (clear_page_dirty_for_io(page)) {
    2380         [ #  # ]:          0 :                 get_page(page);
    2381                 :          0 :                 ret = mapping->a_ops->writepage(page, &wbc);
    2382         [ #  # ]:          0 :                 if (ret == 0)
    2383                 :          0 :                         wait_on_page_writeback(page);
    2384                 :          0 :                 put_page(page);
    2385                 :            :         } else {
    2386                 :          0 :                 unlock_page(page);
    2387                 :            :         }
    2388                 :            : 
    2389         [ #  # ]:          0 :         if (!ret)
    2390                 :          0 :                 ret = filemap_check_errors(mapping);
    2391                 :          0 :         return ret;
    2392                 :            : }
    2393                 :            : EXPORT_SYMBOL(write_one_page);
    2394                 :            : 
    2395                 :            : /*
    2396                 :            :  * For address_spaces which do not use buffers nor write back.
    2397                 :            :  */
    2398                 :      30743 : int __set_page_dirty_no_writeback(struct page *page)
    2399                 :            : {
    2400   [ -  +  +  + ]:      61486 :         if (!PageDirty(page))
    2401         [ -  + ]:      55314 :                 return !TestSetPageDirty(page);
    2402                 :            :         return 0;
    2403                 :            : }
    2404                 :            : 
    2405                 :            : /*
    2406                 :            :  * Helper function for set_page_dirty family.
    2407                 :            :  *
    2408                 :            :  * Caller must hold lock_page_memcg().
    2409                 :            :  *
    2410                 :            :  * NOTE: This relies on being atomic wrt interrupts.
    2411                 :            :  */
    2412                 :       9352 : void account_page_dirtied(struct page *page, struct address_space *mapping)
    2413                 :            : {
    2414                 :       9352 :         struct inode *inode = mapping->host;
    2415                 :            : 
    2416                 :       9352 :         trace_writeback_dirty_page(page, mapping);
    2417                 :            : 
    2418         [ +  - ]:       9352 :         if (mapping_cap_account_dirty(mapping)) {
    2419                 :       9352 :                 struct bdi_writeback *wb;
    2420                 :            : 
    2421                 :       9352 :                 inode_attach_wb(inode, page);
    2422                 :       9352 :                 wb = inode_to_wb(inode);
    2423                 :            : 
    2424                 :       9352 :                 __inc_lruvec_page_state(page, NR_FILE_DIRTY);
    2425                 :       9352 :                 __inc_zone_page_state(page, NR_ZONE_WRITE_PENDING);
    2426                 :       9352 :                 __inc_node_page_state(page, NR_DIRTIED);
    2427                 :       9352 :                 inc_wb_stat(wb, WB_RECLAIMABLE);
    2428                 :       9352 :                 inc_wb_stat(wb, WB_DIRTIED);
    2429                 :       9352 :                 task_io_account_write(PAGE_SIZE);
    2430                 :       9352 :                 current->nr_dirtied++;
    2431                 :       9352 :                 this_cpu_inc(bdp_ratelimits);
    2432                 :            : 
    2433                 :       9352 :                 mem_cgroup_track_foreign_dirty(page, wb);
    2434                 :            :         }
    2435                 :       9352 : }
    2436                 :            : 
    2437                 :            : /*
    2438                 :            :  * Helper function for deaccounting dirty page without writeback.
    2439                 :            :  *
    2440                 :            :  * Caller must hold lock_page_memcg().
    2441                 :            :  */
    2442                 :          0 : void account_page_cleaned(struct page *page, struct address_space *mapping,
    2443                 :            :                           struct bdi_writeback *wb)
    2444                 :            : {
    2445         [ #  # ]:          0 :         if (mapping_cap_account_dirty(mapping)) {
    2446                 :          0 :                 dec_lruvec_page_state(page, NR_FILE_DIRTY);
    2447                 :          0 :                 dec_zone_page_state(page, NR_ZONE_WRITE_PENDING);
    2448                 :          0 :                 dec_wb_stat(wb, WB_RECLAIMABLE);
    2449                 :          0 :                 task_io_account_cancelled_write(PAGE_SIZE);
    2450                 :            :         }
    2451                 :          0 : }
    2452                 :            : 
    2453                 :            : /*
    2454                 :            :  * For address_spaces which do not use buffers.  Just tag the page as dirty in
    2455                 :            :  * the xarray.
    2456                 :            :  *
    2457                 :            :  * This is also used when a single buffer is being dirtied: we want to set the
    2458                 :            :  * page dirty in that case, but not all the buffers.  This is a "bottom-up"
    2459                 :            :  * dirtying, whereas __set_page_dirty_buffers() is a "top-down" dirtying.
    2460                 :            :  *
    2461                 :            :  * The caller must ensure this doesn't race with truncation.  Most will simply
    2462                 :            :  * hold the page lock, but e.g. zap_pte_range() calls with the page mapped and
    2463                 :            :  * the pte lock held, which also locks out truncation.
    2464                 :            :  */
    2465                 :          0 : int __set_page_dirty_nobuffers(struct page *page)
    2466                 :            : {
    2467         [ #  # ]:          0 :         lock_page_memcg(page);
    2468   [ #  #  #  # ]:          0 :         if (!TestSetPageDirty(page)) {
    2469                 :          0 :                 struct address_space *mapping = page_mapping(page);
    2470                 :          0 :                 unsigned long flags;
    2471                 :            : 
    2472         [ #  # ]:          0 :                 if (!mapping) {
    2473                 :            :                         unlock_page_memcg(page);
    2474                 :            :                         return 1;
    2475                 :            :                 }
    2476                 :            : 
    2477                 :          0 :                 xa_lock_irqsave(&mapping->i_pages, flags);
    2478         [ #  # ]:          0 :                 BUG_ON(page_mapping(page) != mapping);
    2479   [ #  #  #  #  :          0 :                 WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page));
                   #  # ]
    2480                 :          0 :                 account_page_dirtied(page, mapping);
    2481                 :          0 :                 __xa_set_mark(&mapping->i_pages, page_index(page),
    2482                 :            :                                    PAGECACHE_TAG_DIRTY);
    2483                 :          0 :                 xa_unlock_irqrestore(&mapping->i_pages, flags);
    2484         [ #  # ]:          0 :                 unlock_page_memcg(page);
    2485                 :            : 
    2486         [ #  # ]:          0 :                 if (mapping->host) {
    2487                 :            :                         /* !PageAnon && !swapper_space */
    2488                 :          0 :                         __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
    2489                 :            :                 }
    2490                 :          0 :                 return 1;
    2491                 :            :         }
    2492                 :            :         unlock_page_memcg(page);
    2493                 :            :         return 0;
    2494                 :            : }
    2495                 :            : EXPORT_SYMBOL(__set_page_dirty_nobuffers);
    2496                 :            : 
    2497                 :            : /*
    2498                 :            :  * Call this whenever redirtying a page, to de-account the dirty counters
    2499                 :            :  * (NR_DIRTIED, WB_DIRTIED, tsk->nr_dirtied), so that they match the written
    2500                 :            :  * counters (NR_WRITTEN, WB_WRITTEN) in long term. The mismatches will lead to
    2501                 :            :  * systematic errors in balanced_dirty_ratelimit and the dirty pages position
    2502                 :            :  * control.
    2503                 :            :  */
    2504                 :          0 : void account_page_redirty(struct page *page)
    2505                 :            : {
    2506                 :          0 :         struct address_space *mapping = page->mapping;
    2507                 :            : 
    2508   [ #  #  #  # ]:          0 :         if (mapping && mapping_cap_account_dirty(mapping)) {
    2509                 :          0 :                 struct inode *inode = mapping->host;
    2510                 :          0 :                 struct bdi_writeback *wb;
    2511                 :          0 :                 struct wb_lock_cookie cookie = {};
    2512                 :            : 
    2513                 :          0 :                 wb = unlocked_inode_to_wb_begin(inode, &cookie);
    2514                 :          0 :                 current->nr_dirtied--;
    2515                 :          0 :                 dec_node_page_state(page, NR_DIRTIED);
    2516                 :          0 :                 dec_wb_stat(wb, WB_DIRTIED);
    2517                 :          0 :                 unlocked_inode_to_wb_end(inode, &cookie);
    2518                 :            :         }
    2519                 :          0 : }
    2520                 :            : EXPORT_SYMBOL(account_page_redirty);
    2521                 :            : 
    2522                 :            : /*
    2523                 :            :  * When a writepage implementation decides that it doesn't want to write this
    2524                 :            :  * page for some reason, it should redirty the locked page via
    2525                 :            :  * redirty_page_for_writepage() and it should then unlock the page and return 0
    2526                 :            :  */
    2527                 :          0 : int redirty_page_for_writepage(struct writeback_control *wbc, struct page *page)
    2528                 :            : {
    2529                 :          0 :         int ret;
    2530                 :            : 
    2531                 :          0 :         wbc->pages_skipped++;
    2532                 :          0 :         ret = __set_page_dirty_nobuffers(page);
    2533                 :          0 :         account_page_redirty(page);
    2534                 :          0 :         return ret;
    2535                 :            : }
    2536                 :            : EXPORT_SYMBOL(redirty_page_for_writepage);
    2537                 :            : 
    2538                 :            : /*
    2539                 :            :  * Dirty a page.
    2540                 :            :  *
    2541                 :            :  * For pages with a mapping this should be done under the page lock
    2542                 :            :  * for the benefit of asynchronous memory errors who prefer a consistent
    2543                 :            :  * dirty state. This rule can be broken in some special cases,
    2544                 :            :  * but should be better not to.
    2545                 :            :  *
    2546                 :            :  * If the mapping doesn't provide a set_page_dirty a_op, then
    2547                 :            :  * just fall through and assume that it wants buffer_heads.
    2548                 :            :  */
    2549                 :      30744 : int set_page_dirty(struct page *page)
    2550                 :            : {
    2551                 :      30744 :         struct address_space *mapping = page_mapping(page);
    2552                 :            : 
    2553         [ -  + ]:      30744 :         page = compound_head(page);
    2554         [ +  + ]:      30744 :         if (likely(mapping)) {
    2555                 :      30743 :                 int (*spd)(struct page *) = mapping->a_ops->set_page_dirty;
    2556                 :            :                 /*
    2557                 :            :                  * readahead/lru_deactivate_page could remain
    2558                 :            :                  * PG_readahead/PG_reclaim due to race with end_page_writeback
    2559                 :            :                  * About readahead, if the page is written, the flags would be
    2560                 :            :                  * reset. So no problem.
    2561                 :            :                  * About lru_deactivate_page, if the page is redirty, the flag
    2562                 :            :                  * will be reset. So no problem. but if the page is used by readahead
    2563                 :            :                  * it will confuse readahead and make it restart the size rampup
    2564                 :            :                  * process. But it's a trivial problem.
    2565                 :            :                  */
    2566   [ -  +  -  + ]:      61486 :                 if (PageReclaim(page))
    2567         [ #  # ]:          0 :                         ClearPageReclaim(page);
    2568                 :            : #ifdef CONFIG_BLOCK
    2569         [ -  + ]:      30743 :                 if (!spd)
    2570                 :          0 :                         spd = __set_page_dirty_buffers;
    2571                 :            : #endif
    2572                 :      30743 :                 return (*spd)(page);
    2573                 :            :         }
    2574   [ -  +  +  - ]:          2 :         if (!PageDirty(page)) {
    2575   [ -  +  +  - ]:          2 :                 if (!TestSetPageDirty(page))
    2576                 :          1 :                         return 1;
    2577                 :            :         }
    2578                 :            :         return 0;
    2579                 :            : }
    2580                 :            : EXPORT_SYMBOL(set_page_dirty);
    2581                 :            : 
    2582                 :            : /*
    2583                 :            :  * set_page_dirty() is racy if the caller has no reference against
    2584                 :            :  * page->mapping->host, and if the page is unlocked.  This is because another
    2585                 :            :  * CPU could truncate the page off the mapping and then free the mapping.
    2586                 :            :  *
    2587                 :            :  * Usually, the page _is_ locked, or the caller is a user-space process which
    2588                 :            :  * holds a reference on the inode by having an open file.
    2589                 :            :  *
    2590                 :            :  * In other cases, the page should be locked before running set_page_dirty().
    2591                 :            :  */
    2592                 :          1 : int set_page_dirty_lock(struct page *page)
    2593                 :            : {
    2594                 :          1 :         int ret;
    2595                 :            : 
    2596                 :          1 :         lock_page(page);
    2597                 :          1 :         ret = set_page_dirty(page);
    2598                 :          1 :         unlock_page(page);
    2599                 :          1 :         return ret;
    2600                 :            : }
    2601                 :            : EXPORT_SYMBOL(set_page_dirty_lock);
    2602                 :            : 
    2603                 :            : /*
    2604                 :            :  * This cancels just the dirty bit on the kernel page itself, it does NOT
    2605                 :            :  * actually remove dirty bits on any mmap's that may be around. It also
    2606                 :            :  * leaves the page tagged dirty, so any sync activity will still find it on
    2607                 :            :  * the dirty lists, and in particular, clear_page_dirty_for_io() will still
    2608                 :            :  * look at the dirty bits in the VM.
    2609                 :            :  *
    2610                 :            :  * Doing this should *normally* only ever be done when a page is truncated,
    2611                 :            :  * and is not actually mapped anywhere at all. However, fs/buffer.c does
    2612                 :            :  * this when it notices that somebody has cleaned out all the buffers on a
    2613                 :            :  * page without actually doing it through the VM. Can you say "ext3 is
    2614                 :            :  * horribly ugly"? Thought you could.
    2615                 :            :  */
    2616                 :        735 : void __cancel_dirty_page(struct page *page)
    2617                 :            : {
    2618                 :        735 :         struct address_space *mapping = page_mapping(page);
    2619                 :            : 
    2620         [ -  + ]:        735 :         if (mapping_cap_account_dirty(mapping)) {
    2621                 :          0 :                 struct inode *inode = mapping->host;
    2622                 :          0 :                 struct bdi_writeback *wb;
    2623                 :          0 :                 struct wb_lock_cookie cookie = {};
    2624                 :            : 
    2625                 :          0 :                 lock_page_memcg(page);
    2626                 :          0 :                 wb = unlocked_inode_to_wb_begin(inode, &cookie);
    2627                 :            : 
    2628   [ #  #  #  # ]:          0 :                 if (TestClearPageDirty(page))
    2629                 :          0 :                         account_page_cleaned(page, mapping, wb);
    2630                 :            : 
    2631                 :          0 :                 unlocked_inode_to_wb_end(inode, &cookie);
    2632                 :          0 :                 unlock_page_memcg(page);
    2633                 :            :         } else {
    2634         [ -  + ]:        735 :                 ClearPageDirty(page);
    2635                 :            :         }
    2636                 :        735 : }
    2637                 :            : EXPORT_SYMBOL(__cancel_dirty_page);
    2638                 :            : 
    2639                 :            : /*
    2640                 :            :  * Clear a page's dirty flag, while caring for dirty memory accounting.
    2641                 :            :  * Returns true if the page was previously dirty.
    2642                 :            :  *
    2643                 :            :  * This is for preparing to put the page under writeout.  We leave the page
    2644                 :            :  * tagged as dirty in the xarray so that a concurrent write-for-sync
    2645                 :            :  * can discover it via a PAGECACHE_TAG_DIRTY walk.  The ->writepage
    2646                 :            :  * implementation will run either set_page_writeback() or set_page_dirty(),
    2647                 :            :  * at which stage we bring the page's dirty flag and xarray dirty tag
    2648                 :            :  * back into sync.
    2649                 :            :  *
    2650                 :            :  * This incoherency between the page's dirty flag and xarray tag is
    2651                 :            :  * unfortunate, but it only exists while the page is locked.
    2652                 :            :  */
    2653                 :        168 : int clear_page_dirty_for_io(struct page *page)
    2654                 :            : {
    2655                 :        168 :         struct address_space *mapping = page_mapping(page);
    2656                 :        168 :         int ret = 0;
    2657                 :            : 
    2658   [ -  +  -  + ]:        336 :         BUG_ON(!PageLocked(page));
    2659                 :            : 
    2660   [ +  -  +  - ]:        168 :         if (mapping && mapping_cap_account_dirty(mapping)) {
    2661                 :        168 :                 struct inode *inode = mapping->host;
    2662                 :        168 :                 struct bdi_writeback *wb;
    2663                 :        168 :                 struct wb_lock_cookie cookie = {};
    2664                 :            : 
    2665                 :            :                 /*
    2666                 :            :                  * Yes, Virginia, this is indeed insane.
    2667                 :            :                  *
    2668                 :            :                  * We use this sequence to make sure that
    2669                 :            :                  *  (a) we account for dirty stats properly
    2670                 :            :                  *  (b) we tell the low-level filesystem to
    2671                 :            :                  *      mark the whole page dirty if it was
    2672                 :            :                  *      dirty in a pagetable. Only to then
    2673                 :            :                  *  (c) clean the page again and return 1 to
    2674                 :            :                  *      cause the writeback.
    2675                 :            :                  *
    2676                 :            :                  * This way we avoid all nasty races with the
    2677                 :            :                  * dirty bit in multiple places and clearing
    2678                 :            :                  * them concurrently from different threads.
    2679                 :            :                  *
    2680                 :            :                  * Note! Normally the "set_page_dirty(page)"
    2681                 :            :                  * has no effect on the actual dirty bit - since
    2682                 :            :                  * that will already usually be set. But we
    2683                 :            :                  * need the side effects, and it can help us
    2684                 :            :                  * avoid races.
    2685                 :            :                  *
    2686                 :            :                  * We basically use the page "master dirty bit"
    2687                 :            :                  * as a serialization point for all the different
    2688                 :            :                  * threads doing their things.
    2689                 :            :                  */
    2690         [ -  + ]:        168 :                 if (page_mkclean(page))
    2691                 :          0 :                         set_page_dirty(page);
    2692                 :            :                 /*
    2693                 :            :                  * We carefully synchronise fault handlers against
    2694                 :            :                  * installing a dirty pte and marking the page dirty
    2695                 :            :                  * at this point.  We do this by having them hold the
    2696                 :            :                  * page lock while dirtying the page, and pages are
    2697                 :            :                  * always locked coming in here, so we get the desired
    2698                 :            :                  * exclusion.
    2699                 :            :                  */
    2700                 :        168 :                 wb = unlocked_inode_to_wb_begin(inode, &cookie);
    2701   [ -  +  +  - ]:        336 :                 if (TestClearPageDirty(page)) {
    2702                 :        168 :                         dec_lruvec_page_state(page, NR_FILE_DIRTY);
    2703                 :        168 :                         dec_zone_page_state(page, NR_ZONE_WRITE_PENDING);
    2704                 :        168 :                         dec_wb_stat(wb, WB_RECLAIMABLE);
    2705                 :        168 :                         ret = 1;
    2706                 :            :                 }
    2707                 :        168 :                 unlocked_inode_to_wb_end(inode, &cookie);
    2708                 :        168 :                 return ret;
    2709                 :            :         }
    2710         [ #  # ]:          0 :         return TestClearPageDirty(page);
    2711                 :            : }
    2712                 :            : EXPORT_SYMBOL(clear_page_dirty_for_io);
    2713                 :            : 
    2714                 :        168 : int test_clear_page_writeback(struct page *page)
    2715                 :            : {
    2716                 :        168 :         struct address_space *mapping = page_mapping(page);
    2717                 :        168 :         struct mem_cgroup *memcg;
    2718                 :        168 :         struct lruvec *lruvec;
    2719                 :        168 :         int ret;
    2720                 :            : 
    2721         [ +  - ]:        168 :         memcg = lock_page_memcg(page);
    2722         [ +  - ]:        168 :         lruvec = mem_cgroup_page_lruvec(page, page_pgdat(page));
    2723   [ +  -  +  - ]:        336 :         if (mapping && mapping_use_writeback_tags(mapping)) {
    2724                 :        168 :                 struct inode *inode = mapping->host;
    2725                 :        168 :                 struct backing_dev_info *bdi = inode_to_bdi(inode);
    2726                 :        168 :                 unsigned long flags;
    2727                 :            : 
    2728                 :        168 :                 xa_lock_irqsave(&mapping->i_pages, flags);
    2729         [ -  + ]:        168 :                 ret = TestClearPageWriteback(page);
    2730         [ +  - ]:        168 :                 if (ret) {
    2731                 :        168 :                         __xa_clear_mark(&mapping->i_pages, page_index(page),
    2732                 :            :                                                 PAGECACHE_TAG_WRITEBACK);
    2733         [ +  - ]:        168 :                         if (bdi_cap_account_writeback(bdi)) {
    2734                 :        168 :                                 struct bdi_writeback *wb = inode_to_wb(inode);
    2735                 :            : 
    2736                 :        168 :                                 dec_wb_stat(wb, WB_WRITEBACK);
    2737                 :        168 :                                 __wb_writeout_inc(wb);
    2738                 :            :                         }
    2739                 :            :                 }
    2740                 :            : 
    2741   [ +  -  +  - ]:        168 :                 if (mapping->host && !mapping_tagged(mapping,
    2742                 :            :                                                      PAGECACHE_TAG_WRITEBACK))
    2743                 :        168 :                         sb_clear_inode_writeback(mapping->host);
    2744                 :            : 
    2745                 :        168 :                 xa_unlock_irqrestore(&mapping->i_pages, flags);
    2746                 :            :         } else {
    2747         [ #  # ]:          0 :                 ret = TestClearPageWriteback(page);
    2748                 :            :         }
    2749                 :            :         /*
    2750                 :            :          * NOTE: Page might be free now! Writeback doesn't hold a page
    2751                 :            :          * reference on its own, it relies on truncation to wait for
    2752                 :            :          * the clearing of PG_writeback. The below can only access
    2753                 :            :          * page state that is static across allocation cycles.
    2754                 :            :          */
    2755         [ +  - ]:        168 :         if (ret) {
    2756                 :        168 :                 dec_lruvec_state(lruvec, NR_WRITEBACK);
    2757                 :        168 :                 dec_zone_page_state(page, NR_ZONE_WRITE_PENDING);
    2758                 :        168 :                 inc_node_page_state(page, NR_WRITTEN);
    2759                 :            :         }
    2760                 :        168 :         __unlock_page_memcg(memcg);
    2761                 :        168 :         return ret;
    2762                 :            : }
    2763                 :            : 
    2764                 :        168 : int __test_set_page_writeback(struct page *page, bool keep_write)
    2765                 :            : {
    2766                 :        168 :         struct address_space *mapping = page_mapping(page);
    2767                 :        168 :         int ret;
    2768                 :            : 
    2769         [ +  - ]:        168 :         lock_page_memcg(page);
    2770   [ +  -  +  - ]:        504 :         if (mapping && mapping_use_writeback_tags(mapping)) {
    2771                 :        168 :                 XA_STATE(xas, &mapping->i_pages, page_index(page));
    2772                 :        168 :                 struct inode *inode = mapping->host;
    2773                 :        168 :                 struct backing_dev_info *bdi = inode_to_bdi(inode);
    2774                 :        168 :                 unsigned long flags;
    2775                 :            : 
    2776                 :        168 :                 xas_lock_irqsave(&xas, flags);
    2777                 :        168 :                 xas_load(&xas);
    2778         [ -  + ]:        168 :                 ret = TestSetPageWriteback(page);
    2779         [ +  - ]:        168 :                 if (!ret) {
    2780                 :        168 :                         bool on_wblist;
    2781                 :            : 
    2782                 :        168 :                         on_wblist = mapping_tagged(mapping,
    2783                 :            :                                                    PAGECACHE_TAG_WRITEBACK);
    2784                 :            : 
    2785                 :        168 :                         xas_set_mark(&xas, PAGECACHE_TAG_WRITEBACK);
    2786         [ +  - ]:        168 :                         if (bdi_cap_account_writeback(bdi))
    2787                 :        168 :                                 inc_wb_stat(inode_to_wb(inode), WB_WRITEBACK);
    2788                 :            : 
    2789                 :            :                         /*
    2790                 :            :                          * We can come through here when swapping anonymous
    2791                 :            :                          * pages, so we don't necessarily have an inode to track
    2792                 :            :                          * for sync.
    2793                 :            :                          */
    2794   [ +  -  +  - ]:        168 :                         if (mapping->host && !on_wblist)
    2795                 :        168 :                                 sb_mark_inode_writeback(mapping->host);
    2796                 :            :                 }
    2797   [ -  +  +  - ]:        336 :                 if (!PageDirty(page))
    2798                 :        168 :                         xas_clear_mark(&xas, PAGECACHE_TAG_DIRTY);
    2799         [ +  - ]:        168 :                 if (!keep_write)
    2800                 :        168 :                         xas_clear_mark(&xas, PAGECACHE_TAG_TOWRITE);
    2801                 :        168 :                 xas_unlock_irqrestore(&xas, flags);
    2802                 :            :         } else {
    2803         [ #  # ]:          0 :                 ret = TestSetPageWriteback(page);
    2804                 :            :         }
    2805         [ +  - ]:        168 :         if (!ret) {
    2806                 :        168 :                 inc_lruvec_page_state(page, NR_WRITEBACK);
    2807                 :        168 :                 inc_zone_page_state(page, NR_ZONE_WRITE_PENDING);
    2808                 :            :         }
    2809                 :        168 :         unlock_page_memcg(page);
    2810                 :        168 :         return ret;
    2811                 :            : 
    2812                 :            : }
    2813                 :            : EXPORT_SYMBOL(__test_set_page_writeback);
    2814                 :            : 
    2815                 :            : /*
    2816                 :            :  * Wait for a page to complete writeback
    2817                 :            :  */
    2818                 :        483 : void wait_on_page_writeback(struct page *page)
    2819                 :            : {
    2820   [ -  +  +  + ]:        966 :         if (PageWriteback(page)) {
    2821                 :        147 :                 trace_wait_on_page_writeback(page, page_mapping(page));
    2822                 :        147 :                 wait_on_page_bit(page, PG_writeback);
    2823                 :            :         }
    2824                 :        483 : }
    2825                 :            : EXPORT_SYMBOL_GPL(wait_on_page_writeback);
    2826                 :            : 
    2827                 :            : /**
    2828                 :            :  * wait_for_stable_page() - wait for writeback to finish, if necessary.
    2829                 :            :  * @page:       The page to wait on.
    2830                 :            :  *
    2831                 :            :  * This function determines if the given page is related to a backing device
    2832                 :            :  * that requires page contents to be held stable during writeback.  If so, then
    2833                 :            :  * it will wait for any pending writeback to complete.
    2834                 :            :  */
    2835                 :      22674 : void wait_for_stable_page(struct page *page)
    2836                 :            : {
    2837         [ -  + ]:      22674 :         if (bdi_cap_stable_pages_required(inode_to_bdi(page->mapping->host)))
    2838                 :          0 :                 wait_on_page_writeback(page);
    2839                 :      22674 : }
    2840                 :            : EXPORT_SYMBOL_GPL(wait_for_stable_page);

Generated by: LCOV version 1.14