LCOV - code coverage report
Current view: top level - kernel/rcu - tree.c (source / functions) Hit Total Coverage
Test: combined.info Lines: 892 1399 63.8 %
Date: 2022-03-28 13:20:08 Functions: 63 93 67.7 %
Branches: 306 719 42.6 %

           Branch data     Line data    Source code
       1                 :            : // SPDX-License-Identifier: GPL-2.0+
       2                 :            : /*
       3                 :            :  * Read-Copy Update mechanism for mutual exclusion
       4                 :            :  *
       5                 :            :  * Copyright IBM Corporation, 2008
       6                 :            :  *
       7                 :            :  * Authors: Dipankar Sarma <dipankar@in.ibm.com>
       8                 :            :  *          Manfred Spraul <manfred@colorfullife.com>
       9                 :            :  *          Paul E. McKenney <paulmck@linux.ibm.com> Hierarchical version
      10                 :            :  *
      11                 :            :  * Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
      12                 :            :  * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
      13                 :            :  *
      14                 :            :  * For detailed explanation of Read-Copy Update mechanism see -
      15                 :            :  *      Documentation/RCU
      16                 :            :  */
      17                 :            : 
      18                 :            : #define pr_fmt(fmt) "rcu: " fmt
      19                 :            : 
      20                 :            : #include <linux/types.h>
      21                 :            : #include <linux/kernel.h>
      22                 :            : #include <linux/init.h>
      23                 :            : #include <linux/spinlock.h>
      24                 :            : #include <linux/smp.h>
      25                 :            : #include <linux/rcupdate_wait.h>
      26                 :            : #include <linux/interrupt.h>
      27                 :            : #include <linux/sched.h>
      28                 :            : #include <linux/sched/debug.h>
      29                 :            : #include <linux/nmi.h>
      30                 :            : #include <linux/atomic.h>
      31                 :            : #include <linux/bitops.h>
      32                 :            : #include <linux/export.h>
      33                 :            : #include <linux/completion.h>
      34                 :            : #include <linux/moduleparam.h>
      35                 :            : #include <linux/percpu.h>
      36                 :            : #include <linux/notifier.h>
      37                 :            : #include <linux/cpu.h>
      38                 :            : #include <linux/mutex.h>
      39                 :            : #include <linux/time.h>
      40                 :            : #include <linux/kernel_stat.h>
      41                 :            : #include <linux/wait.h>
      42                 :            : #include <linux/kthread.h>
      43                 :            : #include <uapi/linux/sched/types.h>
      44                 :            : #include <linux/prefetch.h>
      45                 :            : #include <linux/delay.h>
      46                 :            : #include <linux/random.h>
      47                 :            : #include <linux/trace_events.h>
      48                 :            : #include <linux/suspend.h>
      49                 :            : #include <linux/ftrace.h>
      50                 :            : #include <linux/tick.h>
      51                 :            : #include <linux/sysrq.h>
      52                 :            : #include <linux/kprobes.h>
      53                 :            : #include <linux/gfp.h>
      54                 :            : #include <linux/oom.h>
      55                 :            : #include <linux/smpboot.h>
      56                 :            : #include <linux/jiffies.h>
      57                 :            : #include <linux/slab.h>
      58                 :            : #include <linux/sched/isolation.h>
      59                 :            : #include <linux/sched/clock.h>
      60                 :            : #include "../time/tick-internal.h"
      61                 :            : 
      62                 :            : #include "tree.h"
      63                 :            : #include "rcu.h"
      64                 :            : 
      65                 :            : #ifdef MODULE_PARAM_PREFIX
      66                 :            : #undef MODULE_PARAM_PREFIX
      67                 :            : #endif
      68                 :            : #define MODULE_PARAM_PREFIX "rcutree."
      69                 :            : 
      70                 :            : /* Data structures. */
      71                 :            : 
      72                 :            : /*
      73                 :            :  * Steal a bit from the bottom of ->dynticks for idle entry/exit
      74                 :            :  * control.  Initially this is for TLB flushing.
      75                 :            :  */
      76                 :            : #define RCU_DYNTICK_CTRL_MASK 0x1
      77                 :            : #define RCU_DYNTICK_CTRL_CTR  (RCU_DYNTICK_CTRL_MASK + 1)
      78                 :            : #ifndef rcu_eqs_special_exit
      79                 :            : #define rcu_eqs_special_exit() do { } while (0)
      80                 :            : #endif
      81                 :            : 
      82                 :            : static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = {
      83                 :            :         .dynticks_nesting = 1,
      84                 :            :         .dynticks_nmi_nesting = DYNTICK_IRQ_NONIDLE,
      85                 :            :         .dynticks = ATOMIC_INIT(RCU_DYNTICK_CTRL_CTR),
      86                 :            : };
      87                 :            : static struct rcu_state rcu_state = {
      88                 :            :         .level = { &rcu_state.node[0] },
      89                 :            :         .gp_state = RCU_GP_IDLE,
      90                 :            :         .gp_seq = (0UL - 300UL) << RCU_SEQ_CTR_SHIFT,
      91                 :            :         .barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex),
      92                 :            :         .name = RCU_NAME,
      93                 :            :         .abbr = RCU_ABBR,
      94                 :            :         .exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex),
      95                 :            :         .exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex),
      96                 :            :         .ofl_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.ofl_lock),
      97                 :            : };
      98                 :            : 
      99                 :            : /* Dump rcu_node combining tree at boot to verify correct setup. */
     100                 :            : static bool dump_tree;
     101                 :            : module_param(dump_tree, bool, 0444);
     102                 :            : /* By default, use RCU_SOFTIRQ instead of rcuc kthreads. */
     103                 :            : static bool use_softirq = 1;
     104                 :            : module_param(use_softirq, bool, 0444);
     105                 :            : /* Control rcu_node-tree auto-balancing at boot time. */
     106                 :            : static bool rcu_fanout_exact;
     107                 :            : module_param(rcu_fanout_exact, bool, 0444);
     108                 :            : /* Increase (but not decrease) the RCU_FANOUT_LEAF at boot time. */
     109                 :            : static int rcu_fanout_leaf = RCU_FANOUT_LEAF;
     110                 :            : module_param(rcu_fanout_leaf, int, 0444);
     111                 :            : int rcu_num_lvls __read_mostly = RCU_NUM_LVLS;
     112                 :            : /* Number of rcu_nodes at specified level. */
     113                 :            : int num_rcu_lvl[] = NUM_RCU_LVL_INIT;
     114                 :            : int rcu_num_nodes __read_mostly = NUM_RCU_NODES; /* Total # rcu_nodes in use. */
     115                 :            : 
     116                 :            : /*
     117                 :            :  * The rcu_scheduler_active variable is initialized to the value
     118                 :            :  * RCU_SCHEDULER_INACTIVE and transitions RCU_SCHEDULER_INIT just before the
     119                 :            :  * first task is spawned.  So when this variable is RCU_SCHEDULER_INACTIVE,
     120                 :            :  * RCU can assume that there is but one task, allowing RCU to (for example)
     121                 :            :  * optimize synchronize_rcu() to a simple barrier().  When this variable
     122                 :            :  * is RCU_SCHEDULER_INIT, RCU must actually do all the hard work required
     123                 :            :  * to detect real grace periods.  This variable is also used to suppress
     124                 :            :  * boot-time false positives from lockdep-RCU error checking.  Finally, it
     125                 :            :  * transitions from RCU_SCHEDULER_INIT to RCU_SCHEDULER_RUNNING after RCU
     126                 :            :  * is fully initialized, including all of its kthreads having been spawned.
     127                 :            :  */
     128                 :            : int rcu_scheduler_active __read_mostly;
     129                 :            : EXPORT_SYMBOL_GPL(rcu_scheduler_active);
     130                 :            : 
     131                 :            : /*
     132                 :            :  * The rcu_scheduler_fully_active variable transitions from zero to one
     133                 :            :  * during the early_initcall() processing, which is after the scheduler
     134                 :            :  * is capable of creating new tasks.  So RCU processing (for example,
     135                 :            :  * creating tasks for RCU priority boosting) must be delayed until after
     136                 :            :  * rcu_scheduler_fully_active transitions from zero to one.  We also
     137                 :            :  * currently delay invocation of any RCU callbacks until after this point.
     138                 :            :  *
     139                 :            :  * It might later prove better for people registering RCU callbacks during
     140                 :            :  * early boot to take responsibility for these callbacks, but one step at
     141                 :            :  * a time.
     142                 :            :  */
     143                 :            : static int rcu_scheduler_fully_active __read_mostly;
     144                 :            : 
     145                 :            : static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp,
     146                 :            :                               unsigned long gps, unsigned long flags);
     147                 :            : static void rcu_init_new_rnp(struct rcu_node *rnp_leaf);
     148                 :            : static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf);
     149                 :            : static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);
     150                 :            : static void invoke_rcu_core(void);
     151                 :            : static void rcu_report_exp_rdp(struct rcu_data *rdp);
     152                 :            : static void sync_sched_exp_online_cleanup(int cpu);
     153                 :            : 
     154                 :            : /* rcuc/rcub kthread realtime priority */
     155                 :            : static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;
     156                 :            : module_param(kthread_prio, int, 0444);
     157                 :            : 
     158                 :            : /* Delay in jiffies for grace-period initialization delays, debug only. */
     159                 :            : 
     160                 :            : static int gp_preinit_delay;
     161                 :            : module_param(gp_preinit_delay, int, 0444);
     162                 :            : static int gp_init_delay;
     163                 :            : module_param(gp_init_delay, int, 0444);
     164                 :            : static int gp_cleanup_delay;
     165                 :            : module_param(gp_cleanup_delay, int, 0444);
     166                 :            : 
     167                 :            : /* Retrieve RCU kthreads priority for rcutorture */
     168                 :          0 : int rcu_get_gp_kthreads_prio(void)
     169                 :            : {
     170                 :          0 :         return kthread_prio;
     171                 :            : }
     172                 :            : EXPORT_SYMBOL_GPL(rcu_get_gp_kthreads_prio);
     173                 :            : 
     174                 :            : /*
     175                 :            :  * Number of grace periods between delays, normalized by the duration of
     176                 :            :  * the delay.  The longer the delay, the more the grace periods between
     177                 :            :  * each delay.  The reason for this normalization is that it means that,
     178                 :            :  * for non-zero delays, the overall slowdown of grace periods is constant
     179                 :            :  * regardless of the duration of the delay.  This arrangement balances
     180                 :            :  * the need for long delays to increase some race probabilities with the
     181                 :            :  * need for fast grace periods to increase other race probabilities.
     182                 :            :  */
     183                 :            : #define PER_RCU_NODE_PERIOD 3   /* Number of grace periods between delays. */
     184                 :            : 
     185                 :            : /*
     186                 :            :  * Compute the mask of online CPUs for the specified rcu_node structure.
     187                 :            :  * This will not be stable unless the rcu_node structure's ->lock is
     188                 :            :  * held, but the bit corresponding to the current CPU will be stable
     189                 :            :  * in most contexts.
     190                 :            :  */
     191                 :          0 : static unsigned long rcu_rnp_online_cpus(struct rcu_node *rnp)
     192                 :            : {
     193                 :          0 :         return READ_ONCE(rnp->qsmaskinitnext);
     194                 :            : }
     195                 :            : 
     196                 :            : /*
     197                 :            :  * Return true if an RCU grace period is in progress.  The READ_ONCE()s
     198                 :            :  * permit this function to be invoked without holding the root rcu_node
     199                 :            :  * structure's ->lock, but of course results can be subject to change.
     200                 :            :  */
     201                 :     480258 : static int rcu_gp_in_progress(void)
     202                 :            : {
     203   [ #  #  #  # ]:          0 :         return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq));
     204                 :            : }
     205                 :            : 
     206                 :            : /*
     207                 :            :  * Return the number of callbacks queued on the specified CPU.
     208                 :            :  * Handles both the nocbs and normal cases.
     209                 :            :  */
     210                 :          0 : static long rcu_get_n_cbs_cpu(int cpu)
     211                 :            : {
     212                 :          0 :         struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
     213                 :            : 
     214   [ #  #  #  #  :          0 :         if (rcu_segcblist_is_enabled(&rdp->cblist))
                   #  # ]
     215                 :          0 :                 return rcu_segcblist_n_cbs(&rdp->cblist);
     216                 :            :         return 0;
     217                 :            : }
     218                 :            : 
     219                 :     194049 : void rcu_softirq_qs(void)
     220                 :            : {
     221                 :     194049 :         rcu_qs();
     222                 :     194049 :         rcu_preempt_deferred_qs(current);
     223                 :     194049 : }
     224                 :            : 
     225                 :            : /*
     226                 :            :  * Record entry into an extended quiescent state.  This is only to be
     227                 :            :  * called when not already in an extended quiescent state.
     228                 :            :  */
     229                 :      22123 : static void rcu_dynticks_eqs_enter(void)
     230                 :            : {
     231                 :      44246 :         struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
     232                 :      22123 :         int seq;
     233                 :            : 
     234                 :            :         /*
     235                 :            :          * CPUs seeing atomic_add_return() must see prior RCU read-side
     236                 :            :          * critical sections, and we also must force ordering with the
     237                 :            :          * next idle sojourn.
     238                 :            :          */
     239                 :      22123 :         seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
     240                 :            :         /* Better be in an extended quiescent state! */
     241                 :      22123 :         WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
     242                 :            :                      (seq & RCU_DYNTICK_CTRL_CTR));
     243                 :            :         /* Better not have special action (TLB flush) pending! */
     244                 :      22123 :         WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
     245                 :            :                      (seq & RCU_DYNTICK_CTRL_MASK));
     246                 :      22123 : }
     247                 :            : 
     248                 :            : /*
     249                 :            :  * Record exit from an extended quiescent state.  This is only to be
     250                 :            :  * called from an extended quiescent state.
     251                 :            :  */
     252                 :      22123 : static void rcu_dynticks_eqs_exit(void)
     253                 :            : {
     254                 :      22123 :         struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
     255                 :      22123 :         int seq;
     256                 :            : 
     257                 :            :         /*
     258                 :            :          * CPUs seeing atomic_add_return() must see prior idle sojourns,
     259                 :            :          * and we also must force ordering with the next RCU read-side
     260                 :            :          * critical section.
     261                 :            :          */
     262                 :      22123 :         seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
     263                 :      22123 :         WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
     264                 :            :                      !(seq & RCU_DYNTICK_CTRL_CTR));
     265         [ -  + ]:      22123 :         if (seq & RCU_DYNTICK_CTRL_MASK) {
     266                 :          0 :                 atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdp->dynticks);
     267                 :      22123 :                 smp_mb__after_atomic(); /* _exit after clearing mask. */
     268                 :            :                 /* Prefer duplicate flushes to losing a flush. */
     269                 :      22123 :                 rcu_eqs_special_exit();
     270                 :            :         }
     271                 :      22123 : }
     272                 :            : 
     273                 :            : /*
     274                 :            :  * Reset the current CPU's ->dynticks counter to indicate that the
     275                 :            :  * newly onlined CPU is no longer in an extended quiescent state.
     276                 :            :  * This will either leave the counter unchanged, or increment it
     277                 :            :  * to the next non-quiescent value.
     278                 :            :  *
     279                 :            :  * The non-atomic test/increment sequence works because the upper bits
     280                 :            :  * of the ->dynticks counter are manipulated only by the corresponding CPU,
     281                 :            :  * or when the corresponding CPU is offline.
     282                 :            :  */
     283                 :         30 : static void rcu_dynticks_eqs_online(void)
     284                 :            : {
     285                 :         30 :         struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
     286                 :            : 
     287         [ -  + ]:         30 :         if (atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR)
     288                 :            :                 return;
     289                 :          0 :         atomic_add(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
     290                 :            : }
     291                 :            : 
     292                 :            : /*
     293                 :            :  * Is the current CPU in an extended quiescent state?
     294                 :            :  *
     295                 :            :  * No ordering, as we are sampling CPU-local information.
     296                 :            :  */
     297                 :    8009351 : static bool rcu_dynticks_curr_cpu_in_eqs(void)
     298                 :            : {
     299                 :         30 :         struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
     300                 :            : 
     301                 :    8009351 :         return !(atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR);
     302                 :            : }
     303                 :            : 
     304                 :            : /*
     305                 :            :  * Snapshot the ->dynticks counter with full ordering so as to allow
     306                 :            :  * stable comparison of this counter with past and future snapshots.
     307                 :            :  */
     308                 :         30 : static int rcu_dynticks_snap(struct rcu_data *rdp)
     309                 :            : {
     310                 :          0 :         int snap = atomic_add_return(0, &rdp->dynticks);
     311                 :            : 
     312   [ #  #  #  #  :          0 :         return snap & ~RCU_DYNTICK_CTRL_MASK;
                   #  # ]
     313                 :            : }
     314                 :            : 
     315                 :            : /*
     316                 :            :  * Return true if the snapshot returned from rcu_dynticks_snap()
     317                 :            :  * indicates that RCU is in an extended quiescent state.
     318                 :            :  */
     319                 :         30 : static bool rcu_dynticks_in_eqs(int snap)
     320                 :            : {
     321         [ #  # ]:          0 :         return !(snap & RCU_DYNTICK_CTRL_CTR);
     322                 :            : }
     323                 :            : 
     324                 :            : /*
     325                 :            :  * Return true if the CPU corresponding to the specified rcu_data
     326                 :            :  * structure has spent some time in an extended quiescent state since
     327                 :            :  * rcu_dynticks_snap() returned the specified snapshot.
     328                 :            :  */
     329                 :          0 : static bool rcu_dynticks_in_eqs_since(struct rcu_data *rdp, int snap)
     330                 :            : {
     331                 :          0 :         return snap != rcu_dynticks_snap(rdp);
     332                 :            : }
     333                 :            : 
     334                 :            : /*
     335                 :            :  * Set the special (bottom) bit of the specified CPU so that it
     336                 :            :  * will take special action (such as flushing its TLB) on the
     337                 :            :  * next exit from an extended quiescent state.  Returns true if
     338                 :            :  * the bit was successfully set, or false if the CPU was not in
     339                 :            :  * an extended quiescent state.
     340                 :            :  */
     341                 :          0 : bool rcu_eqs_special_set(int cpu)
     342                 :            : {
     343                 :          0 :         int old;
     344                 :          0 :         int new;
     345                 :          0 :         struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
     346                 :            : 
     347                 :          0 :         do {
     348                 :          0 :                 old = atomic_read(&rdp->dynticks);
     349         [ #  # ]:          0 :                 if (old & RCU_DYNTICK_CTRL_CTR)
     350                 :            :                         return false;
     351                 :          0 :                 new = old | RCU_DYNTICK_CTRL_MASK;
     352         [ #  # ]:          0 :         } while (atomic_cmpxchg(&rdp->dynticks, old, new) != old);
     353                 :            :         return true;
     354                 :            : }
     355                 :            : 
     356                 :            : /*
     357                 :            :  * Let the RCU core know that this CPU has gone through the scheduler,
     358                 :            :  * which is a quiescent state.  This is called when the need for a
     359                 :            :  * quiescent state is urgent, so we burn an atomic operation and full
     360                 :            :  * memory barriers to let the RCU core know about it, regardless of what
     361                 :            :  * this CPU might (or might not) do in the near future.
     362                 :            :  *
     363                 :            :  * We inform the RCU core by emulating a zero-duration dyntick-idle period.
     364                 :            :  *
     365                 :            :  * The caller must have disabled interrupts and must not be idle.
     366                 :            :  */
     367                 :        240 : void rcu_momentary_dyntick_idle(void)
     368                 :            : {
     369                 :        240 :         int special;
     370                 :            : 
     371                 :        240 :         raw_cpu_write(rcu_data.rcu_need_heavy_qs, false);
     372                 :        480 :         special = atomic_add_return(2 * RCU_DYNTICK_CTRL_CTR,
     373                 :        240 :                                     &this_cpu_ptr(&rcu_data)->dynticks);
     374                 :            :         /* It is illegal to call this from idle state. */
     375         [ -  + ]:        240 :         WARN_ON_ONCE(!(special & RCU_DYNTICK_CTRL_CTR));
     376                 :        240 :         rcu_preempt_deferred_qs(current);
     377                 :        240 : }
     378                 :            : EXPORT_SYMBOL_GPL(rcu_momentary_dyntick_idle);
     379                 :            : 
     380                 :            : /**
     381                 :            :  * rcu_is_cpu_rrupt_from_idle - see if interrupted from idle
     382                 :            :  *
     383                 :            :  * If the current CPU is idle and running at a first-level (not nested)
     384                 :            :  * interrupt from idle, return true.  The caller must have at least
     385                 :            :  * disabled preemption.
     386                 :            :  */
     387                 :     177974 : static int rcu_is_cpu_rrupt_from_idle(void)
     388                 :            : {
     389                 :            :         /* Called only from within the scheduling-clock interrupt */
     390                 :     177974 :         lockdep_assert_in_irq();
     391                 :            : 
     392                 :            :         /* Check for counter underflows */
     393                 :     177974 :         RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nesting) < 0,
     394                 :            :                          "RCU dynticks_nesting counter underflow!");
     395                 :     177974 :         RCU_LOCKDEP_WARN(__this_cpu_read(rcu_data.dynticks_nmi_nesting) <= 0,
     396                 :            :                          "RCU dynticks_nmi_nesting counter underflow/zero!");
     397                 :            : 
     398                 :            :         /* Are we at first interrupt nesting level? */
     399   [ +  +  -  -  :     177974 :         if (__this_cpu_read(rcu_data.dynticks_nmi_nesting) != 1)
                   +  + ]
     400                 :            :                 return false;
     401                 :            : 
     402                 :            :         /* Does CPU appear to be idle from an RCU standpoint? */
     403   [ +  -  -  -  :       4186 :         return __this_cpu_read(rcu_data.dynticks_nesting) == 0;
                   -  - ]
     404                 :            : }
     405                 :            : 
     406                 :            : #define DEFAULT_RCU_BLIMIT 10     /* Maximum callbacks per rcu_do_batch ... */
     407                 :            : #define DEFAULT_MAX_RCU_BLIMIT 10000 /* ... even during callback flood. */
     408                 :            : static long blimit = DEFAULT_RCU_BLIMIT;
     409                 :            : #define DEFAULT_RCU_QHIMARK 10000 /* If this many pending, ignore blimit. */
     410                 :            : static long qhimark = DEFAULT_RCU_QHIMARK;
     411                 :            : #define DEFAULT_RCU_QLOMARK 100   /* Once only this many pending, use blimit. */
     412                 :            : static long qlowmark = DEFAULT_RCU_QLOMARK;
     413                 :            : 
     414                 :            : module_param(blimit, long, 0444);
     415                 :            : module_param(qhimark, long, 0444);
     416                 :            : module_param(qlowmark, long, 0444);
     417                 :            : 
     418                 :            : static ulong jiffies_till_first_fqs = ULONG_MAX;
     419                 :            : static ulong jiffies_till_next_fqs = ULONG_MAX;
     420                 :            : static bool rcu_kick_kthreads;
     421                 :            : static int rcu_divisor = 7;
     422                 :            : module_param(rcu_divisor, int, 0644);
     423                 :            : 
     424                 :            : /* Force an exit from rcu_do_batch() after 3 milliseconds. */
     425                 :            : static long rcu_resched_ns = 3 * NSEC_PER_MSEC;
     426                 :            : module_param(rcu_resched_ns, long, 0644);
     427                 :            : 
     428                 :            : /*
     429                 :            :  * How long the grace period must be before we start recruiting
     430                 :            :  * quiescent-state help from rcu_note_context_switch().
     431                 :            :  */
     432                 :            : static ulong jiffies_till_sched_qs = ULONG_MAX;
     433                 :            : module_param(jiffies_till_sched_qs, ulong, 0444);
     434                 :            : static ulong jiffies_to_sched_qs; /* See adjust_jiffies_till_sched_qs(). */
     435                 :            : module_param(jiffies_to_sched_qs, ulong, 0444); /* Display only! */
     436                 :            : 
     437                 :            : /*
     438                 :            :  * Make sure that we give the grace-period kthread time to detect any
     439                 :            :  * idle CPUs before taking active measures to force quiescent states.
     440                 :            :  * However, don't go below 100 milliseconds, adjusted upwards for really
     441                 :            :  * large systems.
     442                 :            :  */
     443                 :         30 : static void adjust_jiffies_till_sched_qs(void)
     444                 :            : {
     445                 :         30 :         unsigned long j;
     446                 :            : 
     447                 :            :         /* If jiffies_till_sched_qs was specified, respect the request. */
     448         [ -  + ]:         30 :         if (jiffies_till_sched_qs != ULONG_MAX) {
     449                 :          0 :                 WRITE_ONCE(jiffies_to_sched_qs, jiffies_till_sched_qs);
     450                 :          0 :                 return;
     451                 :            :         }
     452                 :            :         /* Otherwise, set to third fqs scan, but bound below on large system. */
     453                 :         30 :         j = READ_ONCE(jiffies_till_first_fqs) +
     454                 :         30 :                       2 * READ_ONCE(jiffies_till_next_fqs);
     455                 :         30 :         if (j < HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV)
     456                 :            :                 j = HZ / 10 + nr_cpu_ids / RCU_JIFFIES_FQS_DIV;
     457                 :         30 :         pr_info("RCU calculated value of scheduler-enlistment delay is %ld jiffies.\n", j);
     458                 :         30 :         WRITE_ONCE(jiffies_to_sched_qs, j);
     459                 :            : }
     460                 :            : 
     461                 :          0 : static int param_set_first_fqs_jiffies(const char *val, const struct kernel_param *kp)
     462                 :            : {
     463                 :          0 :         ulong j;
     464                 :          0 :         int ret = kstrtoul(val, 0, &j);
     465                 :            : 
     466         [ #  # ]:          0 :         if (!ret) {
     467                 :          0 :                 WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : j);
     468                 :          0 :                 adjust_jiffies_till_sched_qs();
     469                 :            :         }
     470                 :          0 :         return ret;
     471                 :            : }
     472                 :            : 
     473                 :          0 : static int param_set_next_fqs_jiffies(const char *val, const struct kernel_param *kp)
     474                 :            : {
     475                 :          0 :         ulong j;
     476                 :          0 :         int ret = kstrtoul(val, 0, &j);
     477                 :            : 
     478         [ #  # ]:          0 :         if (!ret) {
     479                 :          0 :                 WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : (j ?: 1));
     480                 :          0 :                 adjust_jiffies_till_sched_qs();
     481                 :            :         }
     482                 :          0 :         return ret;
     483                 :            : }
     484                 :            : 
     485                 :            : static struct kernel_param_ops first_fqs_jiffies_ops = {
     486                 :            :         .set = param_set_first_fqs_jiffies,
     487                 :            :         .get = param_get_ulong,
     488                 :            : };
     489                 :            : 
     490                 :            : static struct kernel_param_ops next_fqs_jiffies_ops = {
     491                 :            :         .set = param_set_next_fqs_jiffies,
     492                 :            :         .get = param_get_ulong,
     493                 :            : };
     494                 :            : 
     495                 :            : module_param_cb(jiffies_till_first_fqs, &first_fqs_jiffies_ops, &jiffies_till_first_fqs, 0644);
     496                 :            : module_param_cb(jiffies_till_next_fqs, &next_fqs_jiffies_ops, &jiffies_till_next_fqs, 0644);
     497                 :            : module_param(rcu_kick_kthreads, bool, 0644);
     498                 :            : 
     499                 :            : static void force_qs_rnp(int (*f)(struct rcu_data *rdp));
     500                 :            : static int rcu_pending(int user);
     501                 :            : 
     502                 :            : /*
     503                 :            :  * Return the number of RCU GPs completed thus far for debug & stats.
     504                 :            :  */
     505                 :          0 : unsigned long rcu_get_gp_seq(void)
     506                 :            : {
     507                 :          0 :         return READ_ONCE(rcu_state.gp_seq);
     508                 :            : }
     509                 :            : EXPORT_SYMBOL_GPL(rcu_get_gp_seq);
     510                 :            : 
     511                 :            : /*
     512                 :            :  * Return the number of RCU expedited batches completed thus far for
     513                 :            :  * debug & stats.  Odd numbers mean that a batch is in progress, even
     514                 :            :  * numbers mean idle.  The value returned will thus be roughly double
     515                 :            :  * the cumulative batches since boot.
     516                 :            :  */
     517                 :          0 : unsigned long rcu_exp_batches_completed(void)
     518                 :            : {
     519                 :          0 :         return rcu_state.expedited_sequence;
     520                 :            : }
     521                 :            : EXPORT_SYMBOL_GPL(rcu_exp_batches_completed);
     522                 :            : 
     523                 :            : /*
     524                 :            :  * Return the root node of the rcu_state structure.
     525                 :            :  */
     526                 :     855855 : static struct rcu_node *rcu_get_root(void)
     527                 :            : {
     528   [ -  -  -  - ]:     247986 :         return &rcu_state.node[0];
     529                 :            : }
     530                 :            : 
     531                 :            : /*
     532                 :            :  * Send along grace-period-related data for rcutorture diagnostics.
     533                 :            :  */
     534                 :          0 : void rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags,
     535                 :            :                             unsigned long *gp_seq)
     536                 :            : {
     537         [ #  # ]:          0 :         switch (test_type) {
     538                 :            :         case RCU_FLAVOR:
     539                 :          0 :                 *flags = READ_ONCE(rcu_state.gp_flags);
     540                 :          0 :                 *gp_seq = rcu_seq_current(&rcu_state.gp_seq);
     541                 :          0 :                 break;
     542                 :            :         default:
     543                 :            :                 break;
     544                 :            :         }
     545                 :          0 : }
     546                 :            : EXPORT_SYMBOL_GPL(rcutorture_get_gp_data);
     547                 :            : 
     548                 :            : /*
     549                 :            :  * Enter an RCU extended quiescent state, which can be either the
     550                 :            :  * idle loop or adaptive-tickless usermode execution.
     551                 :            :  *
     552                 :            :  * We crowbar the ->dynticks_nmi_nesting field to zero to allow for
     553                 :            :  * the possibility of usermode upcalls having messed up our count
     554                 :            :  * of interrupt nesting level during the prior busy period.
     555                 :            :  */
     556                 :      11058 : static void rcu_eqs_enter(bool user)
     557                 :            : {
     558                 :      11058 :         struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
     559                 :            : 
     560         [ -  + ]:      11058 :         WARN_ON_ONCE(rdp->dynticks_nmi_nesting != DYNTICK_IRQ_NONIDLE);
     561         [ -  + ]:      11058 :         WRITE_ONCE(rdp->dynticks_nmi_nesting, 0);
     562                 :      11058 :         WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
     563                 :            :                      rdp->dynticks_nesting == 0);
     564         [ -  + ]:      11058 :         if (rdp->dynticks_nesting != 1) {
     565                 :          0 :                 rdp->dynticks_nesting--;
     566                 :          0 :                 return;
     567                 :            :         }
     568                 :            : 
     569                 :      11058 :         lockdep_assert_irqs_disabled();
     570                 :      11058 :         trace_rcu_dyntick(TPS("Start"), rdp->dynticks_nesting, 0, atomic_read(&rdp->dynticks));
     571                 :      11058 :         WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
     572                 :      11058 :         rdp = this_cpu_ptr(&rcu_data);
     573                 :      11058 :         do_nocb_deferred_wakeup(rdp);
     574                 :      11058 :         rcu_prepare_for_idle();
     575                 :      11058 :         rcu_preempt_deferred_qs(current);
     576                 :      11058 :         WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */
     577                 :      11058 :         rcu_dynticks_eqs_enter();
     578                 :            :         rcu_dynticks_task_enter();
     579                 :            : }
     580                 :            : 
     581                 :            : /**
     582                 :            :  * rcu_idle_enter - inform RCU that current CPU is entering idle
     583                 :            :  *
     584                 :            :  * Enter idle mode, in other words, -leave- the mode in which RCU
     585                 :            :  * read-side critical sections can occur.  (Though RCU read-side
     586                 :            :  * critical sections can occur in irq handlers in idle, a possibility
     587                 :            :  * handled by irq_enter() and irq_exit().)
     588                 :            :  *
     589                 :            :  * If you add or remove a call to rcu_idle_enter(), be sure to test with
     590                 :            :  * CONFIG_RCU_EQS_DEBUG=y.
     591                 :            :  */
     592                 :      11058 : void rcu_idle_enter(void)
     593                 :            : {
     594                 :      11058 :         lockdep_assert_irqs_disabled();
     595                 :      11058 :         rcu_eqs_enter(false);
     596                 :      11058 : }
     597                 :            : 
     598                 :            : #ifdef CONFIG_NO_HZ_FULL
     599                 :            : /**
     600                 :            :  * rcu_user_enter - inform RCU that we are resuming userspace.
     601                 :            :  *
     602                 :            :  * Enter RCU idle mode right before resuming userspace.  No use of RCU
     603                 :            :  * is permitted between this call and rcu_user_exit(). This way the
     604                 :            :  * CPU doesn't need to maintain the tick for RCU maintenance purposes
     605                 :            :  * when the CPU runs in userspace.
     606                 :            :  *
     607                 :            :  * If you add or remove a call to rcu_user_enter(), be sure to test with
     608                 :            :  * CONFIG_RCU_EQS_DEBUG=y.
     609                 :            :  */
     610                 :            : void rcu_user_enter(void)
     611                 :            : {
     612                 :            :         lockdep_assert_irqs_disabled();
     613                 :            :         rcu_eqs_enter(true);
     614                 :            : }
     615                 :            : #endif /* CONFIG_NO_HZ_FULL */
     616                 :            : 
     617                 :            : /*
     618                 :            :  * If we are returning from the outermost NMI handler that interrupted an
     619                 :            :  * RCU-idle period, update rdp->dynticks and rdp->dynticks_nmi_nesting
     620                 :            :  * to let the RCU grace-period handling know that the CPU is back to
     621                 :            :  * being RCU-idle.
     622                 :            :  *
     623                 :            :  * If you add or remove a call to rcu_nmi_exit_common(), be sure to test
     624                 :            :  * with CONFIG_RCU_EQS_DEBUG=y.
     625                 :            :  */
     626                 :    2872637 : static __always_inline void rcu_nmi_exit_common(bool irq)
     627                 :            : {
     628                 :    5745274 :         struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
     629                 :            : 
     630                 :            :         /*
     631                 :            :          * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks.
     632                 :            :          * (We are exiting an NMI handler, so RCU better be paying attention
     633                 :            :          * to us!)
     634                 :            :          */
     635   [ -  +  -  + ]:    2872637 :         WARN_ON_ONCE(rdp->dynticks_nmi_nesting <= 0);
     636   [ -  +  -  + ]:    5745274 :         WARN_ON_ONCE(rcu_dynticks_curr_cpu_in_eqs());
     637                 :            : 
     638                 :            :         /*
     639                 :            :          * If the nesting level is not 1, the CPU wasn't RCU-idle, so
     640                 :            :          * leave it in non-RCU-idle state.
     641                 :            :          */
     642   [ +  +  +  - ]:    2872637 :         if (rdp->dynticks_nmi_nesting != 1) {
     643                 :    2861572 :                 trace_rcu_dyntick(TPS("--="), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting - 2,
     644                 :    2861572 :                                   atomic_read(&rdp->dynticks));
     645                 :    2861572 :                 WRITE_ONCE(rdp->dynticks_nmi_nesting, /* No store tearing. */
     646                 :            :                            rdp->dynticks_nmi_nesting - 2);
     647                 :    2861572 :                 return;
     648                 :            :         }
     649                 :            : 
     650                 :            :         /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
     651                 :      11065 :         trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, atomic_read(&rdp->dynticks));
     652                 :      11065 :         WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
     653                 :            : 
     654                 :      11065 :         if (irq)
     655                 :      11065 :                 rcu_prepare_for_idle();
     656                 :            : 
     657                 :      11065 :         rcu_dynticks_eqs_enter();
     658                 :            : 
     659                 :            :         if (irq)
     660                 :            :                 rcu_dynticks_task_enter();
     661                 :            : }
     662                 :            : 
     663                 :            : /**
     664                 :            :  * rcu_nmi_exit - inform RCU of exit from NMI context
     665                 :            :  *
     666                 :            :  * If you add or remove a call to rcu_nmi_exit(), be sure to test
     667                 :            :  * with CONFIG_RCU_EQS_DEBUG=y.
     668                 :            :  */
     669                 :         30 : void rcu_nmi_exit(void)
     670                 :            : {
     671                 :         30 :         rcu_nmi_exit_common(false);
     672                 :         30 : }
     673                 :            : 
     674                 :            : /**
     675                 :            :  * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle
     676                 :            :  *
     677                 :            :  * Exit from an interrupt handler, which might possibly result in entering
     678                 :            :  * idle mode, in other words, leaving the mode in which read-side critical
     679                 :            :  * sections can occur.  The caller must have disabled interrupts.
     680                 :            :  *
     681                 :            :  * This code assumes that the idle loop never does anything that might
     682                 :            :  * result in unbalanced calls to irq_enter() and irq_exit().  If your
     683                 :            :  * architecture's idle loop violates this assumption, RCU will give you what
     684                 :            :  * you deserve, good and hard.  But very infrequently and irreproducibly.
     685                 :            :  *
     686                 :            :  * Use things like work queues to work around this limitation.
     687                 :            :  *
     688                 :            :  * You have been warned.
     689                 :            :  *
     690                 :            :  * If you add or remove a call to rcu_irq_exit(), be sure to test with
     691                 :            :  * CONFIG_RCU_EQS_DEBUG=y.
     692                 :            :  */
     693                 :    2872607 : void rcu_irq_exit(void)
     694                 :            : {
     695                 :    2872607 :         lockdep_assert_irqs_disabled();
     696                 :    2872607 :         rcu_nmi_exit_common(true);
     697                 :    2872607 : }
     698                 :            : 
     699                 :            : /*
     700                 :            :  * Wrapper for rcu_irq_exit() where interrupts are enabled.
     701                 :            :  *
     702                 :            :  * If you add or remove a call to rcu_irq_exit_irqson(), be sure to test
     703                 :            :  * with CONFIG_RCU_EQS_DEBUG=y.
     704                 :            :  */
     705                 :          0 : void rcu_irq_exit_irqson(void)
     706                 :            : {
     707                 :          0 :         unsigned long flags;
     708                 :            : 
     709                 :          0 :         local_irq_save(flags);
     710                 :          0 :         rcu_irq_exit();
     711                 :          0 :         local_irq_restore(flags);
     712                 :          0 : }
     713                 :            : 
     714                 :            : /*
     715                 :            :  * Exit an RCU extended quiescent state, which can be either the
     716                 :            :  * idle loop or adaptive-tickless usermode execution.
     717                 :            :  *
     718                 :            :  * We crowbar the ->dynticks_nmi_nesting field to DYNTICK_IRQ_NONIDLE to
     719                 :            :  * allow for the possibility of usermode upcalls messing up our count of
     720                 :            :  * interrupt nesting level during the busy period that is just now starting.
     721                 :            :  */
     722                 :      11058 : static void rcu_eqs_exit(bool user)
     723                 :            : {
     724                 :      11058 :         struct rcu_data *rdp;
     725                 :      11058 :         long oldval;
     726                 :            : 
     727                 :      11058 :         lockdep_assert_irqs_disabled();
     728                 :      11058 :         rdp = this_cpu_ptr(&rcu_data);
     729                 :      11058 :         oldval = rdp->dynticks_nesting;
     730                 :      11058 :         WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && oldval < 0);
     731         [ -  + ]:      11058 :         if (oldval) {
     732                 :          0 :                 rdp->dynticks_nesting++;
     733                 :          0 :                 return;
     734                 :            :         }
     735                 :      11058 :         rcu_dynticks_task_exit();
     736                 :      11058 :         rcu_dynticks_eqs_exit();
     737                 :      11058 :         rcu_cleanup_after_idle();
     738                 :      11058 :         trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks));
     739                 :      11058 :         WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
     740         [ -  + ]:      11058 :         WRITE_ONCE(rdp->dynticks_nesting, 1);
     741         [ -  + ]:      11058 :         WARN_ON_ONCE(rdp->dynticks_nmi_nesting);
     742                 :      11058 :         WRITE_ONCE(rdp->dynticks_nmi_nesting, DYNTICK_IRQ_NONIDLE);
     743                 :            : }
     744                 :            : 
     745                 :            : /**
     746                 :            :  * rcu_idle_exit - inform RCU that current CPU is leaving idle
     747                 :            :  *
     748                 :            :  * Exit idle mode, in other words, -enter- the mode in which RCU
     749                 :            :  * read-side critical sections can occur.
     750                 :            :  *
     751                 :            :  * If you add or remove a call to rcu_idle_exit(), be sure to test with
     752                 :            :  * CONFIG_RCU_EQS_DEBUG=y.
     753                 :            :  */
     754                 :      11058 : void rcu_idle_exit(void)
     755                 :            : {
     756                 :      11058 :         unsigned long flags;
     757                 :            : 
     758                 :      11058 :         local_irq_save(flags);
     759                 :      11058 :         rcu_eqs_exit(false);
     760                 :      11058 :         local_irq_restore(flags);
     761                 :      11058 : }
     762                 :            : 
     763                 :            : #ifdef CONFIG_NO_HZ_FULL
     764                 :            : /**
     765                 :            :  * rcu_user_exit - inform RCU that we are exiting userspace.
     766                 :            :  *
     767                 :            :  * Exit RCU idle mode while entering the kernel because it can
     768                 :            :  * run a RCU read side critical section anytime.
     769                 :            :  *
     770                 :            :  * If you add or remove a call to rcu_user_exit(), be sure to test with
     771                 :            :  * CONFIG_RCU_EQS_DEBUG=y.
     772                 :            :  */
     773                 :            : void rcu_user_exit(void)
     774                 :            : {
     775                 :            :         rcu_eqs_exit(1);
     776                 :            : }
     777                 :            : #endif /* CONFIG_NO_HZ_FULL */
     778                 :            : 
     779                 :            : /**
     780                 :            :  * rcu_nmi_enter_common - inform RCU of entry to NMI context
     781                 :            :  * @irq: Is this call from rcu_irq_enter?
     782                 :            :  *
     783                 :            :  * If the CPU was idle from RCU's viewpoint, update rdp->dynticks and
     784                 :            :  * rdp->dynticks_nmi_nesting to let the RCU grace-period handling know
     785                 :            :  * that the CPU is active.  This implementation permits nested NMIs, as
     786                 :            :  * long as the nesting level does not overflow an int.  (You will probably
     787                 :            :  * run out of stack space first.)
     788                 :            :  *
     789                 :            :  * If you add or remove a call to rcu_nmi_enter_common(), be sure to test
     790                 :            :  * with CONFIG_RCU_EQS_DEBUG=y.
     791                 :            :  */
     792                 :    2872637 : static __always_inline void rcu_nmi_enter_common(bool irq)
     793                 :            : {
     794                 :    2872637 :         long incby = 2;
     795                 :    5745274 :         struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
     796                 :            : 
     797                 :            :         /* Complain about underflow. */
     798   [ -  +  -  + ]:    2872637 :         WARN_ON_ONCE(rdp->dynticks_nmi_nesting < 0);
     799                 :            : 
     800                 :            :         /*
     801                 :            :          * If idle from RCU viewpoint, atomically increment ->dynticks
     802                 :            :          * to mark non-idle and increment ->dynticks_nmi_nesting by one.
     803                 :            :          * Otherwise, increment ->dynticks_nmi_nesting by two.  This means
     804                 :            :          * if ->dynticks_nmi_nesting is equal to one, we are guaranteed
     805                 :            :          * to be in the outermost NMI handler that interrupted an RCU-idle
     806                 :            :          * period (observation due to Andy Lutomirski).
     807                 :            :          */
     808   [ +  +  -  + ]:    5745244 :         if (rcu_dynticks_curr_cpu_in_eqs()) {
     809                 :            : 
     810                 :      11065 :                 if (irq)
     811                 :      11065 :                         rcu_dynticks_task_exit();
     812                 :            : 
     813                 :      11065 :                 rcu_dynticks_eqs_exit();
     814                 :            : 
     815                 :      11065 :                 if (irq)
     816                 :            :                         rcu_cleanup_after_idle();
     817                 :            : 
     818                 :            :                 incby = 1;
     819                 :            :         } else if (tick_nohz_full_cpu(rdp->cpu) &&
     820                 :            :                    rdp->dynticks_nmi_nesting == DYNTICK_IRQ_NONIDLE &&
     821                 :            :                    READ_ONCE(rdp->rcu_urgent_qs) && !rdp->rcu_forced_tick) {
     822                 :            :                 raw_spin_lock_rcu_node(rdp->mynode);
     823                 :            :                 // Recheck under lock.
     824                 :            :                 if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) {
     825                 :            :                         rdp->rcu_forced_tick = true;
     826                 :            :                         tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
     827                 :            :                 }
     828                 :            :                 raw_spin_unlock_rcu_node(rdp->mynode);
     829                 :            :         }
     830                 :    2872637 :         trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),
     831                 :            :                           rdp->dynticks_nmi_nesting,
     832   [ +  +  -  + ]:    2872637 :                           rdp->dynticks_nmi_nesting + incby, atomic_read(&rdp->dynticks));
     833                 :    2872637 :         WRITE_ONCE(rdp->dynticks_nmi_nesting, /* Prevent store tearing. */
     834                 :            :                    rdp->dynticks_nmi_nesting + incby);
     835                 :    2872637 :         barrier();
     836                 :            : }
     837                 :            : 
     838                 :            : /**
     839                 :            :  * rcu_nmi_enter - inform RCU of entry to NMI context
     840                 :            :  */
     841                 :         30 : void rcu_nmi_enter(void)
     842                 :            : {
     843                 :         30 :         rcu_nmi_enter_common(false);
     844                 :         30 : }
     845                 :            : NOKPROBE_SYMBOL(rcu_nmi_enter);
     846                 :            : 
     847                 :            : /**
     848                 :            :  * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle
     849                 :            :  *
     850                 :            :  * Enter an interrupt handler, which might possibly result in exiting
     851                 :            :  * idle mode, in other words, entering the mode in which read-side critical
     852                 :            :  * sections can occur.  The caller must have disabled interrupts.
     853                 :            :  *
     854                 :            :  * Note that the Linux kernel is fully capable of entering an interrupt
     855                 :            :  * handler that it never exits, for example when doing upcalls to user mode!
     856                 :            :  * This code assumes that the idle loop never does upcalls to user mode.
     857                 :            :  * If your architecture's idle loop does do upcalls to user mode (or does
     858                 :            :  * anything else that results in unbalanced calls to the irq_enter() and
     859                 :            :  * irq_exit() functions), RCU will give you what you deserve, good and hard.
     860                 :            :  * But very infrequently and irreproducibly.
     861                 :            :  *
     862                 :            :  * Use things like work queues to work around this limitation.
     863                 :            :  *
     864                 :            :  * You have been warned.
     865                 :            :  *
     866                 :            :  * If you add or remove a call to rcu_irq_enter(), be sure to test with
     867                 :            :  * CONFIG_RCU_EQS_DEBUG=y.
     868                 :            :  */
     869                 :    2872607 : void rcu_irq_enter(void)
     870                 :            : {
     871                 :    2872607 :         lockdep_assert_irqs_disabled();
     872                 :    2872607 :         rcu_nmi_enter_common(true);
     873                 :    2872607 : }
     874                 :            : 
     875                 :            : /*
     876                 :            :  * Wrapper for rcu_irq_enter() where interrupts are enabled.
     877                 :            :  *
     878                 :            :  * If you add or remove a call to rcu_irq_enter_irqson(), be sure to test
     879                 :            :  * with CONFIG_RCU_EQS_DEBUG=y.
     880                 :            :  */
     881                 :          0 : void rcu_irq_enter_irqson(void)
     882                 :            : {
     883                 :          0 :         unsigned long flags;
     884                 :            : 
     885                 :          0 :         local_irq_save(flags);
     886                 :          0 :         rcu_irq_enter();
     887                 :          0 :         local_irq_restore(flags);
     888                 :          0 : }
     889                 :            : 
     890                 :            : /*
     891                 :            :  * If any sort of urgency was applied to the current CPU (for example,
     892                 :            :  * the scheduler-clock interrupt was enabled on a nohz_full CPU) in order
     893                 :            :  * to get to a quiescent state, disable it.
     894                 :            :  */
     895                 :          0 : static void rcu_disable_urgency_upon_qs(struct rcu_data *rdp)
     896                 :            : {
     897                 :          0 :         raw_lockdep_assert_held_rcu_node(rdp->mynode);
     898                 :          0 :         WRITE_ONCE(rdp->rcu_urgent_qs, false);
     899                 :          0 :         WRITE_ONCE(rdp->rcu_need_heavy_qs, false);
     900                 :          0 :         if (tick_nohz_full_cpu(rdp->cpu) && rdp->rcu_forced_tick) {
     901                 :            :                 tick_dep_clear_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
     902                 :            :                 rdp->rcu_forced_tick = false;
     903                 :            :         }
     904                 :          0 : }
     905                 :            : 
     906                 :            : /**
     907                 :            :  * rcu_is_watching - see if RCU thinks that the current CPU is not idle
     908                 :            :  *
     909                 :            :  * Return true if RCU is watching the running CPU, which means that this
     910                 :            :  * CPU can safely enter RCU read-side critical sections.  In other words,
     911                 :            :  * if the current CPU is not in its idle loop or is in an interrupt or
     912                 :            :  * NMI handler, return true.
     913                 :            :  */
     914                 :    2264077 : bool notrace rcu_is_watching(void)
     915                 :            : {
     916                 :    2264077 :         bool ret;
     917                 :            : 
     918                 :      44239 :         preempt_disable_notrace();
     919                 :    2264077 :         ret = !rcu_dynticks_curr_cpu_in_eqs();
     920                 :    2264077 :         preempt_enable_notrace();
     921                 :    2264077 :         return ret;
     922                 :            : }
     923                 :            : EXPORT_SYMBOL_GPL(rcu_is_watching);
     924                 :            : 
     925                 :            : /*
     926                 :            :  * If a holdout task is actually running, request an urgent quiescent
     927                 :            :  * state from its CPU.  This is unsynchronized, so migrations can cause
     928                 :            :  * the request to go to the wrong CPU.  Which is OK, all that will happen
     929                 :            :  * is that the CPU's next context switch will be a bit slower and next
     930                 :            :  * time around this task will generate another request.
     931                 :            :  */
     932                 :          0 : void rcu_request_urgent_qs_task(struct task_struct *t)
     933                 :            : {
     934                 :          0 :         int cpu;
     935                 :            : 
     936                 :          0 :         barrier();
     937                 :          0 :         cpu = task_cpu(t);
     938         [ #  # ]:          0 :         if (!task_curr(t))
     939                 :            :                 return; /* This task is not running on that CPU. */
     940                 :          0 :         smp_store_release(per_cpu_ptr(&rcu_data.rcu_urgent_qs, cpu), true);
     941                 :            : }
     942                 :            : 
     943                 :            : #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU)
     944                 :            : 
     945                 :            : /*
     946                 :            :  * Is the current CPU online as far as RCU is concerned?
     947                 :            :  *
     948                 :            :  * Disable preemption to avoid false positives that could otherwise
     949                 :            :  * happen due to the current CPU number being sampled, this task being
     950                 :            :  * preempted, its old CPU being taken offline, resuming on some other CPU,
     951                 :            :  * then determining that its old CPU is now offline.
     952                 :            :  *
     953                 :            :  * Disable checking if in an NMI handler because we cannot safely
     954                 :            :  * report errors from NMI handlers anyway.  In addition, it is OK to use
     955                 :            :  * RCU on an offline processor during initial boot, hence the check for
     956                 :            :  * rcu_scheduler_fully_active.
     957                 :            :  */
     958                 :            : bool rcu_lockdep_current_cpu_online(void)
     959                 :            : {
     960                 :            :         struct rcu_data *rdp;
     961                 :            :         struct rcu_node *rnp;
     962                 :            :         bool ret = false;
     963                 :            : 
     964                 :            :         if (in_nmi() || !rcu_scheduler_fully_active)
     965                 :            :                 return true;
     966                 :            :         preempt_disable();
     967                 :            :         rdp = this_cpu_ptr(&rcu_data);
     968                 :            :         rnp = rdp->mynode;
     969                 :            :         if (rdp->grpmask & rcu_rnp_online_cpus(rnp))
     970                 :            :                 ret = true;
     971                 :            :         preempt_enable();
     972                 :            :         return ret;
     973                 :            : }
     974                 :            : EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online);
     975                 :            : 
     976                 :            : #endif /* #if defined(CONFIG_PROVE_RCU) && defined(CONFIG_HOTPLUG_CPU) */
     977                 :            : 
     978                 :            : /*
     979                 :            :  * We are reporting a quiescent state on behalf of some other CPU, so
     980                 :            :  * it is our responsibility to check for and handle potential overflow
     981                 :            :  * of the rcu_node ->gp_seq counter with respect to the rcu_data counters.
     982                 :            :  * After all, the CPU might be in deep idle state, and thus executing no
     983                 :            :  * code whatsoever.
     984                 :            :  */
     985                 :     135102 : static void rcu_gpnum_ovf(struct rcu_node *rnp, struct rcu_data *rdp)
     986                 :            : {
     987                 :     135102 :         raw_lockdep_assert_held_rcu_node(rnp);
     988                 :     135102 :         if (ULONG_CMP_LT(rcu_seq_current(&rdp->gp_seq) + ULONG_MAX / 4,
     989                 :            :                          rnp->gp_seq))
     990                 :          0 :                 WRITE_ONCE(rdp->gpwrap, true);
     991   [ -  +  -  +  :     135102 :         if (ULONG_CMP_LT(rdp->rcu_iw_gp_seq + ULONG_MAX / 4, rnp->gp_seq))
             -  -  -  - ]
     992                 :          0 :                 rdp->rcu_iw_gp_seq = rnp->gp_seq + ULONG_MAX / 4;
     993                 :            : }
     994                 :            : 
     995                 :            : /*
     996                 :            :  * Snapshot the specified CPU's dynticks counter so that we can later
     997                 :            :  * credit them with an implicit quiescent state.  Return 1 if this CPU
     998                 :            :  * is in dynticks idle mode, which is an extended quiescent state.
     999                 :            :  */
    1000                 :          0 : static int dyntick_save_progress_counter(struct rcu_data *rdp)
    1001                 :            : {
    1002                 :          0 :         rdp->dynticks_snap = rcu_dynticks_snap(rdp);
    1003         [ #  # ]:          0 :         if (rcu_dynticks_in_eqs(rdp->dynticks_snap)) {
    1004                 :          0 :                 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
    1005         [ #  # ]:          0 :                 rcu_gpnum_ovf(rdp->mynode, rdp);
    1006                 :          0 :                 return 1;
    1007                 :            :         }
    1008                 :            :         return 0;
    1009                 :            : }
    1010                 :            : 
    1011                 :            : /*
    1012                 :            :  * Return true if the specified CPU has passed through a quiescent
    1013                 :            :  * state by virtue of being in or having passed through an dynticks
    1014                 :            :  * idle state since the last call to dyntick_save_progress_counter()
    1015                 :            :  * for this same CPU, or by virtue of having been offline.
    1016                 :            :  */
    1017                 :          0 : static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
    1018                 :            : {
    1019                 :          0 :         unsigned long jtsq;
    1020                 :          0 :         bool *rnhqp;
    1021                 :          0 :         bool *ruqp;
    1022                 :          0 :         struct rcu_node *rnp = rdp->mynode;
    1023                 :            : 
    1024                 :            :         /*
    1025                 :            :          * If the CPU passed through or entered a dynticks idle phase with
    1026                 :            :          * no active irq/NMI handlers, then we can safely pretend that the CPU
    1027                 :            :          * already acknowledged the request to pass through a quiescent
    1028                 :            :          * state.  Either way, that CPU cannot possibly be in an RCU
    1029                 :            :          * read-side critical section that started before the beginning
    1030                 :            :          * of the current RCU grace period.
    1031                 :            :          */
    1032         [ #  # ]:          0 :         if (rcu_dynticks_in_eqs_since(rdp, rdp->dynticks_snap)) {
    1033                 :          0 :                 trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));
    1034         [ #  # ]:          0 :                 rcu_gpnum_ovf(rnp, rdp);
    1035                 :          0 :                 return 1;
    1036                 :            :         }
    1037                 :            : 
    1038                 :            :         /* If waiting too long on an offline CPU, complain. */
    1039         [ #  # ]:          0 :         if (!(rdp->grpmask & rcu_rnp_online_cpus(rnp)) &&
    1040         [ #  # ]:          0 :             time_after(jiffies, rcu_state.gp_start + HZ)) {
    1041                 :          0 :                 bool onl;
    1042                 :          0 :                 struct rcu_node *rnp1;
    1043                 :            : 
    1044                 :          0 :                 WARN_ON(1);  /* Offline CPUs are supposed to report QS! */
    1045                 :          0 :                 pr_info("%s: grp: %d-%d level: %d ->gp_seq %ld ->completedqs %ld\n",
    1046                 :            :                         __func__, rnp->grplo, rnp->grphi, rnp->level,
    1047                 :            :                         (long)rnp->gp_seq, (long)rnp->completedqs);
    1048         [ #  # ]:          0 :                 for (rnp1 = rnp; rnp1; rnp1 = rnp1->parent)
    1049                 :          0 :                         pr_info("%s: %d:%d ->qsmask %#lx ->qsmaskinit %#lx ->qsmaskinitnext %#lx ->rcu_gp_init_mask %#lx\n",
    1050                 :            :                                 __func__, rnp1->grplo, rnp1->grphi, rnp1->qsmask, rnp1->qsmaskinit, rnp1->qsmaskinitnext, rnp1->rcu_gp_init_mask);
    1051                 :          0 :                 onl = !!(rdp->grpmask & rcu_rnp_online_cpus(rnp));
    1052                 :          0 :                 pr_info("%s %d: %c online: %ld(%d) offline: %ld(%d)\n",
    1053                 :            :                         __func__, rdp->cpu, ".o"[onl],
    1054                 :            :                         (long)rdp->rcu_onl_gp_seq, rdp->rcu_onl_gp_flags,
    1055                 :            :                         (long)rdp->rcu_ofl_gp_seq, rdp->rcu_ofl_gp_flags);
    1056                 :          0 :                 return 1; /* Break things loose after complaining. */
    1057                 :            :         }
    1058                 :            : 
    1059                 :            :         /*
    1060                 :            :          * A CPU running for an extended time within the kernel can
    1061                 :            :          * delay RCU grace periods: (1) At age jiffies_to_sched_qs,
    1062                 :            :          * set .rcu_urgent_qs, (2) At age 2*jiffies_to_sched_qs, set
    1063                 :            :          * both .rcu_need_heavy_qs and .rcu_urgent_qs.  Note that the
    1064                 :            :          * unsynchronized assignments to the per-CPU rcu_need_heavy_qs
    1065                 :            :          * variable are safe because the assignments are repeated if this
    1066                 :            :          * CPU failed to pass through a quiescent state.  This code
    1067                 :            :          * also checks .jiffies_resched in case jiffies_to_sched_qs
    1068                 :            :          * is set way high.
    1069                 :            :          */
    1070         [ #  # ]:          0 :         jtsq = READ_ONCE(jiffies_to_sched_qs);
    1071                 :          0 :         ruqp = per_cpu_ptr(&rcu_data.rcu_urgent_qs, rdp->cpu);
    1072                 :          0 :         rnhqp = &per_cpu(rcu_data.rcu_need_heavy_qs, rdp->cpu);
    1073         [ #  # ]:          0 :         if (!READ_ONCE(*rnhqp) &&
    1074         [ #  # ]:          0 :             (time_after(jiffies, rcu_state.gp_start + jtsq * 2) ||
    1075         [ #  # ]:          0 :              time_after(jiffies, rcu_state.jiffies_resched))) {
    1076                 :          0 :                 WRITE_ONCE(*rnhqp, true);
    1077                 :            :                 /* Store rcu_need_heavy_qs before rcu_urgent_qs. */
    1078                 :          0 :                 smp_store_release(ruqp, true);
    1079         [ #  # ]:          0 :         } else if (time_after(jiffies, rcu_state.gp_start + jtsq)) {
    1080                 :          0 :                 WRITE_ONCE(*ruqp, true);
    1081                 :            :         }
    1082                 :            : 
    1083                 :            :         /*
    1084                 :            :          * NO_HZ_FULL CPUs can run in-kernel without rcu_sched_clock_irq!
    1085                 :            :          * The above code handles this, but only for straight cond_resched().
    1086                 :            :          * And some in-kernel loops check need_resched() before calling
    1087                 :            :          * cond_resched(), which defeats the above code for CPUs that are
    1088                 :            :          * running in-kernel with scheduling-clock interrupts disabled.
    1089                 :            :          * So hit them over the head with the resched_cpu() hammer!
    1090                 :            :          */
    1091         [ #  # ]:          0 :         if (tick_nohz_full_cpu(rdp->cpu) &&
    1092                 :            :                    time_after(jiffies,
    1093                 :            :                               READ_ONCE(rdp->last_fqs_resched) + jtsq * 3)) {
    1094                 :            :                 WRITE_ONCE(*ruqp, true);
    1095                 :            :                 resched_cpu(rdp->cpu);
    1096                 :            :                 WRITE_ONCE(rdp->last_fqs_resched, jiffies);
    1097                 :            :         }
    1098                 :            : 
    1099                 :            :         /*
    1100                 :            :          * If more than halfway to RCU CPU stall-warning time, invoke
    1101                 :            :          * resched_cpu() more frequently to try to loosen things up a bit.
    1102                 :            :          * Also check to see if the CPU is getting hammered with interrupts,
    1103                 :            :          * but only once per grace period, just to keep the IPIs down to
    1104                 :            :          * a dull roar.
    1105                 :            :          */
    1106         [ #  # ]:          0 :         if (time_after(jiffies, rcu_state.jiffies_resched)) {
    1107         [ #  # ]:          0 :                 if (time_after(jiffies,
    1108                 :            :                                READ_ONCE(rdp->last_fqs_resched) + jtsq)) {
    1109                 :          0 :                         resched_cpu(rdp->cpu);
    1110                 :          0 :                         WRITE_ONCE(rdp->last_fqs_resched, jiffies);
    1111                 :            :                 }
    1112                 :          0 :                 if (IS_ENABLED(CONFIG_IRQ_WORK) &&
    1113   [ #  #  #  # ]:          0 :                     !rdp->rcu_iw_pending && rdp->rcu_iw_gp_seq != rnp->gp_seq &&
    1114         [ #  # ]:          0 :                     (rnp->ffmask & rdp->grpmask)) {
    1115                 :          0 :                         init_irq_work(&rdp->rcu_iw, rcu_iw_handler);
    1116                 :          0 :                         rdp->rcu_iw_pending = true;
    1117                 :          0 :                         rdp->rcu_iw_gp_seq = rnp->gp_seq;
    1118                 :          0 :                         irq_work_queue_on(&rdp->rcu_iw, rdp->cpu);
    1119                 :            :                 }
    1120                 :            :         }
    1121                 :            : 
    1122                 :            :         return 0;
    1123                 :            : }
    1124                 :            : 
    1125                 :            : /* Trace-event wrapper function for trace_rcu_future_grace_period.  */
    1126                 :      67521 : static void trace_rcu_this_gp(struct rcu_node *rnp, struct rcu_data *rdp,
    1127                 :            :                               unsigned long gp_seq_req, const char *s)
    1128                 :            : {
    1129                 :      67521 :         trace_rcu_future_grace_period(rcu_state.name, rnp->gp_seq, gp_seq_req,
    1130                 :      67521 :                                       rnp->level, rnp->grplo, rnp->grphi, s);
    1131                 :            : }
    1132                 :            : 
    1133                 :            : /*
    1134                 :            :  * rcu_start_this_gp - Request the start of a particular grace period
    1135                 :            :  * @rnp_start: The leaf node of the CPU from which to start.
    1136                 :            :  * @rdp: The rcu_data corresponding to the CPU from which to start.
    1137                 :            :  * @gp_seq_req: The gp_seq of the grace period to start.
    1138                 :            :  *
    1139                 :            :  * Start the specified grace period, as needed to handle newly arrived
    1140                 :            :  * callbacks.  The required future grace periods are recorded in each
    1141                 :            :  * rcu_node structure's ->gp_seq_needed field.  Returns true if there
    1142                 :            :  * is reason to awaken the grace-period kthread.
    1143                 :            :  *
    1144                 :            :  * The caller must hold the specified rcu_node structure's ->lock, which
    1145                 :            :  * is why the caller is responsible for waking the grace-period kthread.
    1146                 :            :  *
    1147                 :            :  * Returns true if the GP thread needs to be awakened else false.
    1148                 :            :  */
    1149                 :            : static bool rcu_start_this_gp(struct rcu_node *rnp_start, struct rcu_data *rdp,
    1150                 :            :                               unsigned long gp_seq_req)
    1151                 :            : {
    1152                 :            :         bool ret = false;
    1153                 :            :         struct rcu_node *rnp;
    1154                 :            : 
    1155                 :            :         /*
    1156                 :            :          * Use funnel locking to either acquire the root rcu_node
    1157                 :            :          * structure's lock or bail out if the need for this grace period
    1158                 :            :          * has already been recorded -- or if that grace period has in
    1159                 :            :          * fact already started.  If there is already a grace period in
    1160                 :            :          * progress in a non-leaf node, no recording is needed because the
    1161                 :            :          * end of the grace period will scan the leaf rcu_node structures.
    1162                 :            :          * Note that rnp_start->lock must not be released.
    1163                 :            :          */
    1164                 :            :         raw_lockdep_assert_held_rcu_node(rnp_start);
    1165                 :            :         trace_rcu_this_gp(rnp_start, rdp, gp_seq_req, TPS("Startleaf"));
    1166                 :            :         for (rnp = rnp_start; 1; rnp = rnp->parent) {
    1167                 :            :                 if (rnp != rnp_start)
    1168                 :            :                         raw_spin_lock_rcu_node(rnp);
    1169                 :            :                 if (ULONG_CMP_GE(rnp->gp_seq_needed, gp_seq_req) ||
    1170                 :            :                     rcu_seq_started(&rnp->gp_seq, gp_seq_req) ||
    1171                 :            :                     (rnp != rnp_start &&
    1172                 :            :                      rcu_seq_state(rcu_seq_current(&rnp->gp_seq)))) {
    1173                 :            :                         trace_rcu_this_gp(rnp, rdp, gp_seq_req,
    1174                 :            :                                           TPS("Prestarted"));
    1175                 :            :                         goto unlock_out;
    1176                 :            :                 }
    1177                 :            :                 rnp->gp_seq_needed = gp_seq_req;
    1178                 :            :                 if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) {
    1179                 :            :                         /*
    1180                 :            :                          * We just marked the leaf or internal node, and a
    1181                 :            :                          * grace period is in progress, which means that
    1182                 :            :                          * rcu_gp_cleanup() will see the marking.  Bail to
    1183                 :            :                          * reduce contention.
    1184                 :            :                          */
    1185                 :            :                         trace_rcu_this_gp(rnp_start, rdp, gp_seq_req,
    1186                 :            :                                           TPS("Startedleaf"));
    1187                 :            :                         goto unlock_out;
    1188                 :            :                 }
    1189                 :            :                 if (rnp != rnp_start && rnp->parent != NULL)
    1190                 :            :                         raw_spin_unlock_rcu_node(rnp);
    1191                 :            :                 if (!rnp->parent)
    1192                 :            :                         break;  /* At root, and perhaps also leaf. */
    1193                 :            :         }
    1194                 :            : 
    1195                 :            :         /* If GP already in progress, just leave, otherwise start one. */
    1196                 :            :         if (rcu_gp_in_progress()) {
    1197                 :            :                 trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedleafroot"));
    1198                 :            :                 goto unlock_out;
    1199                 :            :         }
    1200                 :            :         trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedroot"));
    1201                 :            :         WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT);
    1202                 :            :         rcu_state.gp_req_activity = jiffies;
    1203                 :            :         if (!rcu_state.gp_kthread) {
    1204                 :            :                 trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("NoGPkthread"));
    1205                 :            :                 goto unlock_out;
    1206                 :            :         }
    1207                 :            :         trace_rcu_grace_period(rcu_state.name, READ_ONCE(rcu_state.gp_seq), TPS("newreq"));
    1208                 :            :         ret = true;  /* Caller must wake GP kthread. */
    1209                 :            : unlock_out:
    1210                 :            :         /* Push furthest requested GP to leaf node and rcu_data structure. */
    1211                 :            :         if (ULONG_CMP_LT(gp_seq_req, rnp->gp_seq_needed)) {
    1212                 :            :                 rnp_start->gp_seq_needed = rnp->gp_seq_needed;
    1213                 :            :                 rdp->gp_seq_needed = rnp->gp_seq_needed;
    1214                 :            :         }
    1215                 :            :         if (rnp != rnp_start)
    1216                 :            :                 raw_spin_unlock_rcu_node(rnp);
    1217                 :            :         return ret;
    1218                 :            : }
    1219                 :            : 
    1220                 :            : /*
    1221                 :            :  * Clean up any old requests for the just-ended grace period.  Also return
    1222                 :            :  * whether any additional grace periods have been requested.
    1223                 :            :  */
    1224                 :      67521 : static bool rcu_future_gp_cleanup(struct rcu_node *rnp)
    1225                 :            : {
    1226                 :      67521 :         bool needmore;
    1227                 :      67521 :         struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
    1228                 :            : 
    1229                 :      67521 :         needmore = ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed);
    1230         [ +  + ]:      67521 :         if (!needmore)
    1231                 :       2397 :                 rnp->gp_seq_needed = rnp->gp_seq; /* Avoid counter wrap. */
    1232         [ +  + ]:      67521 :         trace_rcu_this_gp(rnp, rdp, rnp->gp_seq,
    1233                 :      67521 :                           needmore ? TPS("CleanupMore") : TPS("Cleanup"));
    1234                 :      67521 :         return needmore;
    1235                 :            : }
    1236                 :            : 
    1237                 :            : /*
    1238                 :            :  * Awaken the grace-period kthread.  Don't do a self-awaken (unless in
    1239                 :            :  * an interrupt or softirq handler), and don't bother awakening when there
    1240                 :            :  * is nothing for the grace-period kthread to do (as in several CPUs raced
    1241                 :            :  * to awaken, and we lost), and finally don't try to awaken a kthread that
    1242                 :            :  * has not yet been created.  If all those checks are passed, track some
    1243                 :            :  * debug information and awaken.
    1244                 :            :  *
    1245                 :            :  * So why do the self-wakeup when in an interrupt or softirq handler
    1246                 :            :  * in the grace-period kthread's context?  Because the kthread might have
    1247                 :            :  * been interrupted just as it was going to sleep, and just after the final
    1248                 :            :  * pre-sleep check of the awaken condition.  In this case, a wakeup really
    1249                 :            :  * is required, and is therefore supplied.
    1250                 :            :  */
    1251                 :      69917 : static void rcu_gp_kthread_wake(void)
    1252                 :            : {
    1253   [ -  +  -  - ]:      69917 :         if ((current == rcu_state.gp_kthread &&
    1254   [ -  -  +  - ]:      69917 :              !in_irq() && !in_serving_softirq()) ||
    1255   [ +  -  +  - ]:      69917 :             !READ_ONCE(rcu_state.gp_flags) ||
    1256                 :            :             !rcu_state.gp_kthread)
    1257                 :            :                 return;
    1258                 :      69917 :         WRITE_ONCE(rcu_state.gp_wake_time, jiffies);
    1259                 :      69917 :         WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));
    1260                 :      69917 :         swake_up_one(&rcu_state.gp_wq);
    1261                 :            : }
    1262                 :            : 
    1263                 :            : /*
    1264                 :            :  * If there is room, assign a ->gp_seq number to any callbacks on this
    1265                 :            :  * CPU that have not already been assigned.  Also accelerate any callbacks
    1266                 :            :  * that were previously assigned a ->gp_seq number that has since proven
    1267                 :            :  * to be too conservative, which can happen if callbacks get assigned a
    1268                 :            :  * ->gp_seq number while RCU is idle, but with reference to a non-root
    1269                 :            :  * rcu_node structure.  This function is idempotent, so it does not hurt
    1270                 :            :  * to call it repeatedly.  Returns an flag saying that we should awaken
    1271                 :            :  * the RCU grace-period kthread.
    1272                 :            :  *
    1273                 :            :  * The caller must hold rnp->lock with interrupts disabled.
    1274                 :            :  */
    1275                 :     270202 : static bool rcu_accelerate_cbs(struct rcu_node *rnp, struct rcu_data *rdp)
    1276                 :            : {
    1277                 :     270202 :         unsigned long gp_seq_req;
    1278                 :     270202 :         bool ret = false;
    1279                 :            : 
    1280                 :     270202 :         rcu_lockdep_assert_cblist_protected(rdp);
    1281                 :     270202 :         raw_lockdep_assert_held_rcu_node(rnp);
    1282                 :            : 
    1283                 :            :         /* If no pending (not yet ready to invoke) callbacks, nothing to do. */
    1284         [ +  + ]:     270202 :         if (!rcu_segcblist_pend_cbs(&rdp->cblist))
    1285                 :            :                 return false;
    1286                 :            : 
    1287                 :            :         /*
    1288                 :            :          * Callbacks are often registered with incomplete grace-period
    1289                 :            :          * information.  Something about the fact that getting exact
    1290                 :            :          * information requires acquiring a global lock...  RCU therefore
    1291                 :            :          * makes a conservative estimate of the grace period number at which
    1292                 :            :          * a given callback will become ready to invoke.        The following
    1293                 :            :          * code checks this estimate and improves it when possible, thus
    1294                 :            :          * accelerating callback invocation to an earlier grace-period
    1295                 :            :          * number.
    1296                 :            :          */
    1297                 :     214921 :         gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq);
    1298         [ +  - ]:     214921 :         if (rcu_segcblist_accelerate(&rdp->cblist, gp_seq_req))
    1299                 :     214921 :                 ret = rcu_start_this_gp(rnp, rdp, gp_seq_req);
    1300                 :            : 
    1301                 :            :         /* Trace depending on how much we were able to accelerate. */
    1302         [ +  + ]:     214921 :         if (rcu_segcblist_restempty(&rdp->cblist, RCU_WAIT_TAIL))
    1303                 :     190721 :                 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("AccWaitCB"));
    1304                 :            :         else
    1305                 :      24200 :                 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("AccReadyCB"));
    1306                 :            :         return ret;
    1307                 :            : }
    1308                 :            : 
    1309                 :            : /*
    1310                 :            :  * Similar to rcu_accelerate_cbs(), but does not require that the leaf
    1311                 :            :  * rcu_node structure's ->lock be held.  It consults the cached value
    1312                 :            :  * of ->gp_seq_needed in the rcu_data structure, and if that indicates
    1313                 :            :  * that a new grace-period request be made, invokes rcu_accelerate_cbs()
    1314                 :            :  * while holding the leaf rcu_node structure's ->lock.
    1315                 :            :  */
    1316                 :       2485 : static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp,
    1317                 :            :                                         struct rcu_data *rdp)
    1318                 :            : {
    1319                 :       2485 :         unsigned long c;
    1320                 :       2485 :         bool needwake;
    1321                 :            : 
    1322                 :       2485 :         rcu_lockdep_assert_cblist_protected(rdp);
    1323                 :       2485 :         c = rcu_seq_snap(&rcu_state.gp_seq);
    1324   [ +  -  -  + ]:       2485 :         if (!rdp->gpwrap && ULONG_CMP_GE(rdp->gp_seq_needed, c)) {
    1325                 :            :                 /* Old request still live, so mark recent callbacks. */
    1326                 :          0 :                 (void)rcu_segcblist_accelerate(&rdp->cblist, c);
    1327                 :          0 :                 return;
    1328                 :            :         }
    1329                 :       2485 :         raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
    1330                 :       2485 :         needwake = rcu_accelerate_cbs(rnp, rdp);
    1331                 :       2485 :         raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
    1332         [ +  + ]:       2485 :         if (needwake)
    1333                 :       2396 :                 rcu_gp_kthread_wake();
    1334                 :            : }
    1335                 :            : 
    1336                 :            : /*
    1337                 :            :  * Move any callbacks whose grace period has completed to the
    1338                 :            :  * RCU_DONE_TAIL sublist, then compact the remaining sublists and
    1339                 :            :  * assign ->gp_seq numbers to any callbacks in the RCU_NEXT_TAIL
    1340                 :            :  * sublist.  This function is idempotent, so it does not hurt to
    1341                 :            :  * invoke it repeatedly.  As long as it is not invoked -too- often...
    1342                 :            :  * Returns true if the RCU grace-period kthread needs to be awakened.
    1343                 :            :  *
    1344                 :            :  * The caller must hold rnp->lock with interrupts disabled.
    1345                 :            :  */
    1346                 :      67521 : static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp)
    1347                 :            : {
    1348                 :      67521 :         rcu_lockdep_assert_cblist_protected(rdp);
    1349                 :      67521 :         raw_lockdep_assert_held_rcu_node(rnp);
    1350                 :            : 
    1351                 :            :         /* If no pending (not yet ready to invoke) callbacks, nothing to do. */
    1352         [ +  + ]:      67521 :         if (!rcu_segcblist_pend_cbs(&rdp->cblist))
    1353                 :            :                 return false;
    1354                 :            : 
    1355                 :            :         /*
    1356                 :            :          * Find all callbacks whose ->gp_seq numbers indicate that they
    1357                 :            :          * are ready to invoke, and put them into the RCU_DONE_TAIL sublist.
    1358                 :            :          */
    1359                 :      65124 :         rcu_segcblist_advance(&rdp->cblist, rnp->gp_seq);
    1360                 :            : 
    1361                 :            :         /* Classify any remaining callbacks. */
    1362                 :      65124 :         return rcu_accelerate_cbs(rnp, rdp);
    1363                 :            : }
    1364                 :            : 
    1365                 :            : /*
    1366                 :            :  * Move and classify callbacks, but only if doing so won't require
    1367                 :            :  * that the RCU grace-period kthread be awakened.
    1368                 :            :  */
    1369                 :            : static void __maybe_unused rcu_advance_cbs_nowake(struct rcu_node *rnp,
    1370                 :            :                                                   struct rcu_data *rdp)
    1371                 :            : {
    1372                 :            :         rcu_lockdep_assert_cblist_protected(rdp);
    1373                 :            :         if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq)) ||
    1374                 :            :             !raw_spin_trylock_rcu_node(rnp))
    1375                 :            :                 return;
    1376                 :            :         WARN_ON_ONCE(rcu_advance_cbs(rnp, rdp));
    1377                 :            :         raw_spin_unlock_rcu_node(rnp);
    1378                 :            : }
    1379                 :            : 
    1380                 :            : /*
    1381                 :            :  * Update CPU-local rcu_data state to record the beginnings and ends of
    1382                 :            :  * grace periods.  The caller must hold the ->lock of the leaf rcu_node
    1383                 :            :  * structure corresponding to the current CPU, and must have irqs disabled.
    1384                 :            :  * Returns true if the grace-period kthread needs to be awakened.
    1385                 :            :  */
    1386                 :     135072 : static bool __note_gp_changes(struct rcu_node *rnp, struct rcu_data *rdp)
    1387                 :            : {
    1388                 :     135072 :         bool ret = false;
    1389                 :     135072 :         bool need_gp;
    1390                 :     135072 :         const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
    1391                 :            :                                rcu_segcblist_is_offloaded(&rdp->cblist);
    1392                 :            : 
    1393                 :     135072 :         raw_lockdep_assert_held_rcu_node(rnp);
    1394                 :            : 
    1395         [ +  - ]:     135072 :         if (rdp->gp_seq == rnp->gp_seq)
    1396                 :            :                 return false; /* Nothing to do. */
    1397                 :            : 
    1398                 :            :         /* Handle the ends of any preceding grace periods first. */
    1399   [ +  +  -  + ]:     135072 :         if (rcu_seq_completed_gp(rdp->gp_seq, rnp->gp_seq) ||
    1400         [ -  + ]:      67551 :             unlikely(READ_ONCE(rdp->gpwrap))) {
    1401                 :      67521 :                 if (!offloaded)
    1402                 :      67521 :                         ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */
    1403                 :      67521 :                 trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));
    1404                 :            :         } else {
    1405                 :      67551 :                 if (!offloaded)
    1406                 :      67551 :                         ret = rcu_accelerate_cbs(rnp, rdp); /* Recent CBs. */
    1407                 :            :         }
    1408                 :            : 
    1409                 :            :         /* Now handle the beginnings of any new-to-this-CPU grace periods. */
    1410   [ +  +  -  + ]:     135072 :         if (rcu_seq_new_gp(rdp->gp_seq, rnp->gp_seq) ||
    1411         [ -  + ]:      67521 :             unlikely(READ_ONCE(rdp->gpwrap))) {
    1412                 :            :                 /*
    1413                 :            :                  * If the current grace period is waiting for this CPU,
    1414                 :            :                  * set up to detect a quiescent state, otherwise don't
    1415                 :            :                  * go looking for one.
    1416                 :            :                  */
    1417                 :      67551 :                 trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));
    1418                 :      67551 :                 need_gp = !!(rnp->qsmask & rdp->grpmask);
    1419                 :      67551 :                 rdp->cpu_no_qs.b.norm = need_gp;
    1420                 :      67551 :                 rdp->core_needs_qs = need_gp;
    1421                 :      67551 :                 zero_cpu_stall_ticks(rdp);
    1422                 :            :         }
    1423                 :     135072 :         rdp->gp_seq = rnp->gp_seq;  /* Remember new grace-period state. */
    1424   [ +  +  -  + ]:     135072 :         if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) || rdp->gpwrap)
    1425                 :      65152 :                 rdp->gp_seq_needed = rnp->gp_seq_needed;
    1426         [ -  + ]:     135072 :         WRITE_ONCE(rdp->gpwrap, false);
    1427         [ -  + ]:     135072 :         rcu_gpnum_ovf(rnp, rdp);
    1428                 :            :         return ret;
    1429                 :            : }
    1430                 :            : 
    1431                 :     247986 : static void note_gp_changes(struct rcu_data *rdp)
    1432                 :            : {
    1433                 :     247986 :         unsigned long flags;
    1434                 :     247986 :         bool needwake;
    1435                 :     247986 :         struct rcu_node *rnp;
    1436                 :            : 
    1437                 :     247986 :         local_irq_save(flags);
    1438                 :     247986 :         rnp = rdp->mynode;
    1439   [ +  -  -  + ]:     247986 :         if ((rdp->gp_seq == rcu_seq_current(&rnp->gp_seq) &&
    1440         [ -  + ]:     247986 :              !unlikely(READ_ONCE(rdp->gpwrap))) || /* w/out lock. */
    1441         [ #  # ]:          0 :             !raw_spin_trylock_rcu_node(rnp)) { /* irqs already off, so later. */
    1442                 :     247986 :                 local_irq_restore(flags);
    1443                 :     247986 :                 return;
    1444                 :            :         }
    1445                 :          0 :         needwake = __note_gp_changes(rnp, rdp);
    1446                 :          0 :         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    1447         [ #  # ]:          0 :         if (needwake)
    1448                 :          0 :                 rcu_gp_kthread_wake();
    1449                 :            : }
    1450                 :            : 
    1451                 :     202623 : static void rcu_gp_slow(int delay)
    1452                 :            : {
    1453         [ -  + ]:     202623 :         if (delay > 0 &&
    1454         [ #  # ]:          0 :             !(rcu_seq_ctr(rcu_state.gp_seq) %
    1455         [ #  # ]:          0 :               (rcu_num_nodes * PER_RCU_NODE_PERIOD * delay)))
    1456                 :          0 :                 schedule_timeout_uninterruptible(delay);
    1457                 :     202623 : }
    1458                 :            : 
    1459                 :            : /*
    1460                 :            :  * Initialize a new grace period.  Return false if no grace period required.
    1461                 :            :  */
    1462                 :      67551 : static bool rcu_gp_init(void)
    1463                 :            : {
    1464                 :      67551 :         unsigned long flags;
    1465                 :      67551 :         unsigned long oldmask;
    1466                 :      67551 :         unsigned long mask;
    1467                 :      67551 :         struct rcu_data *rdp;
    1468                 :      67551 :         struct rcu_node *rnp = rcu_get_root();
    1469                 :            : 
    1470                 :      67551 :         WRITE_ONCE(rcu_state.gp_activity, jiffies);
    1471                 :      67551 :         raw_spin_lock_irq_rcu_node(rnp);
    1472         [ -  + ]:      67551 :         if (!READ_ONCE(rcu_state.gp_flags)) {
    1473                 :            :                 /* Spurious wakeup, tell caller to go back to sleep.  */
    1474                 :          0 :                 raw_spin_unlock_irq_rcu_node(rnp);
    1475                 :          0 :                 return false;
    1476                 :            :         }
    1477         [ -  + ]:      67551 :         WRITE_ONCE(rcu_state.gp_flags, 0); /* Clear all flags: New GP. */
    1478                 :            : 
    1479   [ -  +  -  + ]:      67551 :         if (WARN_ON_ONCE(rcu_gp_in_progress())) {
    1480                 :            :                 /*
    1481                 :            :                  * Grace period already in progress, don't start another.
    1482                 :            :                  * Not supposed to be able to happen.
    1483                 :            :                  */
    1484                 :          0 :                 raw_spin_unlock_irq_rcu_node(rnp);
    1485                 :          0 :                 return false;
    1486                 :            :         }
    1487                 :            : 
    1488                 :            :         /* Advance to a new grace period and initialize state. */
    1489                 :      67551 :         record_gp_stall_check_time();
    1490                 :            :         /* Record GP times before starting GP, hence rcu_seq_start(). */
    1491                 :      67551 :         rcu_seq_start(&rcu_state.gp_seq);
    1492                 :      67551 :         trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start"));
    1493                 :      67551 :         raw_spin_unlock_irq_rcu_node(rnp);
    1494                 :            : 
    1495                 :            :         /*
    1496                 :            :          * Apply per-leaf buffered online and offline operations to the
    1497                 :            :          * rcu_node tree.  Note that this new grace period need not wait
    1498                 :            :          * for subsequent online CPUs, and that quiescent-state forcing
    1499                 :            :          * will handle subsequent offline CPUs.
    1500                 :            :          */
    1501                 :      67551 :         rcu_state.gp_state = RCU_GP_ONOFF;
    1502         [ +  + ]:     135102 :         rcu_for_each_leaf_node(rnp) {
    1503                 :      67551 :                 raw_spin_lock(&rcu_state.ofl_lock);
    1504                 :      67551 :                 raw_spin_lock_irq_rcu_node(rnp);
    1505         [ +  + ]:      67551 :                 if (rnp->qsmaskinit == rnp->qsmaskinitnext &&
    1506         [ +  - ]:      67521 :                     !rnp->wait_blkd_tasks) {
    1507                 :            :                         /* Nothing to do on this leaf rcu_node structure. */
    1508                 :      67521 :                         raw_spin_unlock_irq_rcu_node(rnp);
    1509                 :      67521 :                         raw_spin_unlock(&rcu_state.ofl_lock);
    1510                 :      67521 :                         continue;
    1511                 :            :                 }
    1512                 :            : 
    1513                 :            :                 /* Record old state, apply changes to ->qsmaskinit field. */
    1514                 :         30 :                 oldmask = rnp->qsmaskinit;
    1515                 :         30 :                 rnp->qsmaskinit = rnp->qsmaskinitnext;
    1516                 :            : 
    1517                 :            :                 /* If zero-ness of ->qsmaskinit changed, propagate up tree. */
    1518         [ +  - ]:         30 :                 if (!oldmask != !rnp->qsmaskinit) {
    1519         [ +  - ]:         30 :                         if (!oldmask) { /* First online CPU for rcu_node. */
    1520         [ +  - ]:         30 :                                 if (!rnp->wait_blkd_tasks) /* Ever offline? */
    1521                 :         30 :                                         rcu_init_new_rnp(rnp);
    1522                 :          0 :                         } else if (rcu_preempt_has_tasks(rnp)) {
    1523                 :            :                                 rnp->wait_blkd_tasks = true; /* blocked tasks */
    1524                 :            :                         } else { /* Last offline CPU and can propagate. */
    1525                 :          0 :                                 rcu_cleanup_dead_rnp(rnp);
    1526                 :            :                         }
    1527                 :            :                 }
    1528                 :            : 
    1529                 :            :                 /*
    1530                 :            :                  * If all waited-on tasks from prior grace period are
    1531                 :            :                  * done, and if all this rcu_node structure's CPUs are
    1532                 :            :                  * still offline, propagate up the rcu_node tree and
    1533                 :            :                  * clear ->wait_blkd_tasks.  Otherwise, if one of this
    1534                 :            :                  * rcu_node structure's CPUs has since come back online,
    1535                 :            :                  * simply clear ->wait_blkd_tasks.
    1536                 :            :                  */
    1537         [ -  + ]:         30 :                 if (rnp->wait_blkd_tasks &&
    1538                 :            :                     (!rcu_preempt_has_tasks(rnp) || rnp->qsmaskinit)) {
    1539                 :          0 :                         rnp->wait_blkd_tasks = false;
    1540         [ #  # ]:          0 :                         if (!rnp->qsmaskinit)
    1541                 :          0 :                                 rcu_cleanup_dead_rnp(rnp);
    1542                 :            :                 }
    1543                 :            : 
    1544                 :         30 :                 raw_spin_unlock_irq_rcu_node(rnp);
    1545                 :         30 :                 raw_spin_unlock(&rcu_state.ofl_lock);
    1546                 :            :         }
    1547                 :      67551 :         rcu_gp_slow(gp_preinit_delay); /* Races with CPU hotplug. */
    1548                 :            : 
    1549                 :            :         /*
    1550                 :            :          * Set the quiescent-state-needed bits in all the rcu_node
    1551                 :            :          * structures for all currently online CPUs in breadth-first
    1552                 :            :          * order, starting from the root rcu_node structure, relying on the
    1553                 :            :          * layout of the tree within the rcu_state.node[] array.  Note that
    1554                 :            :          * other CPUs will access only the leaves of the hierarchy, thus
    1555                 :            :          * seeing that no grace period is in progress, at least until the
    1556                 :            :          * corresponding leaf node has been initialized.
    1557                 :            :          *
    1558                 :            :          * The grace period cannot complete until the initialization
    1559                 :            :          * process finishes, because this kthread handles both.
    1560                 :            :          */
    1561                 :      67551 :         rcu_state.gp_state = RCU_GP_INIT;
    1562         [ +  + ]:     135102 :         rcu_for_each_node_breadth_first(rnp) {
    1563                 :      67551 :                 rcu_gp_slow(gp_init_delay);
    1564                 :      67551 :                 raw_spin_lock_irqsave_rcu_node(rnp, flags);
    1565                 :      67551 :                 rdp = this_cpu_ptr(&rcu_data);
    1566         [ -  + ]:      67551 :                 rcu_preempt_check_blocked_tasks(rnp);
    1567                 :      67551 :                 rnp->qsmask = rnp->qsmaskinit;
    1568         [ +  - ]:      67551 :                 WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq);
    1569         [ +  - ]:      67551 :                 if (rnp == rdp->mynode)
    1570                 :      67551 :                         (void)__note_gp_changes(rnp, rdp);
    1571                 :      67551 :                 rcu_preempt_boost_start_gp(rnp);
    1572                 :      67551 :                 trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq,
    1573                 :      67551 :                                             rnp->level, rnp->grplo,
    1574                 :            :                                             rnp->grphi, rnp->qsmask);
    1575                 :            :                 /* Quiescent states for tasks on any now-offline CPUs. */
    1576                 :      67551 :                 mask = rnp->qsmask & ~rnp->qsmaskinitnext;
    1577                 :      67551 :                 rnp->rcu_gp_init_mask = mask;
    1578   [ +  -  -  +  :      67551 :                 if ((mask || rnp->wait_blkd_tasks) && rcu_is_leaf_node(rnp))
                   -  - ]
    1579                 :          0 :                         rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
    1580                 :            :                 else
    1581                 :      67551 :                         raw_spin_unlock_irq_rcu_node(rnp);
    1582                 :      67551 :                 cond_resched_tasks_rcu_qs();
    1583                 :      67551 :                 WRITE_ONCE(rcu_state.gp_activity, jiffies);
    1584                 :            :         }
    1585                 :            : 
    1586                 :            :         return true;
    1587                 :            : }
    1588                 :            : 
    1589                 :            : /*
    1590                 :            :  * Helper function for swait_event_idle_exclusive() wakeup at force-quiescent-state
    1591                 :            :  * time.
    1592                 :            :  */
    1593                 :     202623 : static bool rcu_gp_fqs_check_wake(int *gfp)
    1594                 :            : {
    1595                 :     202623 :         struct rcu_node *rnp = rcu_get_root();
    1596                 :            : 
    1597                 :            :         /* Someone like call_rcu() requested a force-quiescent-state scan. */
    1598                 :     202623 :         *gfp = READ_ONCE(rcu_state.gp_flags);
    1599   [ +  -  +  + ]:     202623 :         if (*gfp & RCU_GP_FLAG_FQS)
    1600                 :            :                 return true;
    1601                 :            : 
    1602                 :            :         /* The current grace period has completed. */
    1603   [ -  +  -  + ]:     135102 :         if (!READ_ONCE(rnp->qsmask) && !rcu_preempt_blocked_readers_cgp(rnp))
    1604                 :          0 :                 return true;
    1605                 :            : 
    1606                 :            :         return false;
    1607                 :            : }
    1608                 :            : 
    1609                 :            : /*
    1610                 :            :  * Do one round of quiescent-state forcing.
    1611                 :            :  */
    1612                 :          0 : static void rcu_gp_fqs(bool first_time)
    1613                 :            : {
    1614                 :          0 :         struct rcu_node *rnp = rcu_get_root();
    1615                 :            : 
    1616         [ #  # ]:          0 :         WRITE_ONCE(rcu_state.gp_activity, jiffies);
    1617                 :          0 :         rcu_state.n_force_qs++;
    1618         [ #  # ]:          0 :         if (first_time) {
    1619                 :            :                 /* Collect dyntick-idle snapshots. */
    1620                 :          0 :                 force_qs_rnp(dyntick_save_progress_counter);
    1621                 :            :         } else {
    1622                 :            :                 /* Handle dyntick-idle and offline CPUs. */
    1623                 :          0 :                 force_qs_rnp(rcu_implicit_dynticks_qs);
    1624                 :            :         }
    1625                 :            :         /* Clear flag to prevent immediate re-entry. */
    1626         [ #  # ]:          0 :         if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {
    1627                 :          0 :                 raw_spin_lock_irq_rcu_node(rnp);
    1628                 :          0 :                 WRITE_ONCE(rcu_state.gp_flags,
    1629                 :            :                            READ_ONCE(rcu_state.gp_flags) & ~RCU_GP_FLAG_FQS);
    1630                 :          0 :                 raw_spin_unlock_irq_rcu_node(rnp);
    1631                 :            :         }
    1632                 :          0 : }
    1633                 :            : 
    1634                 :            : /*
    1635                 :            :  * Loop doing repeated quiescent-state forcing until the grace period ends.
    1636                 :            :  */
    1637                 :      67551 : static void rcu_gp_fqs_loop(void)
    1638                 :            : {
    1639                 :      67551 :         bool first_gp_fqs;
    1640                 :      67551 :         int gf;
    1641                 :      67551 :         unsigned long j;
    1642                 :      67551 :         int ret;
    1643                 :      67551 :         struct rcu_node *rnp = rcu_get_root();
    1644                 :            : 
    1645                 :      67551 :         first_gp_fqs = true;
    1646                 :      67551 :         j = READ_ONCE(jiffies_till_first_fqs);
    1647                 :      67551 :         ret = 0;
    1648                 :      67551 :         for (;;) {
    1649         [ +  - ]:      67551 :                 if (!ret) {
    1650                 :      67551 :                         rcu_state.jiffies_force_qs = jiffies + j;
    1651         [ +  - ]:      67551 :                         WRITE_ONCE(rcu_state.jiffies_kick_kthreads,
    1652                 :            :                                    jiffies + (j ? 3 * j : 2));
    1653                 :            :                 }
    1654                 :      67551 :                 trace_rcu_grace_period(rcu_state.name,
    1655                 :      67551 :                                        READ_ONCE(rcu_state.gp_seq),
    1656                 :      67551 :                                        TPS("fqswait"));
    1657                 :      67551 :                 rcu_state.gp_state = RCU_GP_WAIT_FQS;
    1658   [ +  -  -  +  :     270174 :                 ret = swait_event_idle_timeout_exclusive(
          +  -  +  +  +  
                +  +  + ]
    1659                 :            :                                 rcu_state.gp_wq, rcu_gp_fqs_check_wake(&gf), j);
    1660                 :      67521 :                 rcu_state.gp_state = RCU_GP_DOING_FQS;
    1661                 :            :                 /* Locking provides needed memory barriers. */
    1662                 :            :                 /* If grace period done, leave loop. */
    1663         [ -  + ]:      67521 :                 if (!READ_ONCE(rnp->qsmask) &&
    1664                 :            :                     !rcu_preempt_blocked_readers_cgp(rnp))
    1665                 :            :                         break;
    1666                 :            :                 /* If time for quiescent-state forcing, do it. */
    1667         [ #  # ]:          0 :                 if (ULONG_CMP_GE(jiffies, rcu_state.jiffies_force_qs) ||
    1668         [ #  # ]:          0 :                     (gf & RCU_GP_FLAG_FQS)) {
    1669                 :          0 :                         trace_rcu_grace_period(rcu_state.name,
    1670                 :          0 :                                                READ_ONCE(rcu_state.gp_seq),
    1671                 :          0 :                                                TPS("fqsstart"));
    1672                 :          0 :                         rcu_gp_fqs(first_gp_fqs);
    1673                 :          0 :                         first_gp_fqs = false;
    1674                 :          0 :                         trace_rcu_grace_period(rcu_state.name,
    1675                 :          0 :                                                READ_ONCE(rcu_state.gp_seq),
    1676                 :          0 :                                                TPS("fqsend"));
    1677                 :          0 :                         cond_resched_tasks_rcu_qs();
    1678                 :          0 :                         WRITE_ONCE(rcu_state.gp_activity, jiffies);
    1679                 :          0 :                         ret = 0; /* Force full wait till next FQS. */
    1680                 :          0 :                         j = READ_ONCE(jiffies_till_next_fqs);
    1681                 :            :                 } else {
    1682                 :            :                         /* Deal with stray signal. */
    1683                 :          0 :                         cond_resched_tasks_rcu_qs();
    1684                 :          0 :                         WRITE_ONCE(rcu_state.gp_activity, jiffies);
    1685         [ #  # ]:          0 :                         WARN_ON(signal_pending(current));
    1686                 :          0 :                         trace_rcu_grace_period(rcu_state.name,
    1687                 :          0 :                                                READ_ONCE(rcu_state.gp_seq),
    1688                 :          0 :                                                TPS("fqswaitsig"));
    1689                 :          0 :                         ret = 1; /* Keep old FQS timing. */
    1690                 :          0 :                         j = jiffies;
    1691         [ #  # ]:          0 :                         if (time_after(jiffies, rcu_state.jiffies_force_qs))
    1692                 :            :                                 j = 1;
    1693                 :            :                         else
    1694                 :          0 :                                 j = rcu_state.jiffies_force_qs - j;
    1695                 :            :                 }
    1696                 :            :         }
    1697                 :      67521 : }
    1698                 :            : 
    1699                 :            : /*
    1700                 :            :  * Clean up after the old grace period.
    1701                 :            :  */
    1702                 :      67521 : static void rcu_gp_cleanup(void)
    1703                 :            : {
    1704                 :      67521 :         unsigned long gp_duration;
    1705                 :      67521 :         bool needgp = false;
    1706                 :      67521 :         unsigned long new_gp_seq;
    1707                 :      67521 :         bool offloaded;
    1708                 :      67521 :         struct rcu_data *rdp;
    1709                 :      67521 :         struct rcu_node *rnp = rcu_get_root();
    1710                 :      67521 :         struct swait_queue_head *sq;
    1711                 :            : 
    1712                 :      67521 :         WRITE_ONCE(rcu_state.gp_activity, jiffies);
    1713                 :      67521 :         raw_spin_lock_irq_rcu_node(rnp);
    1714                 :      67521 :         rcu_state.gp_end = jiffies;
    1715                 :      67521 :         gp_duration = rcu_state.gp_end - rcu_state.gp_start;
    1716         [ +  + ]:      67521 :         if (gp_duration > rcu_state.gp_max)
    1717                 :        145 :                 rcu_state.gp_max = gp_duration;
    1718                 :            : 
    1719                 :            :         /*
    1720                 :            :          * We know the grace period is complete, but to everyone else
    1721                 :            :          * it appears to still be ongoing.  But it is also the case
    1722                 :            :          * that to everyone else it looks like there is nothing that
    1723                 :            :          * they can do to advance the grace period.  It is therefore
    1724                 :            :          * safe for us to drop the lock in order to mark the grace
    1725                 :            :          * period as completed in all of the rcu_node structures.
    1726                 :            :          */
    1727                 :      67521 :         raw_spin_unlock_irq_rcu_node(rnp);
    1728                 :            : 
    1729                 :            :         /*
    1730                 :            :          * Propagate new ->gp_seq value to rcu_node structures so that
    1731                 :            :          * other CPUs don't have to wait until the start of the next grace
    1732                 :            :          * period to process their callbacks.  This also avoids some nasty
    1733                 :            :          * RCU grace-period initialization races by forcing the end of
    1734                 :            :          * the current grace period to be completely recorded in all of
    1735                 :            :          * the rcu_node structures before the beginning of the next grace
    1736                 :            :          * period is recorded in any of the rcu_node structures.
    1737                 :            :          */
    1738                 :      67521 :         new_gp_seq = rcu_state.gp_seq;
    1739                 :      67521 :         rcu_seq_end(&new_gp_seq);
    1740         [ +  + ]:     135042 :         rcu_for_each_node_breadth_first(rnp) {
    1741                 :      67521 :                 raw_spin_lock_irq_rcu_node(rnp);
    1742         [ -  + ]:      67521 :                 if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)))
    1743                 :            :                         dump_blkd_tasks(rnp, 10);
    1744         [ -  + ]:      67521 :                 WARN_ON_ONCE(rnp->qsmask);
    1745                 :      67521 :                 WRITE_ONCE(rnp->gp_seq, new_gp_seq);
    1746                 :      67521 :                 rdp = this_cpu_ptr(&rcu_data);
    1747         [ +  - ]:      67521 :                 if (rnp == rdp->mynode)
    1748   [ +  -  +  - ]:     135042 :                         needgp = __note_gp_changes(rnp, rdp) || needgp;
    1749                 :            :                 /* smp_mb() provided by prior unlock-lock pair. */
    1750   [ +  +  +  - ]:      67521 :                 needgp = rcu_future_gp_cleanup(rnp) || needgp;
    1751                 :      67521 :                 sq = rcu_nocb_gp_get(rnp);
    1752                 :      67521 :                 raw_spin_unlock_irq_rcu_node(rnp);
    1753                 :      67521 :                 rcu_nocb_gp_cleanup(sq);
    1754                 :      67521 :                 cond_resched_tasks_rcu_qs();
    1755                 :      67521 :                 WRITE_ONCE(rcu_state.gp_activity, jiffies);
    1756                 :      67521 :                 rcu_gp_slow(gp_cleanup_delay);
    1757                 :            :         }
    1758                 :      67521 :         rnp = rcu_get_root();
    1759                 :      67521 :         raw_spin_lock_irq_rcu_node(rnp); /* GP before ->gp_seq update. */
    1760                 :            : 
    1761                 :            :         /* Declare grace period done, trace first to use old GP number. */
    1762                 :      67521 :         trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end"));
    1763                 :      67521 :         rcu_seq_end(&rcu_state.gp_seq);
    1764                 :      67521 :         rcu_state.gp_state = RCU_GP_IDLE;
    1765                 :            :         /* Check for GP requests since above loop. */
    1766                 :      67521 :         rdp = this_cpu_ptr(&rcu_data);
    1767   [ +  +  -  + ]:      67521 :         if (!needgp && ULONG_CMP_LT(rnp->gp_seq, rnp->gp_seq_needed)) {
    1768                 :          0 :                 trace_rcu_this_gp(rnp, rdp, rnp->gp_seq_needed,
    1769                 :          0 :                                   TPS("CleanupMore"));
    1770                 :          0 :                 needgp = true;
    1771                 :            :         }
    1772                 :            :         /* Advance CBs to reduce false positives below. */
    1773                 :      67521 :         offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
    1774                 :            :                     rcu_segcblist_is_offloaded(&rdp->cblist);
    1775   [ +  +  +  + ]:      67521 :         if ((offloaded || !rcu_accelerate_cbs(rnp, rdp)) && needgp) {
    1776                 :      65124 :                 WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT);
    1777                 :      65124 :                 rcu_state.gp_req_activity = jiffies;
    1778                 :      65124 :                 trace_rcu_grace_period(rcu_state.name,
    1779                 :      65124 :                                        READ_ONCE(rcu_state.gp_seq),
    1780                 :      65124 :                                        TPS("newreq"));
    1781                 :            :         } else {
    1782                 :       2397 :                 WRITE_ONCE(rcu_state.gp_flags,
    1783                 :            :                            rcu_state.gp_flags & RCU_GP_FLAG_INIT);
    1784                 :            :         }
    1785                 :      67521 :         raw_spin_unlock_irq_rcu_node(rnp);
    1786                 :      67521 : }
    1787                 :            : 
    1788                 :            : /*
    1789                 :            :  * Body of kthread that handles grace periods.
    1790                 :            :  */
    1791                 :         30 : static int __noreturn rcu_gp_kthread(void *unused)
    1792                 :            : {
    1793                 :         30 :         rcu_bind_gp_kthread();
    1794                 :     135072 :         for (;;) {
    1795                 :            : 
    1796                 :            :                 /* Handle grace-period start. */
    1797                 :          0 :                 for (;;) {
    1798                 :          0 :                         trace_rcu_grace_period(rcu_state.name,
    1799                 :      67551 :                                                READ_ONCE(rcu_state.gp_seq),
    1800                 :      67551 :                                                TPS("reqwait"));
    1801                 :      67551 :                         rcu_state.gp_state = RCU_GP_WAIT_GPS;
    1802   [ +  +  +  + ]:      69947 :                         swait_event_idle_exclusive(rcu_state.gp_wq,
    1803                 :            :                                          READ_ONCE(rcu_state.gp_flags) &
    1804                 :            :                                          RCU_GP_FLAG_INIT);
    1805                 :      67551 :                         rcu_state.gp_state = RCU_GP_DONE_GPS;
    1806                 :            :                         /* Locking provides needed memory barrier. */
    1807         [ -  + ]:      67551 :                         if (rcu_gp_init())
    1808                 :            :                                 break;
    1809                 :          0 :                         cond_resched_tasks_rcu_qs();
    1810                 :          0 :                         WRITE_ONCE(rcu_state.gp_activity, jiffies);
    1811         [ #  # ]:          0 :                         WARN_ON(signal_pending(current));
    1812                 :          0 :                         trace_rcu_grace_period(rcu_state.name,
    1813                 :          0 :                                                READ_ONCE(rcu_state.gp_seq),
    1814                 :          0 :                                                TPS("reqwaitsig"));
    1815                 :            :                 }
    1816                 :            : 
    1817                 :            :                 /* Handle quiescent-state forcing. */
    1818                 :      67551 :                 rcu_gp_fqs_loop();
    1819                 :            : 
    1820                 :            :                 /* Handle grace-period end. */
    1821                 :      67521 :                 rcu_state.gp_state = RCU_GP_CLEANUP;
    1822                 :      67521 :                 rcu_gp_cleanup();
    1823                 :      67521 :                 rcu_state.gp_state = RCU_GP_CLEANED;
    1824                 :            :         }
    1825                 :            : }
    1826                 :            : 
    1827                 :            : /*
    1828                 :            :  * Report a full set of quiescent states to the rcu_state data structure.
    1829                 :            :  * Invoke rcu_gp_kthread_wake() to awaken the grace-period kthread if
    1830                 :            :  * another grace period is required.  Whether we wake the grace-period
    1831                 :            :  * kthread or it awakens itself for the next round of quiescent-state
    1832                 :            :  * forcing, that kthread will clean up after the just-completed grace
    1833                 :            :  * period.  Note that the caller must hold rnp->lock, which is released
    1834                 :            :  * before return.
    1835                 :            :  */
    1836                 :      67521 : static void rcu_report_qs_rsp(unsigned long flags)
    1837                 :            :         __releases(rcu_get_root()->lock)
    1838                 :            : {
    1839                 :      67521 :         raw_lockdep_assert_held_rcu_node(rcu_get_root());
    1840         [ -  + ]:      67521 :         WARN_ON_ONCE(!rcu_gp_in_progress());
    1841                 :      67521 :         WRITE_ONCE(rcu_state.gp_flags,
    1842                 :            :                    READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS);
    1843                 :      67521 :         raw_spin_unlock_irqrestore_rcu_node(rcu_get_root(), flags);
    1844                 :      67521 :         rcu_gp_kthread_wake();
    1845                 :      67521 : }
    1846                 :            : 
    1847                 :            : /*
    1848                 :            :  * Similar to rcu_report_qs_rdp(), for which it is a helper function.
    1849                 :            :  * Allows quiescent states for a group of CPUs to be reported at one go
    1850                 :            :  * to the specified rcu_node structure, though all the CPUs in the group
    1851                 :            :  * must be represented by the same rcu_node structure (which need not be a
    1852                 :            :  * leaf rcu_node structure, though it often will be).  The gps parameter
    1853                 :            :  * is the grace-period snapshot, which means that the quiescent states
    1854                 :            :  * are valid only if rnp->gp_seq is equal to gps.  That structure's lock
    1855                 :            :  * must be held upon entry, and it is released before return.
    1856                 :            :  *
    1857                 :            :  * As a special case, if mask is zero, the bit-already-cleared check is
    1858                 :            :  * disabled.  This allows propagating quiescent state due to resumed tasks
    1859                 :            :  * during grace-period initialization.
    1860                 :            :  */
    1861                 :      67521 : static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp,
    1862                 :            :                               unsigned long gps, unsigned long flags)
    1863                 :            :         __releases(rnp->lock)
    1864                 :            : {
    1865                 :      67521 :         unsigned long oldmask = 0;
    1866                 :      67521 :         struct rcu_node *rnp_c;
    1867                 :            : 
    1868                 :      67521 :         raw_lockdep_assert_held_rcu_node(rnp);
    1869                 :            : 
    1870                 :            :         /* Walk up the rcu_node hierarchy. */
    1871                 :      67521 :         for (;;) {
    1872   [ -  +  -  -  :      67521 :                 if ((!(rnp->qsmask & mask) && mask) || rnp->gp_seq != gps) {
                   -  + ]
    1873                 :            : 
    1874                 :            :                         /*
    1875                 :            :                          * Our bit has already been cleared, or the
    1876                 :            :                          * relevant grace period is already over, so done.
    1877                 :            :                          */
    1878                 :          0 :                         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    1879                 :          0 :                         return;
    1880                 :            :                 }
    1881         [ -  + ]:      67521 :                 WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */
    1882                 :      67521 :                 WARN_ON_ONCE(!rcu_is_leaf_node(rnp) &&
    1883                 :            :                              rcu_preempt_blocked_readers_cgp(rnp));
    1884                 :      67521 :                 rnp->qsmask &= ~mask;
    1885                 :      67521 :                 trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,
    1886                 :            :                                                  mask, rnp->qsmask, rnp->level,
    1887                 :            :                                                  rnp->grplo, rnp->grphi,
    1888                 :      67521 :                                                  !!rnp->gp_tasks);
    1889         [ -  + ]:      67521 :                 if (rnp->qsmask != 0 || rcu_preempt_blocked_readers_cgp(rnp)) {
    1890                 :            : 
    1891                 :            :                         /* Other bits still set at this level, so done. */
    1892                 :          0 :                         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    1893                 :          0 :                         return;
    1894                 :            :                 }
    1895                 :      67521 :                 rnp->completedqs = rnp->gp_seq;
    1896                 :      67521 :                 mask = rnp->grpmask;
    1897         [ -  + ]:      67521 :                 if (rnp->parent == NULL) {
    1898                 :            : 
    1899                 :            :                         /* No more levels.  Exit loop holding root lock. */
    1900                 :            : 
    1901                 :            :                         break;
    1902                 :            :                 }
    1903                 :          0 :                 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    1904                 :          0 :                 rnp_c = rnp;
    1905                 :          0 :                 rnp = rnp->parent;
    1906                 :          0 :                 raw_spin_lock_irqsave_rcu_node(rnp, flags);
    1907                 :          0 :                 oldmask = rnp_c->qsmask;
    1908                 :            :         }
    1909                 :            : 
    1910                 :            :         /*
    1911                 :            :          * Get here if we are the last CPU to pass through a quiescent
    1912                 :            :          * state for this grace period.  Invoke rcu_report_qs_rsp()
    1913                 :            :          * to clean up and start the next grace period if one is needed.
    1914                 :            :          */
    1915                 :      67521 :         rcu_report_qs_rsp(flags); /* releases rnp->lock. */
    1916                 :            : }
    1917                 :            : 
    1918                 :            : /*
    1919                 :            :  * Record a quiescent state for all tasks that were previously queued
    1920                 :            :  * on the specified rcu_node structure and that were blocking the current
    1921                 :            :  * RCU grace period.  The caller must hold the corresponding rnp->lock with
    1922                 :            :  * irqs disabled, and this lock is released upon return, but irqs remain
    1923                 :            :  * disabled.
    1924                 :            :  */
    1925                 :            : static void __maybe_unused
    1926                 :            : rcu_report_unblock_qs_rnp(struct rcu_node *rnp, unsigned long flags)
    1927                 :            :         __releases(rnp->lock)
    1928                 :            : {
    1929                 :            :         unsigned long gps;
    1930                 :            :         unsigned long mask;
    1931                 :            :         struct rcu_node *rnp_p;
    1932                 :            : 
    1933                 :            :         raw_lockdep_assert_held_rcu_node(rnp);
    1934                 :            :         if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT_RCU)) ||
    1935                 :            :             WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)) ||
    1936                 :            :             rnp->qsmask != 0) {
    1937                 :            :                 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    1938                 :            :                 return;  /* Still need more quiescent states! */
    1939                 :            :         }
    1940                 :            : 
    1941                 :            :         rnp->completedqs = rnp->gp_seq;
    1942                 :            :         rnp_p = rnp->parent;
    1943                 :            :         if (rnp_p == NULL) {
    1944                 :            :                 /*
    1945                 :            :                  * Only one rcu_node structure in the tree, so don't
    1946                 :            :                  * try to report up to its nonexistent parent!
    1947                 :            :                  */
    1948                 :            :                 rcu_report_qs_rsp(flags);
    1949                 :            :                 return;
    1950                 :            :         }
    1951                 :            : 
    1952                 :            :         /* Report up the rest of the hierarchy, tracking current ->gp_seq. */
    1953                 :            :         gps = rnp->gp_seq;
    1954                 :            :         mask = rnp->grpmask;
    1955                 :            :         raw_spin_unlock_rcu_node(rnp);  /* irqs remain disabled. */
    1956                 :            :         raw_spin_lock_rcu_node(rnp_p);  /* irqs already disabled. */
    1957                 :            :         rcu_report_qs_rnp(mask, rnp_p, gps, flags);
    1958                 :            : }
    1959                 :            : 
    1960                 :            : /*
    1961                 :            :  * Record a quiescent state for the specified CPU to that CPU's rcu_data
    1962                 :            :  * structure.  This must be called from the specified CPU.
    1963                 :            :  */
    1964                 :            : static void
    1965                 :            : rcu_report_qs_rdp(int cpu, struct rcu_data *rdp)
    1966                 :            : {
    1967                 :            :         unsigned long flags;
    1968                 :            :         unsigned long mask;
    1969                 :            :         bool needwake = false;
    1970                 :            :         const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
    1971                 :            :                                rcu_segcblist_is_offloaded(&rdp->cblist);
    1972                 :            :         struct rcu_node *rnp;
    1973                 :            : 
    1974                 :            :         rnp = rdp->mynode;
    1975                 :            :         raw_spin_lock_irqsave_rcu_node(rnp, flags);
    1976                 :            :         if (rdp->cpu_no_qs.b.norm || rdp->gp_seq != rnp->gp_seq ||
    1977                 :            :             rdp->gpwrap) {
    1978                 :            : 
    1979                 :            :                 /*
    1980                 :            :                  * The grace period in which this quiescent state was
    1981                 :            :                  * recorded has ended, so don't report it upwards.
    1982                 :            :                  * We will instead need a new quiescent state that lies
    1983                 :            :                  * within the current grace period.
    1984                 :            :                  */
    1985                 :            :                 rdp->cpu_no_qs.b.norm = true;        /* need qs for new gp. */
    1986                 :            :                 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    1987                 :            :                 return;
    1988                 :            :         }
    1989                 :            :         mask = rdp->grpmask;
    1990                 :            :         if ((rnp->qsmask & mask) == 0) {
    1991                 :            :                 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    1992                 :            :         } else {
    1993                 :            :                 /*
    1994                 :            :                  * This GP can't end until cpu checks in, so all of our
    1995                 :            :                  * callbacks can be processed during the next GP.
    1996                 :            :                  */
    1997                 :            :                 if (!offloaded)
    1998                 :            :                         needwake = rcu_accelerate_cbs(rnp, rdp);
    1999                 :            : 
    2000                 :            :                 rcu_disable_urgency_upon_qs(rdp);
    2001                 :            :                 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
    2002                 :            :                 /* ^^^ Released rnp->lock */
    2003                 :            :                 if (needwake)
    2004                 :            :                         rcu_gp_kthread_wake();
    2005                 :            :         }
    2006                 :            : }
    2007                 :            : 
    2008                 :            : /*
    2009                 :            :  * Check to see if there is a new grace period of which this CPU
    2010                 :            :  * is not yet aware, and if so, set up local rcu_data state for it.
    2011                 :            :  * Otherwise, see if this CPU has just passed through its first
    2012                 :            :  * quiescent state for this grace period, and record that fact if so.
    2013                 :            :  */
    2014                 :            : static void
    2015                 :     247986 : rcu_check_quiescent_state(struct rcu_data *rdp)
    2016                 :            : {
    2017                 :            :         /* Check for grace-period ends and beginnings. */
    2018                 :     247986 :         note_gp_changes(rdp);
    2019                 :            : 
    2020                 :            :         /*
    2021                 :            :          * Does this CPU still need to do its part for current grace period?
    2022                 :            :          * If no, return and let the other CPUs do their part as well.
    2023                 :            :          */
    2024         [ +  + ]:     247986 :         if (!rdp->core_needs_qs)
    2025                 :            :                 return;
    2026                 :            : 
    2027                 :            :         /*
    2028                 :            :          * Was there a quiescent state since the beginning of the grace
    2029                 :            :          * period? If no, then exit and wait for the next call.
    2030                 :            :          */
    2031         [ +  + ]:     247956 :         if (rdp->cpu_no_qs.b.norm)
    2032                 :            :                 return;
    2033                 :            : 
    2034                 :            :         /*
    2035                 :            :          * Tell RCU we are done (but rcu_report_qs_rdp() will be the
    2036                 :            :          * judge of that).
    2037                 :            :          */
    2038                 :     247954 :         rcu_report_qs_rdp(rdp->cpu, rdp);
    2039                 :            : }
    2040                 :            : 
    2041                 :            : /*
    2042                 :            :  * Near the end of the offline process.  Trace the fact that this CPU
    2043                 :            :  * is going offline.
    2044                 :            :  */
    2045                 :          0 : int rcutree_dying_cpu(unsigned int cpu)
    2046                 :            : {
    2047                 :          0 :         bool blkd;
    2048                 :          0 :         struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
    2049                 :          0 :         struct rcu_node *rnp = rdp->mynode;
    2050                 :            : 
    2051                 :          0 :         if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
    2052                 :            :                 return 0;
    2053                 :            : 
    2054                 :          0 :         blkd = !!(rnp->qsmask & rdp->grpmask);
    2055         [ #  # ]:          0 :         trace_rcu_grace_period(rcu_state.name, rnp->gp_seq,
    2056                 :          0 :                                blkd ? TPS("cpuofl") : TPS("cpuofl-bgp"));
    2057                 :          0 :         return 0;
    2058                 :            : }
    2059                 :            : 
    2060                 :            : /*
    2061                 :            :  * All CPUs for the specified rcu_node structure have gone offline,
    2062                 :            :  * and all tasks that were preempted within an RCU read-side critical
    2063                 :            :  * section while running on one of those CPUs have since exited their RCU
    2064                 :            :  * read-side critical section.  Some other CPU is reporting this fact with
    2065                 :            :  * the specified rcu_node structure's ->lock held and interrupts disabled.
    2066                 :            :  * This function therefore goes up the tree of rcu_node structures,
    2067                 :            :  * clearing the corresponding bits in the ->qsmaskinit fields.  Note that
    2068                 :            :  * the leaf rcu_node structure's ->qsmaskinit field has already been
    2069                 :            :  * updated.
    2070                 :            :  *
    2071                 :            :  * This function does check that the specified rcu_node structure has
    2072                 :            :  * all CPUs offline and no blocked tasks, so it is OK to invoke it
    2073                 :            :  * prematurely.  That said, invoking it after the fact will cost you
    2074                 :            :  * a needless lock acquisition.  So once it has done its work, don't
    2075                 :            :  * invoke it again.
    2076                 :            :  */
    2077                 :          0 : static void rcu_cleanup_dead_rnp(struct rcu_node *rnp_leaf)
    2078                 :            : {
    2079                 :          0 :         long mask;
    2080                 :          0 :         struct rcu_node *rnp = rnp_leaf;
    2081                 :            : 
    2082                 :          0 :         raw_lockdep_assert_held_rcu_node(rnp_leaf);
    2083                 :          0 :         if (!IS_ENABLED(CONFIG_HOTPLUG_CPU) ||
    2084   [ #  #  #  # ]:          0 :             WARN_ON_ONCE(rnp_leaf->qsmaskinit) ||
    2085                 :            :             WARN_ON_ONCE(rcu_preempt_has_tasks(rnp_leaf)))
    2086                 :            :                 return;
    2087                 :          0 :         for (;;) {
    2088                 :          0 :                 mask = rnp->grpmask;
    2089                 :          0 :                 rnp = rnp->parent;
    2090         [ #  # ]:          0 :                 if (!rnp)
    2091                 :            :                         break;
    2092                 :          0 :                 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
    2093                 :          0 :                 rnp->qsmaskinit &= ~mask;
    2094                 :            :                 /* Between grace periods, so better already be zero! */
    2095         [ #  # ]:          0 :                 WARN_ON_ONCE(rnp->qsmask);
    2096         [ #  # ]:          0 :                 if (rnp->qsmaskinit) {
    2097                 :          0 :                         raw_spin_unlock_rcu_node(rnp);
    2098                 :            :                         /* irqs remain disabled. */
    2099                 :          0 :                         return;
    2100                 :            :                 }
    2101                 :          0 :                 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
    2102                 :            :         }
    2103                 :            : }
    2104                 :            : 
    2105                 :            : /*
    2106                 :            :  * The CPU has been completely removed, and some other CPU is reporting
    2107                 :            :  * this fact from process context.  Do the remainder of the cleanup.
    2108                 :            :  * There can only be one CPU hotplug operation at a time, so no need for
    2109                 :            :  * explicit locking.
    2110                 :            :  */
    2111                 :          0 : int rcutree_dead_cpu(unsigned int cpu)
    2112                 :            : {
    2113                 :          0 :         struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
    2114                 :          0 :         struct rcu_node *rnp = rdp->mynode;  /* Outgoing CPU's rdp & rnp. */
    2115                 :            : 
    2116                 :          0 :         if (!IS_ENABLED(CONFIG_HOTPLUG_CPU))
    2117                 :            :                 return 0;
    2118                 :            : 
    2119                 :            :         /* Adjust any no-longer-needed kthreads. */
    2120                 :          0 :         rcu_boost_kthread_setaffinity(rnp, -1);
    2121                 :            :         /* Do any needed no-CB deferred wakeups from this CPU. */
    2122                 :          0 :         do_nocb_deferred_wakeup(per_cpu_ptr(&rcu_data, cpu));
    2123                 :            : 
    2124                 :            :         // Stop-machine done, so allow nohz_full to disable tick.
    2125                 :          0 :         tick_dep_clear(TICK_DEP_BIT_RCU);
    2126                 :          0 :         return 0;
    2127                 :            : }
    2128                 :            : 
    2129                 :            : /*
    2130                 :            :  * Invoke any RCU callbacks that have made it to the end of their grace
    2131                 :            :  * period.  Thottle as specified by rdp->blimit.
    2132                 :            :  */
    2133                 :     238839 : static void rcu_do_batch(struct rcu_data *rdp)
    2134                 :            : {
    2135                 :     238839 :         unsigned long flags;
    2136                 :     238839 :         const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
    2137                 :            :                                rcu_segcblist_is_offloaded(&rdp->cblist);
    2138                 :     238839 :         struct rcu_head *rhp;
    2139                 :     238839 :         struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl);
    2140                 :     238839 :         long bl, count;
    2141                 :     238839 :         long pending, tlimit = 0;
    2142                 :            : 
    2143                 :            :         /* If no callbacks are ready, just return. */
    2144         [ -  + ]:     238839 :         if (!rcu_segcblist_ready_cbs(&rdp->cblist)) {
    2145                 :          0 :                 trace_rcu_batch_start(rcu_state.name,
    2146                 :            :                                       rcu_segcblist_n_cbs(&rdp->cblist), 0);
    2147                 :          0 :                 trace_rcu_batch_end(rcu_state.name, 0,
    2148                 :          0 :                                     !rcu_segcblist_empty(&rdp->cblist),
    2149                 :            :                                     need_resched(), is_idle_task(current),
    2150                 :            :                                     rcu_is_callbacks_kthread());
    2151                 :          0 :                 return;
    2152                 :            :         }
    2153                 :            : 
    2154                 :            :         /*
    2155                 :            :          * Extract the list of ready callbacks, disabling to prevent
    2156                 :            :          * races with call_rcu() from interrupt handlers.  Leave the
    2157                 :            :          * callback counts, as rcu_barrier() needs to be conservative.
    2158                 :            :          */
    2159                 :     238839 :         local_irq_save(flags);
    2160                 :     238839 :         rcu_nocb_lock(rdp);
    2161         [ -  + ]:     238839 :         WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
    2162         [ -  + ]:     238839 :         pending = rcu_segcblist_n_cbs(&rdp->cblist);
    2163                 :     238839 :         bl = max(rdp->blimit, pending >> rcu_divisor);
    2164         [ -  + ]:     238839 :         if (unlikely(bl > 100))
    2165                 :          0 :                 tlimit = local_clock() + rcu_resched_ns;
    2166                 :     238839 :         trace_rcu_batch_start(rcu_state.name,
    2167                 :            :                               rcu_segcblist_n_cbs(&rdp->cblist), bl);
    2168                 :     238839 :         rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl);
    2169                 :     238839 :         if (offloaded)
    2170                 :            :                 rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
    2171                 :     238839 :         rcu_nocb_unlock_irqrestore(rdp, flags);
    2172                 :            : 
    2173                 :            :         /* Invoke callbacks. */
    2174                 :     238839 :         tick_dep_set_task(current, TICK_DEP_BIT_RCU);
    2175                 :     238839 :         rhp = rcu_cblist_dequeue(&rcl);
    2176         [ +  + ]:    2490737 :         for (; rhp; rhp = rcu_cblist_dequeue(&rcl)) {
    2177                 :    2218281 :                 rcu_callback_t f;
    2178                 :            : 
    2179                 :    2218281 :                 debug_rcu_head_unqueue(rhp);
    2180                 :            : 
    2181                 :    2218281 :                 rcu_lock_acquire(&rcu_callback_map);
    2182                 :    2218281 :                 trace_rcu_invoke_callback(rcu_state.name, rhp);
    2183                 :            : 
    2184                 :    2218281 :                 f = rhp->func;
    2185                 :    2218281 :                 WRITE_ONCE(rhp->func, (rcu_callback_t)0L);
    2186                 :    2218281 :                 f(rhp);
    2187                 :            : 
    2188                 :    2218281 :                 rcu_lock_release(&rcu_callback_map);
    2189                 :            : 
    2190                 :            :                 /*
    2191                 :            :                  * Stop only if limit reached and CPU has something to do.
    2192                 :            :                  * Note: The rcl structure counts down from zero.
    2193                 :            :                  */
    2194   [ +  +  +  + ]:    2423503 :                 if (-rcl.len >= bl && !offloaded &&
    2195         [ -  + ]:     179196 :                     (need_resched() ||
    2196                 :            :                      (!is_idle_task(current) && !rcu_is_callbacks_kthread())))
    2197                 :            :                         break;
    2198         [ -  + ]:    2013059 :                 if (unlikely(tlimit)) {
    2199                 :            :                         /* only call local_clock() every 32 callbacks */
    2200   [ #  #  #  # ]:          0 :                         if (likely((-rcl.len & 31) || local_clock() < tlimit))
    2201                 :          0 :                                 continue;
    2202                 :            :                         /* Exceeded the time limit, so leave. */
    2203                 :            :                         break;
    2204                 :            :                 }
    2205                 :            :                 if (offloaded) {
    2206                 :            :                         WARN_ON_ONCE(in_serving_softirq());
    2207                 :            :                         local_bh_enable();
    2208                 :            :                         lockdep_assert_irqs_enabled();
    2209                 :            :                         cond_resched_tasks_rcu_qs();
    2210                 :            :                         lockdep_assert_irqs_enabled();
    2211                 :            :                         local_bh_disable();
    2212                 :            :                 }
    2213                 :            :         }
    2214                 :            : 
    2215                 :     238839 :         local_irq_save(flags);
    2216                 :     238839 :         rcu_nocb_lock(rdp);
    2217                 :     238839 :         count = -rcl.len;
    2218                 :     238839 :         trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(),
    2219                 :            :                             is_idle_task(current), rcu_is_callbacks_kthread());
    2220                 :            : 
    2221                 :            :         /* Update counts and requeue any remaining callbacks. */
    2222                 :     238839 :         rcu_segcblist_insert_done_cbs(&rdp->cblist, &rcl);
    2223                 :     238839 :         smp_mb(); /* List handling before counting for rcu_barrier(). */
    2224                 :     238839 :         rcu_segcblist_insert_count(&rdp->cblist, &rcl);
    2225                 :            : 
    2226                 :            :         /* Reinstate batch limit if we have worked down the excess. */
    2227         [ -  + ]:     238839 :         count = rcu_segcblist_n_cbs(&rdp->cblist);
    2228   [ -  +  -  - ]:     238839 :         if (rdp->blimit >= DEFAULT_MAX_RCU_BLIMIT && count <= qlowmark)
    2229                 :          0 :                 rdp->blimit = blimit;
    2230                 :            : 
    2231                 :            :         /* Reset ->qlen_last_fqs_check trigger if enough CBs have drained. */
    2232   [ +  +  -  + ]:     238839 :         if (count == 0 && rdp->qlen_last_fqs_check != 0) {
    2233                 :          0 :                 rdp->qlen_last_fqs_check = 0;
    2234                 :          0 :                 rdp->n_force_qs_snap = rcu_state.n_force_qs;
    2235         [ -  + ]:     238839 :         } else if (count < rdp->qlen_last_fqs_check - qhimark)
    2236                 :          0 :                 rdp->qlen_last_fqs_check = count;
    2237                 :            : 
    2238                 :            :         /*
    2239                 :            :          * The following usually indicates a double call_rcu().  To track
    2240                 :            :          * this down, try building with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y.
    2241                 :            :          */
    2242   [ +  +  +  -  :     241207 :         WARN_ON_ONCE(count == 0 && !rcu_segcblist_empty(&rdp->cblist));
                   -  + ]
    2243   [ +  +  +  -  :     475310 :         WARN_ON_ONCE(!IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
                   -  + ]
    2244                 :            :                      count != 0 && rcu_segcblist_empty(&rdp->cblist));
    2245                 :            : 
    2246                 :     238839 :         rcu_nocb_unlock_irqrestore(rdp, flags);
    2247                 :            : 
    2248                 :            :         /* Re-invoke RCU core processing if there are callbacks remaining. */
    2249         [ +  + ]:     238839 :         if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist))
    2250                 :     201893 :                 invoke_rcu_core();
    2251                 :     238839 :         tick_dep_clear_task(current, TICK_DEP_BIT_RCU);
    2252                 :            : }
    2253                 :            : 
    2254                 :            : /*
    2255                 :            :  * This function is invoked from each scheduling-clock interrupt,
    2256                 :            :  * and checks to see if this CPU is in a non-context-switch quiescent
    2257                 :            :  * state, for example, user mode or idle loop.  It also schedules RCU
    2258                 :            :  * core processing.  If the current grace period has gone on too long,
    2259                 :            :  * it will ask the scheduler to manufacture a context switch for the sole
    2260                 :            :  * purpose of providing a providing the needed quiescent state.
    2261                 :            :  */
    2262                 :      97200 : void rcu_sched_clock_irq(int user)
    2263                 :            : {
    2264                 :      97200 :         trace_rcu_utilization(TPS("Start scheduler-tick"));
    2265                 :      97200 :         raw_cpu_inc(rcu_data.ticks_this_gp);
    2266                 :            :         /* The load-acquire pairs with the store-release setting to true. */
    2267         [ +  - ]:      97200 :         if (smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) {
    2268                 :            :                 /* Idle and userspace execution already are quiescent states. */
    2269   [ #  #  #  #  :          0 :                 if (!rcu_is_cpu_rrupt_from_idle() && !user) {
                   #  # ]
    2270                 :          0 :                         set_tsk_need_resched(current);
    2271                 :          0 :                         set_preempt_need_resched();
    2272                 :            :                 }
    2273                 :      97200 :                 __this_cpu_write(rcu_data.rcu_urgent_qs, false);
    2274                 :            :         }
    2275                 :      97200 :         rcu_flavor_sched_clock_irq(user);
    2276         [ +  + ]:      97200 :         if (rcu_pending(user))
    2277                 :      59722 :                 invoke_rcu_core();
    2278                 :            : 
    2279                 :      97200 :         trace_rcu_utilization(TPS("End scheduler-tick"));
    2280                 :      97200 : }
    2281                 :            : 
    2282                 :            : /*
    2283                 :            :  * Scan the leaf rcu_node structures.  For each structure on which all
    2284                 :            :  * CPUs have reported a quiescent state and on which there are tasks
    2285                 :            :  * blocking the current grace period, initiate RCU priority boosting.
    2286                 :            :  * Otherwise, invoke the specified function to check dyntick state for
    2287                 :            :  * each CPU that has not yet reported a quiescent state.
    2288                 :            :  */
    2289                 :          0 : static void force_qs_rnp(int (*f)(struct rcu_data *rdp))
    2290                 :            : {
    2291                 :          0 :         int cpu;
    2292                 :          0 :         unsigned long flags;
    2293                 :          0 :         unsigned long mask;
    2294                 :          0 :         struct rcu_data *rdp;
    2295                 :          0 :         struct rcu_node *rnp;
    2296                 :            : 
    2297         [ #  # ]:          0 :         rcu_for_each_leaf_node(rnp) {
    2298                 :          0 :                 cond_resched_tasks_rcu_qs();
    2299                 :          0 :                 mask = 0;
    2300                 :          0 :                 raw_spin_lock_irqsave_rcu_node(rnp, flags);
    2301         [ #  # ]:          0 :                 if (rnp->qsmask == 0) {
    2302                 :          0 :                         if (!IS_ENABLED(CONFIG_PREEMPT_RCU) ||
    2303                 :            :                             rcu_preempt_blocked_readers_cgp(rnp)) {
    2304                 :            :                                 /*
    2305                 :            :                                  * No point in scanning bits because they
    2306                 :            :                                  * are all zero.  But we might need to
    2307                 :            :                                  * priority-boost blocked readers.
    2308                 :            :                                  */
    2309                 :          0 :                                 rcu_initiate_boost(rnp, flags);
    2310                 :            :                                 /* rcu_initiate_boost() releases rnp->lock */
    2311                 :          0 :                                 continue;
    2312                 :            :                         }
    2313                 :            :                         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    2314                 :            :                         continue;
    2315                 :            :                 }
    2316         [ #  # ]:          0 :                 for_each_leaf_node_cpu_mask(rnp, cpu, rnp->qsmask) {
    2317                 :          0 :                         rdp = per_cpu_ptr(&rcu_data, cpu);
    2318         [ #  # ]:          0 :                         if (f(rdp)) {
    2319                 :          0 :                                 mask |= rdp->grpmask;
    2320                 :          0 :                                 rcu_disable_urgency_upon_qs(rdp);
    2321                 :            :                         }
    2322                 :            :                 }
    2323         [ #  # ]:          0 :                 if (mask != 0) {
    2324                 :            :                         /* Idle/offline CPUs, report (releases rnp->lock). */
    2325                 :          0 :                         rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
    2326                 :            :                 } else {
    2327                 :            :                         /* Nothing to do here, so just drop the lock. */
    2328                 :          0 :                         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    2329                 :            :                 }
    2330                 :            :         }
    2331                 :          0 : }
    2332                 :            : 
    2333                 :            : /*
    2334                 :            :  * Force quiescent states on reluctant CPUs, and also detect which
    2335                 :            :  * CPUs are in dyntick-idle mode.
    2336                 :            :  */
    2337                 :          0 : void rcu_force_quiescent_state(void)
    2338                 :            : {
    2339                 :          0 :         unsigned long flags;
    2340                 :          0 :         bool ret;
    2341                 :          0 :         struct rcu_node *rnp;
    2342                 :          0 :         struct rcu_node *rnp_old = NULL;
    2343                 :            : 
    2344                 :            :         /* Funnel through hierarchy to reduce memory contention. */
    2345                 :          0 :         rnp = __this_cpu_read(rcu_data.mynode);
    2346         [ #  # ]:          0 :         for (; rnp != NULL; rnp = rnp->parent) {
    2347   [ #  #  #  # ]:          0 :                 ret = (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) ||
    2348                 :          0 :                        !raw_spin_trylock(&rnp->fqslock);
    2349         [ #  # ]:          0 :                 if (rnp_old != NULL)
    2350                 :          0 :                         raw_spin_unlock(&rnp_old->fqslock);
    2351         [ #  # ]:          0 :                 if (ret)
    2352                 :            :                         return;
    2353                 :          0 :                 rnp_old = rnp;
    2354                 :            :         }
    2355                 :            :         /* rnp_old == rcu_get_root(), rnp == NULL. */
    2356                 :            : 
    2357                 :            :         /* Reached the root of the rcu_node tree, acquire lock. */
    2358                 :          0 :         raw_spin_lock_irqsave_rcu_node(rnp_old, flags);
    2359                 :          0 :         raw_spin_unlock(&rnp_old->fqslock);
    2360         [ #  # ]:          0 :         if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {
    2361                 :          0 :                 raw_spin_unlock_irqrestore_rcu_node(rnp_old, flags);
    2362                 :          0 :                 return;  /* Someone beat us to it. */
    2363                 :            :         }
    2364                 :          0 :         WRITE_ONCE(rcu_state.gp_flags,
    2365                 :            :                    READ_ONCE(rcu_state.gp_flags) | RCU_GP_FLAG_FQS);
    2366                 :          0 :         raw_spin_unlock_irqrestore_rcu_node(rnp_old, flags);
    2367                 :          0 :         rcu_gp_kthread_wake();
    2368                 :            : }
    2369                 :            : EXPORT_SYMBOL_GPL(rcu_force_quiescent_state);
    2370                 :            : 
    2371                 :            : /* Perform RCU core processing work for the current CPU.  */
    2372                 :     247986 : static __latent_entropy void rcu_core(void)
    2373                 :            : {
    2374                 :     247986 :         unsigned long flags;
    2375                 :     247986 :         struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
    2376                 :     247986 :         struct rcu_node *rnp = rdp->mynode;
    2377                 :     247986 :         const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
    2378                 :            :                                rcu_segcblist_is_offloaded(&rdp->cblist);
    2379                 :            : 
    2380         [ +  - ]:     247986 :         if (cpu_is_offline(smp_processor_id()))
    2381                 :            :                 return;
    2382                 :     247986 :         trace_rcu_utilization(TPS("Start RCU core"));
    2383         [ -  + ]:     247986 :         WARN_ON_ONCE(!rdp->beenonline);
    2384                 :            : 
    2385                 :            :         /* Report any deferred quiescent states if preemption enabled. */
    2386         [ +  + ]:     247986 :         if (!(preempt_count() & PREEMPT_MASK)) {
    2387                 :     247934 :                 rcu_preempt_deferred_qs(current);
    2388                 :         52 :         } else if (rcu_preempt_need_deferred_qs(current)) {
    2389                 :            :                 set_tsk_need_resched(current);
    2390                 :            :                 set_preempt_need_resched();
    2391                 :            :         }
    2392                 :            : 
    2393                 :            :         /* Update RCU state based on any recent quiescent states. */
    2394                 :     247986 :         rcu_check_quiescent_state(rdp);
    2395                 :            : 
    2396                 :            :         /* No grace period and unregistered callbacks? */
    2397   [ +  +  +  - ]:     247986 :         if (!rcu_gp_in_progress() &&
    2398         [ +  - ]:       6956 :             rcu_segcblist_is_enabled(&rdp->cblist) && !offloaded) {
    2399                 :       6956 :                 local_irq_save(flags);
    2400         [ +  + ]:       6956 :                 if (!rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
    2401                 :       2485 :                         rcu_accelerate_cbs_unlocked(rnp, rdp);
    2402                 :       6956 :                 local_irq_restore(flags);
    2403                 :            :         }
    2404                 :            : 
    2405         [ -  + ]:     247986 :         rcu_check_gp_start_stall(rnp, rdp, rcu_jiffies_till_stall_check());
    2406                 :            : 
    2407                 :            :         /* If there are callbacks ready, invoke them. */
    2408   [ +  +  +  - ]:     247986 :         if (!offloaded && rcu_segcblist_ready_cbs(&rdp->cblist) &&
    2409         [ +  - ]:     238839 :             likely(READ_ONCE(rcu_scheduler_fully_active)))
    2410                 :     238839 :                 rcu_do_batch(rdp);
    2411                 :            : 
    2412                 :            :         /* Do any needed deferred wakeups of rcuo kthreads. */
    2413                 :     247986 :         do_nocb_deferred_wakeup(rdp);
    2414                 :     247986 :         trace_rcu_utilization(TPS("End RCU core"));
    2415                 :            : }
    2416                 :            : 
    2417                 :     247986 : static void rcu_core_si(struct softirq_action *h)
    2418                 :            : {
    2419                 :     247986 :         rcu_core();
    2420                 :     247986 : }
    2421                 :            : 
    2422                 :          0 : static void rcu_wake_cond(struct task_struct *t, int status)
    2423                 :            : {
    2424                 :            :         /*
    2425                 :            :          * If the thread is yielding, only wake it when this
    2426                 :            :          * is invoked from idle
    2427                 :            :          */
    2428   [ #  #  #  #  :          0 :         if (t && (status != RCU_KTHREAD_YIELDING || is_idle_task(current)))
                   #  # ]
    2429                 :          0 :                 wake_up_process(t);
    2430                 :          0 : }
    2431                 :            : 
    2432                 :          0 : static void invoke_rcu_core_kthread(void)
    2433                 :            : {
    2434                 :          0 :         struct task_struct *t;
    2435                 :          0 :         unsigned long flags;
    2436                 :            : 
    2437                 :          0 :         local_irq_save(flags);
    2438         [ #  # ]:          0 :         __this_cpu_write(rcu_data.rcu_cpu_has_work, 1);
    2439                 :          0 :         t = __this_cpu_read(rcu_data.rcu_cpu_kthread_task);
    2440   [ #  #  #  # ]:          0 :         if (t != NULL && t != current)
    2441                 :          0 :                 rcu_wake_cond(t, __this_cpu_read(rcu_data.rcu_cpu_kthread_status));
    2442                 :          0 :         local_irq_restore(flags);
    2443                 :          0 : }
    2444                 :            : 
    2445                 :            : /*
    2446                 :            :  * Wake up this CPU's rcuc kthread to do RCU core processing.
    2447                 :            :  */
    2448                 :     261615 : static void invoke_rcu_core(void)
    2449                 :            : {
    2450         [ +  - ]:     261615 :         if (!cpu_online(smp_processor_id()))
    2451                 :            :                 return;
    2452         [ +  - ]:     261615 :         if (use_softirq)
    2453                 :     261615 :                 raise_softirq(RCU_SOFTIRQ);
    2454                 :            :         else
    2455                 :          0 :                 invoke_rcu_core_kthread();
    2456                 :            : }
    2457                 :            : 
    2458                 :          0 : static void rcu_cpu_kthread_park(unsigned int cpu)
    2459                 :            : {
    2460                 :          0 :         per_cpu(rcu_data.rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU;
    2461                 :          0 : }
    2462                 :            : 
    2463                 :          0 : static int rcu_cpu_kthread_should_run(unsigned int cpu)
    2464                 :            : {
    2465                 :          0 :         return __this_cpu_read(rcu_data.rcu_cpu_has_work);
    2466                 :            : }
    2467                 :            : 
    2468                 :            : /*
    2469                 :            :  * Per-CPU kernel thread that invokes RCU callbacks.  This replaces
    2470                 :            :  * the RCU softirq used in configurations of RCU that do not support RCU
    2471                 :            :  * priority boosting.
    2472                 :            :  */
    2473                 :          0 : static void rcu_cpu_kthread(unsigned int cpu)
    2474                 :            : {
    2475                 :          0 :         unsigned int *statusp = this_cpu_ptr(&rcu_data.rcu_cpu_kthread_status);
    2476                 :          0 :         char work, *workp = this_cpu_ptr(&rcu_data.rcu_cpu_has_work);
    2477                 :          0 :         int spincnt;
    2478                 :            : 
    2479                 :          0 :         trace_rcu_utilization(TPS("Start CPU kthread@rcu_run"));
    2480         [ #  # ]:          0 :         for (spincnt = 0; spincnt < 10; spincnt++) {
    2481                 :          0 :                 local_bh_disable();
    2482                 :          0 :                 *statusp = RCU_KTHREAD_RUNNING;
    2483                 :          0 :                 local_irq_disable();
    2484                 :          0 :                 work = *workp;
    2485                 :          0 :                 *workp = 0;
    2486                 :          0 :                 local_irq_enable();
    2487         [ #  # ]:          0 :                 if (work)
    2488                 :          0 :                         rcu_core();
    2489                 :          0 :                 local_bh_enable();
    2490         [ #  # ]:          0 :                 if (*workp == 0) {
    2491                 :          0 :                         trace_rcu_utilization(TPS("End CPU kthread@rcu_wait"));
    2492                 :          0 :                         *statusp = RCU_KTHREAD_WAITING;
    2493                 :          0 :                         return;
    2494                 :            :                 }
    2495                 :            :         }
    2496                 :          0 :         *statusp = RCU_KTHREAD_YIELDING;
    2497                 :          0 :         trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield"));
    2498                 :          0 :         schedule_timeout_interruptible(2);
    2499                 :          0 :         trace_rcu_utilization(TPS("End CPU kthread@rcu_yield"));
    2500                 :          0 :         *statusp = RCU_KTHREAD_WAITING;
    2501                 :            : }
    2502                 :            : 
    2503                 :            : static struct smp_hotplug_thread rcu_cpu_thread_spec = {
    2504                 :            :         .store                  = &rcu_data.rcu_cpu_kthread_task,
    2505                 :            :         .thread_should_run      = rcu_cpu_kthread_should_run,
    2506                 :            :         .thread_fn              = rcu_cpu_kthread,
    2507                 :            :         .thread_comm            = "rcuc/%u",
    2508                 :            :         .setup                  = rcu_cpu_kthread_setup,
    2509                 :            :         .park                   = rcu_cpu_kthread_park,
    2510                 :            : };
    2511                 :            : 
    2512                 :            : /*
    2513                 :            :  * Spawn per-CPU RCU core processing kthreads.
    2514                 :            :  */
    2515                 :         30 : static int __init rcu_spawn_core_kthreads(void)
    2516                 :            : {
    2517                 :         30 :         int cpu;
    2518                 :            : 
    2519         [ +  + ]:         60 :         for_each_possible_cpu(cpu)
    2520                 :         30 :                 per_cpu(rcu_data.rcu_cpu_has_work, cpu) = 0;
    2521         [ -  + ]:         30 :         if (!IS_ENABLED(CONFIG_RCU_BOOST) && use_softirq)
    2522                 :            :                 return 0;
    2523   [ #  #  #  # ]:          0 :         WARN_ONCE(smpboot_register_percpu_thread(&rcu_cpu_thread_spec),
    2524                 :            :                   "%s: Could not start rcuc kthread, OOM is now expected behavior\n", __func__);
    2525                 :            :         return 0;
    2526                 :            : }
    2527                 :            : early_initcall(rcu_spawn_core_kthreads);
    2528                 :            : 
    2529                 :            : /*
    2530                 :            :  * Handle any core-RCU processing required by a call_rcu() invocation.
    2531                 :            :  */
    2532                 :    2219838 : static void __call_rcu_core(struct rcu_data *rdp, struct rcu_head *head,
    2533                 :            :                             unsigned long flags)
    2534                 :            : {
    2535                 :            :         /*
    2536                 :            :          * If called from an extended quiescent state, invoke the RCU
    2537                 :            :          * core in order to force a re-evaluation of RCU's idleness.
    2538                 :            :          */
    2539         [ -  + ]:    4439676 :         if (!rcu_is_watching())
    2540                 :          0 :                 invoke_rcu_core();
    2541                 :            : 
    2542                 :            :         /* If interrupts were disabled or CPU offline, don't invoke RCU core. */
    2543   [ +  +  -  + ]:    2219838 :         if (irqs_disabled_flags(flags) || cpu_is_offline(smp_processor_id()))
    2544                 :     241339 :                 return;
    2545                 :            : 
    2546                 :            :         /*
    2547                 :            :          * Force the grace period if too many callbacks or too long waiting.
    2548                 :            :          * Enforce hysteresis, and don't invoke rcu_force_quiescent_state()
    2549                 :            :          * if some other CPU has recently done so.  Also, don't bother
    2550                 :            :          * invoking rcu_force_quiescent_state() if the newly enqueued callback
    2551                 :            :          * is the only one waiting for a grace period to complete.
    2552                 :            :          */
    2553         [ -  + ]:    1978499 :         if (unlikely(rcu_segcblist_n_cbs(&rdp->cblist) >
    2554                 :            :                      rdp->qlen_last_fqs_check + qhimark)) {
    2555                 :            : 
    2556                 :            :                 /* Are we ignoring a completed grace period? */
    2557                 :          0 :                 note_gp_changes(rdp);
    2558                 :            : 
    2559                 :            :                 /* Start a new grace period if one not already started. */
    2560         [ #  # ]:          0 :                 if (!rcu_gp_in_progress()) {
    2561                 :          0 :                         rcu_accelerate_cbs_unlocked(rdp->mynode, rdp);
    2562                 :            :                 } else {
    2563                 :            :                         /* Give the grace period a kick. */
    2564                 :          0 :                         rdp->blimit = DEFAULT_MAX_RCU_BLIMIT;
    2565   [ #  #  #  # ]:          0 :                         if (rcu_state.n_force_qs == rdp->n_force_qs_snap &&
    2566                 :          0 :                             rcu_segcblist_first_pend_cb(&rdp->cblist) != head)
    2567                 :          0 :                                 rcu_force_quiescent_state();
    2568                 :          0 :                         rdp->n_force_qs_snap = rcu_state.n_force_qs;
    2569                 :          0 :                         rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist);
    2570                 :            :                 }
    2571                 :            :         }
    2572                 :            : }
    2573                 :            : 
    2574                 :            : /*
    2575                 :            :  * RCU callback function to leak a callback.
    2576                 :            :  */
    2577                 :            : static void rcu_leak_callback(struct rcu_head *rhp)
    2578                 :            : {
    2579                 :            : }
    2580                 :            : 
    2581                 :            : /*
    2582                 :            :  * Helper function for call_rcu() and friends.  The cpu argument will
    2583                 :            :  * normally be -1, indicating "currently running CPU".  It may specify
    2584                 :            :  * a CPU only if that CPU is a no-CBs CPU.  Currently, only rcu_barrier()
    2585                 :            :  * is expected to specify a CPU.
    2586                 :            :  */
    2587                 :            : static void
    2588                 :    2219838 : __call_rcu(struct rcu_head *head, rcu_callback_t func)
    2589                 :            : {
    2590                 :    2219838 :         unsigned long flags;
    2591                 :    2219838 :         struct rcu_data *rdp;
    2592                 :    2219838 :         bool was_alldone;
    2593                 :            : 
    2594                 :            :         /* Misaligned rcu_head! */
    2595         [ -  + ]:    2219838 :         WARN_ON_ONCE((unsigned long)head & (sizeof(void *) - 1));
    2596                 :            : 
    2597                 :    2219838 :         if (debug_rcu_head_queue(head)) {
    2598                 :            :                 /*
    2599                 :            :                  * Probable double call_rcu(), so leak the callback.
    2600                 :            :                  * Use rcu:rcu_callback trace event to find the previous
    2601                 :            :                  * time callback was passed to __call_rcu().
    2602                 :            :                  */
    2603                 :            :                 WARN_ONCE(1, "__call_rcu(): Double-freed CB %p->%pS()!!!\n",
    2604                 :            :                           head, head->func);
    2605                 :            :                 WRITE_ONCE(head->func, rcu_leak_callback);
    2606                 :            :                 return;
    2607                 :            :         }
    2608                 :    2219838 :         head->func = func;
    2609                 :    2219838 :         head->next = NULL;
    2610                 :    2219838 :         local_irq_save(flags);
    2611                 :    2219838 :         rdp = this_cpu_ptr(&rcu_data);
    2612                 :            : 
    2613                 :            :         /* Add the callback to our list. */
    2614         [ -  + ]:    2219838 :         if (unlikely(!rcu_segcblist_is_enabled(&rdp->cblist))) {
    2615                 :            :                 // This can trigger due to call_rcu() from offline CPU:
    2616         [ #  # ]:          0 :                 WARN_ON_ONCE(rcu_scheduler_active != RCU_SCHEDULER_INACTIVE);
    2617         [ #  # ]:          0 :                 WARN_ON_ONCE(!rcu_is_watching());
    2618                 :            :                 // Very early boot, before rcu_init().  Initialize if needed
    2619                 :            :                 // and then drop through to queue the callback.
    2620         [ #  # ]:          0 :                 if (rcu_segcblist_empty(&rdp->cblist))
    2621                 :          0 :                         rcu_segcblist_init(&rdp->cblist);
    2622                 :            :         }
    2623                 :            : 
    2624                 :    2219838 :         if (rcu_nocb_try_bypass(rdp, head, &was_alldone, flags))
    2625                 :            :                 return; // Enqueued onto ->nocb_bypass, so just leave.
    2626                 :            :         /* If we get here, rcu_nocb_try_bypass() acquired ->nocb_lock. */
    2627                 :    2219838 :         rcu_segcblist_enqueue(&rdp->cblist, head);
    2628         [ -  + ]:    2219838 :         if (__is_kfree_rcu_offset((unsigned long)func))
    2629                 :          0 :                 trace_rcu_kfree_callback(rcu_state.name, head,
    2630                 :            :                                          (unsigned long)func,
    2631                 :            :                                          rcu_segcblist_n_cbs(&rdp->cblist));
    2632                 :            :         else
    2633                 :    2219838 :                 trace_rcu_callback(rcu_state.name, head,
    2634                 :            :                                    rcu_segcblist_n_cbs(&rdp->cblist));
    2635                 :            : 
    2636                 :            :         /* Go handle any RCU core processing required. */
    2637                 :    2219838 :         if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
    2638                 :            :             unlikely(rcu_segcblist_is_offloaded(&rdp->cblist))) {
    2639                 :            :                 __call_rcu_nocb_wake(rdp, was_alldone, flags); /* unlocks */
    2640                 :            :         } else {
    2641                 :    2219838 :                 __call_rcu_core(rdp, head, flags);
    2642                 :    2219838 :                 local_irq_restore(flags);
    2643                 :            :         }
    2644                 :            : }
    2645                 :            : 
    2646                 :            : /**
    2647                 :            :  * call_rcu() - Queue an RCU callback for invocation after a grace period.
    2648                 :            :  * @head: structure to be used for queueing the RCU updates.
    2649                 :            :  * @func: actual callback function to be invoked after the grace period
    2650                 :            :  *
    2651                 :            :  * The callback function will be invoked some time after a full grace
    2652                 :            :  * period elapses, in other words after all pre-existing RCU read-side
    2653                 :            :  * critical sections have completed.  However, the callback function
    2654                 :            :  * might well execute concurrently with RCU read-side critical sections
    2655                 :            :  * that started after call_rcu() was invoked.  RCU read-side critical
    2656                 :            :  * sections are delimited by rcu_read_lock() and rcu_read_unlock(), and
    2657                 :            :  * may be nested.  In addition, regions of code across which interrupts,
    2658                 :            :  * preemption, or softirqs have been disabled also serve as RCU read-side
    2659                 :            :  * critical sections.  This includes hardware interrupt handlers, softirq
    2660                 :            :  * handlers, and NMI handlers.
    2661                 :            :  *
    2662                 :            :  * Note that all CPUs must agree that the grace period extended beyond
    2663                 :            :  * all pre-existing RCU read-side critical section.  On systems with more
    2664                 :            :  * than one CPU, this means that when "func()" is invoked, each CPU is
    2665                 :            :  * guaranteed to have executed a full memory barrier since the end of its
    2666                 :            :  * last RCU read-side critical section whose beginning preceded the call
    2667                 :            :  * to call_rcu().  It also means that each CPU executing an RCU read-side
    2668                 :            :  * critical section that continues beyond the start of "func()" must have
    2669                 :            :  * executed a memory barrier after the call_rcu() but before the beginning
    2670                 :            :  * of that RCU read-side critical section.  Note that these guarantees
    2671                 :            :  * include CPUs that are offline, idle, or executing in user mode, as
    2672                 :            :  * well as CPUs that are executing in the kernel.
    2673                 :            :  *
    2674                 :            :  * Furthermore, if CPU A invoked call_rcu() and CPU B invoked the
    2675                 :            :  * resulting RCU callback function "func()", then both CPU A and CPU B are
    2676                 :            :  * guaranteed to execute a full memory barrier during the time interval
    2677                 :            :  * between the call to call_rcu() and the invocation of "func()" -- even
    2678                 :            :  * if CPU A and CPU B are the same CPU (but again only if the system has
    2679                 :            :  * more than one CPU).
    2680                 :            :  */
    2681                 :    2219838 : void call_rcu(struct rcu_head *head, rcu_callback_t func)
    2682                 :            : {
    2683                 :    2219838 :         __call_rcu(head, func);
    2684                 :    2219838 : }
    2685                 :            : EXPORT_SYMBOL_GPL(call_rcu);
    2686                 :            : 
    2687                 :            : 
    2688                 :            : /* Maximum number of jiffies to wait before draining a batch. */
    2689                 :            : #define KFREE_DRAIN_JIFFIES (HZ / 50)
    2690                 :            : #define KFREE_N_BATCHES 2
    2691                 :            : 
    2692                 :            : /**
    2693                 :            :  * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests
    2694                 :            :  * @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period
    2695                 :            :  * @head_free: List of kfree_rcu() objects waiting for a grace period
    2696                 :            :  * @krcp: Pointer to @kfree_rcu_cpu structure
    2697                 :            :  */
    2698                 :            : 
    2699                 :            : struct kfree_rcu_cpu_work {
    2700                 :            :         struct rcu_work rcu_work;
    2701                 :            :         struct rcu_head *head_free;
    2702                 :            :         struct kfree_rcu_cpu *krcp;
    2703                 :            : };
    2704                 :            : 
    2705                 :            : /**
    2706                 :            :  * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period
    2707                 :            :  * @head: List of kfree_rcu() objects not yet waiting for a grace period
    2708                 :            :  * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period
    2709                 :            :  * @lock: Synchronize access to this structure
    2710                 :            :  * @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES
    2711                 :            :  * @monitor_todo: Tracks whether a @monitor_work delayed work is pending
    2712                 :            :  * @initialized: The @lock and @rcu_work fields have been initialized
    2713                 :            :  *
    2714                 :            :  * This is a per-CPU structure.  The reason that it is not included in
    2715                 :            :  * the rcu_data structure is to permit this code to be extracted from
    2716                 :            :  * the RCU files.  Such extraction could allow further optimization of
    2717                 :            :  * the interactions with the slab allocators.
    2718                 :            :  */
    2719                 :            : struct kfree_rcu_cpu {
    2720                 :            :         struct rcu_head *head;
    2721                 :            :         struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES];
    2722                 :            :         spinlock_t lock;
    2723                 :            :         struct delayed_work monitor_work;
    2724                 :            :         bool monitor_todo;
    2725                 :            :         bool initialized;
    2726                 :            : };
    2727                 :            : 
    2728                 :            : static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc);
    2729                 :            : 
    2730                 :            : /*
    2731                 :            :  * This function is invoked in workqueue context after a grace period.
    2732                 :            :  * It frees all the objects queued on ->head_free.
    2733                 :            :  */
    2734                 :        770 : static void kfree_rcu_work(struct work_struct *work)
    2735                 :            : {
    2736                 :        770 :         unsigned long flags;
    2737                 :        770 :         struct rcu_head *head, *next;
    2738                 :        770 :         struct kfree_rcu_cpu *krcp;
    2739                 :        770 :         struct kfree_rcu_cpu_work *krwp;
    2740                 :            : 
    2741                 :        770 :         krwp = container_of(to_rcu_work(work),
    2742                 :            :                             struct kfree_rcu_cpu_work, rcu_work);
    2743                 :        770 :         krcp = krwp->krcp;
    2744                 :        770 :         spin_lock_irqsave(&krcp->lock, flags);
    2745                 :        770 :         head = krwp->head_free;
    2746                 :        770 :         krwp->head_free = NULL;
    2747                 :        770 :         spin_unlock_irqrestore(&krcp->lock, flags);
    2748                 :            : 
    2749                 :            :         // List "head" is now private, so traverse locklessly.
    2750         [ +  + ]:       3596 :         for (; head; head = next) {
    2751                 :       2826 :                 unsigned long offset = (unsigned long)head->func;
    2752                 :            : 
    2753                 :       2826 :                 next = head->next;
    2754                 :            :                 // Potentially optimize with kfree_bulk in future.
    2755                 :       2826 :                 debug_rcu_head_unqueue(head);
    2756                 :       2826 :                 rcu_lock_acquire(&rcu_callback_map);
    2757                 :       2826 :                 trace_rcu_invoke_kfree_callback(rcu_state.name, head, offset);
    2758                 :            : 
    2759   [ -  +  +  - ]:       2826 :                 if (!WARN_ON_ONCE(!__is_kfree_rcu_offset(offset))) {
    2760                 :            :                         /* Could be optimized with kfree_bulk() in future. */
    2761                 :       2826 :                         kfree((void *)head - offset);
    2762                 :            :                 }
    2763                 :            : 
    2764                 :       2826 :                 rcu_lock_release(&rcu_callback_map);
    2765                 :       2826 :                 cond_resched_tasks_rcu_qs();
    2766                 :            :         }
    2767                 :        770 : }
    2768                 :            : 
    2769                 :            : /*
    2770                 :            :  * Schedule the kfree batch RCU work to run in workqueue context after a GP.
    2771                 :            :  *
    2772                 :            :  * This function is invoked by kfree_rcu_monitor() when the KFREE_DRAIN_JIFFIES
    2773                 :            :  * timeout has been reached.
    2774                 :            :  */
    2775                 :        770 : static inline bool queue_kfree_rcu_work(struct kfree_rcu_cpu *krcp)
    2776                 :            : {
    2777                 :        770 :         int i;
    2778                 :        770 :         struct kfree_rcu_cpu_work *krwp = NULL;
    2779                 :            : 
    2780                 :        770 :         lockdep_assert_held(&krcp->lock);
    2781         [ +  - ]:        770 :         for (i = 0; i < KFREE_N_BATCHES; i++)
    2782         [ +  - ]:        770 :                 if (!krcp->krw_arr[i].head_free) {
    2783                 :        770 :                         krwp = &(krcp->krw_arr[i]);
    2784                 :        770 :                         break;
    2785                 :            :                 }
    2786                 :            : 
    2787                 :            :         // If a previous RCU batch is in progress, we cannot immediately
    2788                 :            :         // queue another one, so return false to tell caller to retry.
    2789         [ +  - ]:        770 :         if (!krwp)
    2790                 :            :                 return false;
    2791                 :            : 
    2792                 :        770 :         krwp->head_free = krcp->head;
    2793                 :        770 :         krcp->head = NULL;
    2794                 :        770 :         INIT_RCU_WORK(&krwp->rcu_work, kfree_rcu_work);
    2795                 :        770 :         queue_rcu_work(system_wq, &krwp->rcu_work);
    2796                 :        770 :         return true;
    2797                 :            : }
    2798                 :            : 
    2799                 :        770 : static inline void kfree_rcu_drain_unlock(struct kfree_rcu_cpu *krcp,
    2800                 :            :                                           unsigned long flags)
    2801                 :            : {
    2802                 :            :         // Attempt to start a new batch.
    2803                 :        770 :         krcp->monitor_todo = false;
    2804         [ +  - ]:        770 :         if (queue_kfree_rcu_work(krcp)) {
    2805                 :            :                 // Success! Our job is done here.
    2806                 :        770 :                 spin_unlock_irqrestore(&krcp->lock, flags);
    2807                 :        770 :                 return;
    2808                 :            :         }
    2809                 :            : 
    2810                 :            :         // Previous RCU batch still in progress, try again later.
    2811                 :          0 :         krcp->monitor_todo = true;
    2812                 :          0 :         schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES);
    2813                 :          0 :         spin_unlock_irqrestore(&krcp->lock, flags);
    2814                 :            : }
    2815                 :            : 
    2816                 :            : /*
    2817                 :            :  * This function is invoked after the KFREE_DRAIN_JIFFIES timeout.
    2818                 :            :  * It invokes kfree_rcu_drain_unlock() to attempt to start another batch.
    2819                 :            :  */
    2820                 :        770 : static void kfree_rcu_monitor(struct work_struct *work)
    2821                 :            : {
    2822                 :        770 :         unsigned long flags;
    2823                 :        770 :         struct kfree_rcu_cpu *krcp = container_of(work, struct kfree_rcu_cpu,
    2824                 :            :                                                  monitor_work.work);
    2825                 :            : 
    2826                 :        770 :         spin_lock_irqsave(&krcp->lock, flags);
    2827         [ +  - ]:        770 :         if (krcp->monitor_todo)
    2828                 :        770 :                 kfree_rcu_drain_unlock(krcp, flags);
    2829                 :            :         else
    2830                 :          0 :                 spin_unlock_irqrestore(&krcp->lock, flags);
    2831                 :        770 : }
    2832                 :            : 
    2833                 :            : /*
    2834                 :            :  * Queue a request for lazy invocation of kfree() after a grace period.
    2835                 :            :  *
    2836                 :            :  * Each kfree_call_rcu() request is added to a batch. The batch will be drained
    2837                 :            :  * every KFREE_DRAIN_JIFFIES number of jiffies. All the objects in the batch
    2838                 :            :  * will be kfree'd in workqueue context. This allows us to:
    2839                 :            :  *
    2840                 :            :  * 1.   Batch requests together to reduce the number of grace periods during
    2841                 :            :  *      heavy kfree_rcu() load.
    2842                 :            :  *
    2843                 :            :  * 2.   It makes it possible to use kfree_bulk() on a large number of
    2844                 :            :  *      kfree_rcu() requests thus reducing cache misses and the per-object
    2845                 :            :  *      overhead of kfree().
    2846                 :            :  */
    2847                 :       2826 : void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
    2848                 :            : {
    2849                 :       2826 :         unsigned long flags;
    2850                 :       2826 :         struct kfree_rcu_cpu *krcp;
    2851                 :            : 
    2852                 :       2826 :         local_irq_save(flags);  // For safely calling this_cpu_ptr().
    2853                 :       2826 :         krcp = this_cpu_ptr(&krc);
    2854         [ +  - ]:       2826 :         if (krcp->initialized)
    2855                 :       2826 :                 spin_lock(&krcp->lock);
    2856                 :            : 
    2857                 :            :         // Queue the object but don't yet schedule the batch.
    2858         [ -  + ]:       2826 :         if (debug_rcu_head_queue(head)) {
    2859                 :            :                 // Probable double kfree_rcu(), just leak.
    2860                 :            :                 WARN_ONCE(1, "%s(): Double-freed call. rcu_head %p\n",
    2861                 :            :                           __func__, head);
    2862                 :            :                 goto unlock_return;
    2863                 :            :         }
    2864                 :       2826 :         head->func = func;
    2865                 :       2826 :         head->next = krcp->head;
    2866                 :       2826 :         krcp->head = head;
    2867                 :            : 
    2868                 :            :         // Set timer to drain after KFREE_DRAIN_JIFFIES.
    2869         [ -  + ]:       2826 :         if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
    2870         [ +  + ]:       2826 :             !krcp->monitor_todo) {
    2871                 :        770 :                 krcp->monitor_todo = true;
    2872                 :        770 :                 schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES);
    2873                 :            :         }
    2874                 :            : 
    2875                 :       2056 : unlock_return:
    2876         [ +  - ]:       2826 :         if (krcp->initialized)
    2877                 :       2826 :                 spin_unlock(&krcp->lock);
    2878                 :       2826 :         local_irq_restore(flags);
    2879                 :       2826 : }
    2880                 :            : EXPORT_SYMBOL_GPL(kfree_call_rcu);
    2881                 :            : 
    2882                 :         30 : void __init kfree_rcu_scheduler_running(void)
    2883                 :            : {
    2884                 :         30 :         int cpu;
    2885                 :         30 :         unsigned long flags;
    2886                 :            : 
    2887         [ +  + ]:         60 :         for_each_online_cpu(cpu) {
    2888                 :         30 :                 struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
    2889                 :            : 
    2890                 :         30 :                 spin_lock_irqsave(&krcp->lock, flags);
    2891   [ -  +  -  - ]:         30 :                 if (!krcp->head || krcp->monitor_todo) {
    2892                 :         30 :                         spin_unlock_irqrestore(&krcp->lock, flags);
    2893                 :         30 :                         continue;
    2894                 :            :                 }
    2895                 :          0 :                 krcp->monitor_todo = true;
    2896                 :          0 :                 schedule_delayed_work_on(cpu, &krcp->monitor_work,
    2897                 :            :                                          KFREE_DRAIN_JIFFIES);
    2898                 :          0 :                 spin_unlock_irqrestore(&krcp->lock, flags);
    2899                 :            :         }
    2900                 :         30 : }
    2901                 :            : 
    2902                 :            : /*
    2903                 :            :  * During early boot, any blocking grace-period wait automatically
    2904                 :            :  * implies a grace period.  Later on, this is never the case for PREEMPTION.
    2905                 :            :  *
    2906                 :            :  * Howevr, because a context switch is a grace period for !PREEMPTION, any
    2907                 :            :  * blocking grace-period wait automatically implies a grace period if
    2908                 :            :  * there is only one CPU online at any point time during execution of
    2909                 :            :  * either synchronize_rcu() or synchronize_rcu_expedited().  It is OK to
    2910                 :            :  * occasionally incorrectly indicate that there are multiple CPUs online
    2911                 :            :  * when there was in fact only one the whole time, as this just adds some
    2912                 :            :  * overhead: RCU still operates correctly.
    2913                 :            :  */
    2914                 :       2171 : static int rcu_blocking_is_gp(void)
    2915                 :            : {
    2916                 :       2171 :         int ret;
    2917                 :            : 
    2918                 :       2171 :         if (IS_ENABLED(CONFIG_PREEMPTION))
    2919                 :            :                 return rcu_scheduler_active == RCU_SCHEDULER_INACTIVE;
    2920                 :       2171 :         might_sleep();  /* Check for RCU read-side critical section. */
    2921                 :       2171 :         preempt_disable();
    2922                 :       2171 :         ret = num_online_cpus() <= 1;
    2923                 :       2171 :         preempt_enable();
    2924                 :       2171 :         return ret;
    2925                 :            : }
    2926                 :            : 
    2927                 :            : /**
    2928                 :            :  * synchronize_rcu - wait until a grace period has elapsed.
    2929                 :            :  *
    2930                 :            :  * Control will return to the caller some time after a full grace
    2931                 :            :  * period has elapsed, in other words after all currently executing RCU
    2932                 :            :  * read-side critical sections have completed.  Note, however, that
    2933                 :            :  * upon return from synchronize_rcu(), the caller might well be executing
    2934                 :            :  * concurrently with new RCU read-side critical sections that began while
    2935                 :            :  * synchronize_rcu() was waiting.  RCU read-side critical sections are
    2936                 :            :  * delimited by rcu_read_lock() and rcu_read_unlock(), and may be nested.
    2937                 :            :  * In addition, regions of code across which interrupts, preemption, or
    2938                 :            :  * softirqs have been disabled also serve as RCU read-side critical
    2939                 :            :  * sections.  This includes hardware interrupt handlers, softirq handlers,
    2940                 :            :  * and NMI handlers.
    2941                 :            :  *
    2942                 :            :  * Note that this guarantee implies further memory-ordering guarantees.
    2943                 :            :  * On systems with more than one CPU, when synchronize_rcu() returns,
    2944                 :            :  * each CPU is guaranteed to have executed a full memory barrier since
    2945                 :            :  * the end of its last RCU read-side critical section whose beginning
    2946                 :            :  * preceded the call to synchronize_rcu().  In addition, each CPU having
    2947                 :            :  * an RCU read-side critical section that extends beyond the return from
    2948                 :            :  * synchronize_rcu() is guaranteed to have executed a full memory barrier
    2949                 :            :  * after the beginning of synchronize_rcu() and before the beginning of
    2950                 :            :  * that RCU read-side critical section.  Note that these guarantees include
    2951                 :            :  * CPUs that are offline, idle, or executing in user mode, as well as CPUs
    2952                 :            :  * that are executing in the kernel.
    2953                 :            :  *
    2954                 :            :  * Furthermore, if CPU A invoked synchronize_rcu(), which returned
    2955                 :            :  * to its caller on CPU B, then both CPU A and CPU B are guaranteed
    2956                 :            :  * to have executed a full memory barrier during the execution of
    2957                 :            :  * synchronize_rcu() -- even if CPU A and CPU B are the same CPU (but
    2958                 :            :  * again only if the system has more than one CPU).
    2959                 :            :  */
    2960                 :       2016 : void synchronize_rcu(void)
    2961                 :            : {
    2962                 :       2016 :         RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
    2963                 :            :                          lock_is_held(&rcu_lock_map) ||
    2964                 :            :                          lock_is_held(&rcu_sched_lock_map),
    2965                 :            :                          "Illegal synchronize_rcu() in RCU read-side critical section");
    2966         [ -  + ]:       2016 :         if (rcu_blocking_is_gp())
    2967                 :            :                 return;
    2968         [ #  # ]:          0 :         if (rcu_gp_is_expedited())
    2969                 :          0 :                 synchronize_rcu_expedited();
    2970                 :            :         else
    2971                 :          0 :                 wait_rcu_gp(call_rcu);
    2972                 :            : }
    2973                 :            : EXPORT_SYMBOL_GPL(synchronize_rcu);
    2974                 :            : 
    2975                 :            : /**
    2976                 :            :  * get_state_synchronize_rcu - Snapshot current RCU state
    2977                 :            :  *
    2978                 :            :  * Returns a cookie that is used by a later call to cond_synchronize_rcu()
    2979                 :            :  * to determine whether or not a full grace period has elapsed in the
    2980                 :            :  * meantime.
    2981                 :            :  */
    2982                 :          0 : unsigned long get_state_synchronize_rcu(void)
    2983                 :            : {
    2984                 :            :         /*
    2985                 :            :          * Any prior manipulation of RCU-protected data must happen
    2986                 :            :          * before the load from ->gp_seq.
    2987                 :            :          */
    2988                 :          0 :         smp_mb();  /* ^^^ */
    2989                 :          0 :         return rcu_seq_snap(&rcu_state.gp_seq);
    2990                 :            : }
    2991                 :            : EXPORT_SYMBOL_GPL(get_state_synchronize_rcu);
    2992                 :            : 
    2993                 :            : /**
    2994                 :            :  * cond_synchronize_rcu - Conditionally wait for an RCU grace period
    2995                 :            :  *
    2996                 :            :  * @oldstate: return value from earlier call to get_state_synchronize_rcu()
    2997                 :            :  *
    2998                 :            :  * If a full RCU grace period has elapsed since the earlier call to
    2999                 :            :  * get_state_synchronize_rcu(), just return.  Otherwise, invoke
    3000                 :            :  * synchronize_rcu() to wait for a full grace period.
    3001                 :            :  *
    3002                 :            :  * Yes, this function does not take counter wrap into account.  But
    3003                 :            :  * counter wrap is harmless.  If the counter wraps, we have waited for
    3004                 :            :  * more than 2 billion grace periods (and way more on a 64-bit system!),
    3005                 :            :  * so waiting for one additional grace period should be just fine.
    3006                 :            :  */
    3007                 :          0 : void cond_synchronize_rcu(unsigned long oldstate)
    3008                 :            : {
    3009         [ #  # ]:          0 :         if (!rcu_seq_done(&rcu_state.gp_seq, oldstate))
    3010                 :          0 :                 synchronize_rcu();
    3011                 :            :         else
    3012                 :          0 :                 smp_mb(); /* Ensure GP ends before subsequent accesses. */
    3013                 :          0 : }
    3014                 :            : EXPORT_SYMBOL_GPL(cond_synchronize_rcu);
    3015                 :            : 
    3016                 :            : /*
    3017                 :            :  * Check to see if there is any immediate RCU-related work to be done by
    3018                 :            :  * the current CPU, returning 1 if so and zero otherwise.  The checks are
    3019                 :            :  * in order of increasing expense: checks that can be carried out against
    3020                 :            :  * CPU-local state are performed first.  However, we must check for CPU
    3021                 :            :  * stalls first, else we might not get a chance.
    3022                 :            :  */
    3023                 :      97200 : static int rcu_pending(int user)
    3024                 :            : {
    3025                 :      97200 :         bool gp_in_progress;
    3026                 :      97200 :         struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
    3027                 :      97200 :         struct rcu_node *rnp = rdp->mynode;
    3028                 :            : 
    3029                 :            :         /* Check for CPU stalls, if enabled. */
    3030                 :      97200 :         check_cpu_stall(rdp);
    3031                 :            : 
    3032                 :            :         /* Does this CPU need a deferred NOCB wakeup? */
    3033         [ +  + ]:      97200 :         if (rcu_nocb_need_deferred_wakeup(rdp))
    3034                 :            :                 return 1;
    3035                 :            : 
    3036                 :            :         /* Is this a nohz_full CPU in userspace or idle?  (Ignore RCU if so.) */
    3037         [ +  + ]:      97200 :         if ((user || rcu_is_cpu_rrupt_from_idle()) && rcu_nohz_full_cpu())
    3038                 :            :                 return 0;
    3039                 :            : 
    3040                 :            :         /* Is the RCU core waiting for a quiescent state from this CPU? */
    3041         [ +  + ]:      97200 :         gp_in_progress = rcu_gp_in_progress();
    3042   [ +  +  +  +  :      97200 :         if (rdp->core_needs_qs && !rdp->cpu_no_qs.b.norm && gp_in_progress)
                   +  + ]
    3043                 :            :                 return 1;
    3044                 :            : 
    3045                 :            :         /* Does this CPU have callbacks ready to invoke? */
    3046         [ +  + ]:      39976 :         if (rcu_segcblist_ready_cbs(&rdp->cblist))
    3047                 :            :                 return 1;
    3048                 :            : 
    3049                 :            :         /* Has RCU gone idle with this CPU needing another grace period? */
    3050   [ +  +  +  - ]:      39973 :         if (!gp_in_progress && rcu_segcblist_is_enabled(&rdp->cblist) &&
    3051                 :            :             (!IS_ENABLED(CONFIG_RCU_NOCB_CPU) ||
    3052         [ +  + ]:      39972 :              !rcu_segcblist_is_offloaded(&rdp->cblist)) &&
    3053                 :            :             !rcu_segcblist_restempty(&rdp->cblist, RCU_NEXT_READY_TAIL))
    3054                 :            :                 return 1;
    3055                 :            : 
    3056                 :            :         /* Have RCU grace period completed or started?  */
    3057   [ +  -  -  + ]:      37478 :         if (rcu_seq_current(&rnp->gp_seq) != rdp->gp_seq ||
    3058         [ -  + ]:      37478 :             unlikely(READ_ONCE(rdp->gpwrap))) /* outside lock */
    3059                 :          0 :                 return 1;
    3060                 :            : 
    3061                 :            :         /* nothing to do */
    3062                 :            :         return 0;
    3063                 :            : }
    3064                 :            : 
    3065                 :            : /*
    3066                 :            :  * Helper function for rcu_barrier() tracing.  If tracing is disabled,
    3067                 :            :  * the compiler is expected to optimize this away.
    3068                 :            :  */
    3069                 :        172 : static void rcu_barrier_trace(const char *s, int cpu, unsigned long done)
    3070                 :            : {
    3071                 :        172 :         trace_rcu_barrier(rcu_state.name, s, cpu,
    3072                 :            :                           atomic_read(&rcu_state.barrier_cpu_count), done);
    3073                 :        172 : }
    3074                 :            : 
    3075                 :            : /*
    3076                 :            :  * RCU callback function for rcu_barrier().  If we are last, wake
    3077                 :            :  * up the task executing rcu_barrier().
    3078                 :            :  */
    3079                 :         26 : static void rcu_barrier_callback(struct rcu_head *rhp)
    3080                 :            : {
    3081         [ +  - ]:         26 :         if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) {
    3082                 :         26 :                 rcu_barrier_trace(TPS("LastCB"), -1,
    3083                 :            :                                   rcu_state.barrier_sequence);
    3084                 :         26 :                 complete(&rcu_state.barrier_completion);
    3085                 :            :         } else {
    3086                 :          0 :                 rcu_barrier_trace(TPS("CB"), -1, rcu_state.barrier_sequence);
    3087                 :            :         }
    3088                 :         26 : }
    3089                 :            : 
    3090                 :            : /*
    3091                 :            :  * Called with preemption disabled, and from cross-cpu IRQ context.
    3092                 :            :  */
    3093                 :         26 : static void rcu_barrier_func(void *unused)
    3094                 :            : {
    3095                 :         26 :         struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
    3096                 :            : 
    3097                 :         26 :         rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);
    3098                 :         26 :         rdp->barrier_head.func = rcu_barrier_callback;
    3099                 :         26 :         debug_rcu_head_queue(&rdp->barrier_head);
    3100                 :         26 :         rcu_nocb_lock(rdp);
    3101                 :         26 :         WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, jiffies));
    3102         [ +  - ]:         26 :         if (rcu_segcblist_entrain(&rdp->cblist, &rdp->barrier_head)) {
    3103                 :         26 :                 atomic_inc(&rcu_state.barrier_cpu_count);
    3104                 :            :         } else {
    3105                 :          0 :                 debug_rcu_head_unqueue(&rdp->barrier_head);
    3106                 :          0 :                 rcu_barrier_trace(TPS("IRQNQ"), -1,
    3107                 :            :                                   rcu_state.barrier_sequence);
    3108                 :            :         }
    3109                 :         26 :         rcu_nocb_unlock(rdp);
    3110                 :         26 : }
    3111                 :            : 
    3112                 :            : /**
    3113                 :            :  * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete.
    3114                 :            :  *
    3115                 :            :  * Note that this primitive does not necessarily wait for an RCU grace period
    3116                 :            :  * to complete.  For example, if there are no RCU callbacks queued anywhere
    3117                 :            :  * in the system, then rcu_barrier() is within its rights to return
    3118                 :            :  * immediately, without waiting for anything, much less an RCU grace period.
    3119                 :            :  */
    3120                 :         30 : void rcu_barrier(void)
    3121                 :            : {
    3122                 :         30 :         int cpu;
    3123                 :         30 :         struct rcu_data *rdp;
    3124                 :         30 :         unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);
    3125                 :            : 
    3126                 :         30 :         rcu_barrier_trace(TPS("Begin"), -1, s);
    3127                 :            : 
    3128                 :            :         /* Take mutex to serialize concurrent rcu_barrier() requests. */
    3129                 :         30 :         mutex_lock(&rcu_state.barrier_mutex);
    3130                 :            : 
    3131                 :            :         /* Did someone else do our work for us? */
    3132         [ -  + ]:         30 :         if (rcu_seq_done(&rcu_state.barrier_sequence, s)) {
    3133                 :          0 :                 rcu_barrier_trace(TPS("EarlyExit"), -1,
    3134                 :            :                                   rcu_state.barrier_sequence);
    3135                 :          0 :                 smp_mb(); /* caller's subsequent code after above check. */
    3136                 :          0 :                 mutex_unlock(&rcu_state.barrier_mutex);
    3137                 :          0 :                 return;
    3138                 :            :         }
    3139                 :            : 
    3140                 :            :         /* Mark the start of the barrier operation. */
    3141                 :         30 :         rcu_seq_start(&rcu_state.barrier_sequence);
    3142                 :         30 :         rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence);
    3143                 :            : 
    3144                 :            :         /*
    3145                 :            :          * Initialize the count to one rather than to zero in order to
    3146                 :            :          * avoid a too-soon return to zero in case of a short grace period
    3147                 :            :          * (or preemption of this task).  Exclude CPU-hotplug operations
    3148                 :            :          * to ensure that no offline CPU has callbacks queued.
    3149                 :            :          */
    3150                 :         30 :         init_completion(&rcu_state.barrier_completion);
    3151                 :         30 :         atomic_set(&rcu_state.barrier_cpu_count, 1);
    3152                 :         30 :         get_online_cpus();
    3153                 :            : 
    3154                 :            :         /*
    3155                 :            :          * Force each CPU with callbacks to register a new callback.
    3156                 :            :          * When that callback is invoked, we will know that all of the
    3157                 :            :          * corresponding CPU's preceding callbacks have been invoked.
    3158                 :            :          */
    3159         [ +  + ]:         90 :         for_each_possible_cpu(cpu) {
    3160                 :         30 :                 rdp = per_cpu_ptr(&rcu_data, cpu);
    3161   [ -  +  -  - ]:         30 :                 if (!cpu_online(cpu) &&
    3162         [ #  # ]:          0 :                     !rcu_segcblist_is_offloaded(&rdp->cblist))
    3163                 :          0 :                         continue;
    3164         [ +  + ]:         30 :                 if (rcu_segcblist_n_cbs(&rdp->cblist)) {
    3165                 :         26 :                         rcu_barrier_trace(TPS("OnlineQ"), cpu,
    3166                 :            :                                           rcu_state.barrier_sequence);
    3167                 :         26 :                         smp_call_function_single(cpu, rcu_barrier_func, NULL, 1);
    3168                 :            :                 } else {
    3169                 :          4 :                         rcu_barrier_trace(TPS("OnlineNQ"), cpu,
    3170                 :            :                                           rcu_state.barrier_sequence);
    3171                 :            :                 }
    3172                 :            :         }
    3173                 :         30 :         put_online_cpus();
    3174                 :            : 
    3175                 :            :         /*
    3176                 :            :          * Now that we have an rcu_barrier_callback() callback on each
    3177                 :            :          * CPU, and thus each counted, remove the initial count.
    3178                 :            :          */
    3179         [ +  + ]:         30 :         if (atomic_dec_and_test(&rcu_state.barrier_cpu_count))
    3180                 :          4 :                 complete(&rcu_state.barrier_completion);
    3181                 :            : 
    3182                 :            :         /* Wait for all rcu_barrier_callback() callbacks to be invoked. */
    3183                 :         30 :         wait_for_completion(&rcu_state.barrier_completion);
    3184                 :            : 
    3185                 :            :         /* Mark the end of the barrier operation. */
    3186                 :         30 :         rcu_barrier_trace(TPS("Inc2"), -1, rcu_state.barrier_sequence);
    3187                 :         30 :         rcu_seq_end(&rcu_state.barrier_sequence);
    3188                 :            : 
    3189                 :            :         /* Other rcu_barrier() invocations can now safely proceed. */
    3190                 :         30 :         mutex_unlock(&rcu_state.barrier_mutex);
    3191                 :            : }
    3192                 :            : EXPORT_SYMBOL_GPL(rcu_barrier);
    3193                 :            : 
    3194                 :            : /*
    3195                 :            :  * Propagate ->qsinitmask bits up the rcu_node tree to account for the
    3196                 :            :  * first CPU in a given leaf rcu_node structure coming online.  The caller
    3197                 :            :  * must hold the corresponding leaf rcu_node ->lock with interrrupts
    3198                 :            :  * disabled.
    3199                 :            :  */
    3200                 :         30 : static void rcu_init_new_rnp(struct rcu_node *rnp_leaf)
    3201                 :            : {
    3202                 :         30 :         long mask;
    3203                 :         30 :         long oldmask;
    3204                 :         30 :         struct rcu_node *rnp = rnp_leaf;
    3205                 :            : 
    3206                 :         30 :         raw_lockdep_assert_held_rcu_node(rnp_leaf);
    3207         [ -  + ]:         30 :         WARN_ON_ONCE(rnp->wait_blkd_tasks);
    3208                 :         30 :         for (;;) {
    3209                 :         30 :                 mask = rnp->grpmask;
    3210                 :         30 :                 rnp = rnp->parent;
    3211         [ -  + ]:         30 :                 if (rnp == NULL)
    3212                 :            :                         return;
    3213                 :          0 :                 raw_spin_lock_rcu_node(rnp); /* Interrupts already disabled. */
    3214                 :          0 :                 oldmask = rnp->qsmaskinit;
    3215                 :          0 :                 rnp->qsmaskinit |= mask;
    3216                 :          0 :                 raw_spin_unlock_rcu_node(rnp); /* Interrupts remain disabled. */
    3217         [ #  # ]:          0 :                 if (oldmask)
    3218                 :            :                         return;
    3219                 :            :         }
    3220                 :            : }
    3221                 :            : 
    3222                 :            : /*
    3223                 :            :  * Do boot-time initialization of a CPU's per-CPU RCU data.
    3224                 :            :  */
    3225                 :            : static void __init
    3226                 :         30 : rcu_boot_init_percpu_data(int cpu)
    3227                 :            : {
    3228                 :         30 :         struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
    3229                 :            : 
    3230                 :            :         /* Set up local state, ensuring consistent view of global state. */
    3231                 :         30 :         rdp->grpmask = leaf_node_cpu_bit(rdp->mynode, cpu);
    3232         [ -  + ]:         30 :         WARN_ON_ONCE(rdp->dynticks_nesting != 1);
    3233         [ -  + ]:         30 :         WARN_ON_ONCE(rcu_dynticks_in_eqs(rcu_dynticks_snap(rdp)));
    3234                 :         30 :         rdp->rcu_ofl_gp_seq = rcu_state.gp_seq;
    3235                 :         30 :         rdp->rcu_ofl_gp_flags = RCU_GP_CLEANED;
    3236                 :         30 :         rdp->rcu_onl_gp_seq = rcu_state.gp_seq;
    3237                 :         30 :         rdp->rcu_onl_gp_flags = RCU_GP_CLEANED;
    3238                 :         30 :         rdp->cpu = cpu;
    3239                 :         30 :         rcu_boot_init_nocb_percpu_data(rdp);
    3240                 :         30 : }
    3241                 :            : 
    3242                 :            : /*
    3243                 :            :  * Invoked early in the CPU-online process, when pretty much all services
    3244                 :            :  * are available.  The incoming CPU is not present.
    3245                 :            :  *
    3246                 :            :  * Initializes a CPU's per-CPU RCU data.  Note that only one online or
    3247                 :            :  * offline event can be happening at a given time.  Note also that we can
    3248                 :            :  * accept some slop in the rsp->gp_seq access due to the fact that this
    3249                 :            :  * CPU cannot possibly have any non-offloaded RCU callbacks in flight yet.
    3250                 :            :  * And any offloaded callbacks are being numbered elsewhere.
    3251                 :            :  */
    3252                 :         30 : int rcutree_prepare_cpu(unsigned int cpu)
    3253                 :            : {
    3254                 :         30 :         unsigned long flags;
    3255                 :         30 :         struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
    3256                 :         30 :         struct rcu_node *rnp = rcu_get_root();
    3257                 :            : 
    3258                 :            :         /* Set up local state, ensuring consistent view of global state. */
    3259                 :         30 :         raw_spin_lock_irqsave_rcu_node(rnp, flags);
    3260                 :         30 :         rdp->qlen_last_fqs_check = 0;
    3261                 :         30 :         rdp->n_force_qs_snap = rcu_state.n_force_qs;
    3262                 :         30 :         rdp->blimit = blimit;
    3263   [ +  -  +  - ]:         30 :         if (rcu_segcblist_empty(&rdp->cblist) && /* No early-boot CBs? */
    3264         [ +  - ]:         30 :             !rcu_segcblist_is_offloaded(&rdp->cblist))
    3265                 :         30 :                 rcu_segcblist_init(&rdp->cblist);  /* Re-enable callbacks. */
    3266                 :         30 :         rdp->dynticks_nesting = 1;   /* CPU not up, no tearing. */
    3267                 :         30 :         rcu_dynticks_eqs_online();
    3268                 :         30 :         raw_spin_unlock_rcu_node(rnp);          /* irqs remain disabled. */
    3269                 :            : 
    3270                 :            :         /*
    3271                 :            :          * Add CPU to leaf rcu_node pending-online bitmask.  Any needed
    3272                 :            :          * propagation up the rcu_node tree will happen at the beginning
    3273                 :            :          * of the next grace period.
    3274                 :            :          */
    3275                 :         30 :         rnp = rdp->mynode;
    3276                 :         30 :         raw_spin_lock_rcu_node(rnp);            /* irqs already disabled. */
    3277                 :         30 :         rdp->beenonline = true;       /* We have now been online. */
    3278                 :         30 :         rdp->gp_seq = rnp->gp_seq;
    3279                 :         30 :         rdp->gp_seq_needed = rnp->gp_seq;
    3280                 :         30 :         rdp->cpu_no_qs.b.norm = true;
    3281                 :         30 :         rdp->core_needs_qs = false;
    3282                 :         30 :         rdp->rcu_iw_pending = false;
    3283                 :         30 :         rdp->rcu_iw_gp_seq = rnp->gp_seq - 1;
    3284                 :         30 :         trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));
    3285                 :         30 :         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    3286                 :         30 :         rcu_prepare_kthreads(cpu);
    3287                 :         30 :         rcu_spawn_cpu_nocb_kthread(cpu);
    3288                 :            : 
    3289                 :         30 :         return 0;
    3290                 :            : }
    3291                 :            : 
    3292                 :            : /*
    3293                 :            :  * Update RCU priority boot kthread affinity for CPU-hotplug changes.
    3294                 :            :  */
    3295                 :          0 : static void rcutree_affinity_setting(unsigned int cpu, int outgoing)
    3296                 :            : {
    3297                 :          0 :         struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
    3298                 :            : 
    3299                 :          0 :         rcu_boost_kthread_setaffinity(rdp->mynode, outgoing);
    3300                 :            : }
    3301                 :            : 
    3302                 :            : /*
    3303                 :            :  * Near the end of the CPU-online process.  Pretty much all services
    3304                 :            :  * enabled, and the CPU is now very much alive.
    3305                 :            :  */
    3306                 :         30 : int rcutree_online_cpu(unsigned int cpu)
    3307                 :            : {
    3308                 :         30 :         unsigned long flags;
    3309                 :         30 :         struct rcu_data *rdp;
    3310                 :         30 :         struct rcu_node *rnp;
    3311                 :            : 
    3312                 :         30 :         rdp = per_cpu_ptr(&rcu_data, cpu);
    3313                 :         30 :         rnp = rdp->mynode;
    3314                 :         30 :         raw_spin_lock_irqsave_rcu_node(rnp, flags);
    3315                 :         30 :         rnp->ffmask |= rdp->grpmask;
    3316                 :         30 :         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    3317         [ -  + ]:         30 :         if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE)
    3318                 :            :                 return 0; /* Too early in boot for scheduler work. */
    3319                 :          0 :         sync_sched_exp_online_cleanup(cpu);
    3320                 :          0 :         rcutree_affinity_setting(cpu, -1);
    3321                 :            : 
    3322                 :            :         // Stop-machine done, so allow nohz_full to disable tick.
    3323                 :          0 :         tick_dep_clear(TICK_DEP_BIT_RCU);
    3324                 :          0 :         return 0;
    3325                 :            : }
    3326                 :            : 
    3327                 :            : /*
    3328                 :            :  * Near the beginning of the process.  The CPU is still very much alive
    3329                 :            :  * with pretty much all services enabled.
    3330                 :            :  */
    3331                 :          0 : int rcutree_offline_cpu(unsigned int cpu)
    3332                 :            : {
    3333                 :          0 :         unsigned long flags;
    3334                 :          0 :         struct rcu_data *rdp;
    3335                 :          0 :         struct rcu_node *rnp;
    3336                 :            : 
    3337                 :          0 :         rdp = per_cpu_ptr(&rcu_data, cpu);
    3338                 :          0 :         rnp = rdp->mynode;
    3339                 :          0 :         raw_spin_lock_irqsave_rcu_node(rnp, flags);
    3340                 :          0 :         rnp->ffmask &= ~rdp->grpmask;
    3341                 :          0 :         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    3342                 :            : 
    3343                 :          0 :         rcutree_affinity_setting(cpu, cpu);
    3344                 :            : 
    3345                 :            :         // nohz_full CPUs need the tick for stop-machine to work quickly
    3346                 :          0 :         tick_dep_set(TICK_DEP_BIT_RCU);
    3347                 :          0 :         return 0;
    3348                 :            : }
    3349                 :            : 
    3350                 :            : static DEFINE_PER_CPU(int, rcu_cpu_started);
    3351                 :            : 
    3352                 :            : /*
    3353                 :            :  * Mark the specified CPU as being online so that subsequent grace periods
    3354                 :            :  * (both expedited and normal) will wait on it.  Note that this means that
    3355                 :            :  * incoming CPUs are not allowed to use RCU read-side critical sections
    3356                 :            :  * until this function is called.  Failing to observe this restriction
    3357                 :            :  * will result in lockdep splats.
    3358                 :            :  *
    3359                 :            :  * Note that this function is special in that it is invoked directly
    3360                 :            :  * from the incoming CPU rather than from the cpuhp_step mechanism.
    3361                 :            :  * This is because this function must be invoked at a precise location.
    3362                 :            :  */
    3363                 :         30 : void rcu_cpu_starting(unsigned int cpu)
    3364                 :            : {
    3365                 :         30 :         unsigned long flags;
    3366                 :         30 :         unsigned long mask;
    3367                 :         30 :         int nbits;
    3368                 :         30 :         unsigned long oldmask;
    3369                 :         30 :         struct rcu_data *rdp;
    3370                 :         30 :         struct rcu_node *rnp;
    3371                 :            : 
    3372         [ +  - ]:         30 :         if (per_cpu(rcu_cpu_started, cpu))
    3373                 :            :                 return;
    3374                 :            : 
    3375                 :         30 :         per_cpu(rcu_cpu_started, cpu) = 1;
    3376                 :            : 
    3377                 :         30 :         rdp = per_cpu_ptr(&rcu_data, cpu);
    3378                 :         30 :         rnp = rdp->mynode;
    3379                 :         30 :         mask = rdp->grpmask;
    3380                 :         30 :         raw_spin_lock_irqsave_rcu_node(rnp, flags);
    3381                 :         30 :         rnp->qsmaskinitnext |= mask;
    3382                 :         30 :         oldmask = rnp->expmaskinitnext;
    3383                 :         30 :         rnp->expmaskinitnext |= mask;
    3384                 :         30 :         oldmask ^= rnp->expmaskinitnext;
    3385         [ -  + ]:         30 :         nbits = bitmap_weight(&oldmask, BITS_PER_LONG);
    3386                 :            :         /* Allow lockless access for expedited grace periods. */
    3387         [ -  + ]:         30 :         smp_store_release(&rcu_state.ncpus, rcu_state.ncpus + nbits); /* ^^^ */
    3388         [ -  + ]:         30 :         rcu_gpnum_ovf(rnp, rdp); /* Offline-induced counter wrap? */
    3389         [ -  + ]:         30 :         rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq);
    3390                 :         30 :         rdp->rcu_onl_gp_flags = READ_ONCE(rcu_state.gp_flags);
    3391         [ -  + ]:         30 :         if (rnp->qsmask & mask) { /* RCU waiting on incoming CPU? */
    3392                 :          0 :                 rcu_disable_urgency_upon_qs(rdp);
    3393                 :            :                 /* Report QS -after- changing ->qsmaskinitnext! */
    3394                 :          0 :                 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
    3395                 :            :         } else {
    3396                 :         30 :                 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    3397                 :            :         }
    3398                 :         30 :         smp_mb(); /* Ensure RCU read-side usage follows above initialization. */
    3399                 :            : }
    3400                 :            : 
    3401                 :            : #ifdef CONFIG_HOTPLUG_CPU
    3402                 :            : /*
    3403                 :            :  * The outgoing function has no further need of RCU, so remove it from
    3404                 :            :  * the rcu_node tree's ->qsmaskinitnext bit masks.
    3405                 :            :  *
    3406                 :            :  * Note that this function is special in that it is invoked directly
    3407                 :            :  * from the outgoing CPU rather than from the cpuhp_step mechanism.
    3408                 :            :  * This is because this function must be invoked at a precise location.
    3409                 :            :  */
    3410                 :          0 : void rcu_report_dead(unsigned int cpu)
    3411                 :            : {
    3412                 :          0 :         unsigned long flags;
    3413                 :          0 :         unsigned long mask;
    3414                 :          0 :         struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
    3415                 :          0 :         struct rcu_node *rnp = rdp->mynode;  /* Outgoing CPU's rdp & rnp. */
    3416                 :            : 
    3417                 :            :         /* QS for any half-done expedited grace period. */
    3418                 :          0 :         preempt_disable();
    3419                 :          0 :         rcu_report_exp_rdp(this_cpu_ptr(&rcu_data));
    3420                 :          0 :         preempt_enable();
    3421                 :          0 :         rcu_preempt_deferred_qs(current);
    3422                 :            : 
    3423                 :            :         /* Remove outgoing CPU from mask in the leaf rcu_node structure. */
    3424                 :          0 :         mask = rdp->grpmask;
    3425                 :          0 :         raw_spin_lock(&rcu_state.ofl_lock);
    3426                 :          0 :         raw_spin_lock_irqsave_rcu_node(rnp, flags); /* Enforce GP memory-order guarantee. */
    3427         [ #  # ]:          0 :         rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq);
    3428                 :          0 :         rdp->rcu_ofl_gp_flags = READ_ONCE(rcu_state.gp_flags);
    3429         [ #  # ]:          0 :         if (rnp->qsmask & mask) { /* RCU waiting on outgoing CPU? */
    3430                 :            :                 /* Report quiescent state -before- changing ->qsmaskinitnext! */
    3431                 :          0 :                 rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
    3432                 :          0 :                 raw_spin_lock_irqsave_rcu_node(rnp, flags);
    3433                 :            :         }
    3434                 :          0 :         rnp->qsmaskinitnext &= ~mask;
    3435                 :          0 :         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    3436                 :          0 :         raw_spin_unlock(&rcu_state.ofl_lock);
    3437                 :            : 
    3438                 :          0 :         per_cpu(rcu_cpu_started, cpu) = 0;
    3439                 :          0 : }
    3440                 :            : 
    3441                 :            : /*
    3442                 :            :  * The outgoing CPU has just passed through the dying-idle state, and we
    3443                 :            :  * are being invoked from the CPU that was IPIed to continue the offline
    3444                 :            :  * operation.  Migrate the outgoing CPU's callbacks to the current CPU.
    3445                 :            :  */
    3446                 :          0 : void rcutree_migrate_callbacks(int cpu)
    3447                 :            : {
    3448                 :          0 :         unsigned long flags;
    3449                 :          0 :         struct rcu_data *my_rdp;
    3450                 :          0 :         struct rcu_node *my_rnp;
    3451                 :          0 :         struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
    3452                 :          0 :         bool needwake;
    3453                 :            : 
    3454   [ #  #  #  # ]:          0 :         if (rcu_segcblist_is_offloaded(&rdp->cblist) ||
    3455                 :            :             rcu_segcblist_empty(&rdp->cblist))
    3456                 :            :                 return;  /* No callbacks to migrate. */
    3457                 :            : 
    3458                 :          0 :         local_irq_save(flags);
    3459                 :          0 :         my_rdp = this_cpu_ptr(&rcu_data);
    3460                 :          0 :         my_rnp = my_rdp->mynode;
    3461                 :          0 :         rcu_nocb_lock(my_rdp); /* irqs already disabled. */
    3462                 :          0 :         WARN_ON_ONCE(!rcu_nocb_flush_bypass(my_rdp, NULL, jiffies));
    3463                 :          0 :         raw_spin_lock_rcu_node(my_rnp); /* irqs already disabled. */
    3464                 :            :         /* Leverage recent GPs and set GP for new callbacks. */
    3465   [ #  #  #  # ]:          0 :         needwake = rcu_advance_cbs(my_rnp, rdp) ||
    3466                 :          0 :                    rcu_advance_cbs(my_rnp, my_rdp);
    3467                 :          0 :         rcu_segcblist_merge(&my_rdp->cblist, &rdp->cblist);
    3468   [ #  #  #  # ]:          0 :         needwake = needwake || rcu_advance_cbs(my_rnp, my_rdp);
    3469                 :          0 :         rcu_segcblist_disable(&rdp->cblist);
    3470         [ #  # ]:          0 :         WARN_ON_ONCE(rcu_segcblist_empty(&my_rdp->cblist) !=
    3471                 :            :                      !rcu_segcblist_n_cbs(&my_rdp->cblist));
    3472         [ #  # ]:          0 :         if (rcu_segcblist_is_offloaded(&my_rdp->cblist)) {
    3473                 :          0 :                 raw_spin_unlock_rcu_node(my_rnp); /* irqs remain disabled. */
    3474                 :          0 :                 __call_rcu_nocb_wake(my_rdp, true, flags);
    3475                 :            :         } else {
    3476                 :          0 :                 rcu_nocb_unlock(my_rdp); /* irqs remain disabled. */
    3477                 :          0 :                 raw_spin_unlock_irqrestore_rcu_node(my_rnp, flags);
    3478                 :            :         }
    3479         [ #  # ]:          0 :         if (needwake)
    3480                 :          0 :                 rcu_gp_kthread_wake();
    3481                 :          0 :         lockdep_assert_irqs_enabled();
    3482   [ #  #  #  #  :          0 :         WARN_ONCE(rcu_segcblist_n_cbs(&rdp->cblist) != 0 ||
             #  #  #  # ]
    3483                 :            :                   !rcu_segcblist_empty(&rdp->cblist),
    3484                 :            :                   "rcu_cleanup_dead_cpu: Callbacks on offline CPU %d: qlen=%lu, 1stCB=%p\n",
    3485                 :            :                   cpu, rcu_segcblist_n_cbs(&rdp->cblist),
    3486                 :            :                   rcu_segcblist_first_cb(&rdp->cblist));
    3487                 :            : }
    3488                 :            : #endif
    3489                 :            : 
    3490                 :            : /*
    3491                 :            :  * On non-huge systems, use expedited RCU grace periods to make suspend
    3492                 :            :  * and hibernation run faster.
    3493                 :            :  */
    3494                 :          0 : static int rcu_pm_notify(struct notifier_block *self,
    3495                 :            :                          unsigned long action, void *hcpu)
    3496                 :            : {
    3497      [ #  #  # ]:          0 :         switch (action) {
    3498                 :          0 :         case PM_HIBERNATION_PREPARE:
    3499                 :            :         case PM_SUSPEND_PREPARE:
    3500                 :          0 :                 rcu_expedite_gp();
    3501                 :          0 :                 break;
    3502                 :          0 :         case PM_POST_HIBERNATION:
    3503                 :            :         case PM_POST_SUSPEND:
    3504                 :          0 :                 rcu_unexpedite_gp();
    3505                 :          0 :                 break;
    3506                 :            :         default:
    3507                 :            :                 break;
    3508                 :            :         }
    3509                 :          0 :         return NOTIFY_OK;
    3510                 :            : }
    3511                 :            : 
    3512                 :            : /*
    3513                 :            :  * Spawn the kthreads that handle RCU's grace periods.
    3514                 :            :  */
    3515                 :         30 : static int __init rcu_spawn_gp_kthread(void)
    3516                 :            : {
    3517                 :         30 :         unsigned long flags;
    3518                 :         30 :         int kthread_prio_in = kthread_prio;
    3519                 :         30 :         struct rcu_node *rnp;
    3520                 :         30 :         struct sched_param sp;
    3521                 :         30 :         struct task_struct *t;
    3522                 :            : 
    3523                 :            :         /* Force priority into range. */
    3524                 :         30 :         if (IS_ENABLED(CONFIG_RCU_BOOST) && kthread_prio < 2
    3525                 :            :             && IS_BUILTIN(CONFIG_RCU_TORTURE_TEST))
    3526                 :            :                 kthread_prio = 2;
    3527                 :         30 :         else if (IS_ENABLED(CONFIG_RCU_BOOST) && kthread_prio < 1)
    3528                 :            :                 kthread_prio = 1;
    3529         [ -  + ]:         30 :         else if (kthread_prio < 0)
    3530                 :          0 :                 kthread_prio = 0;
    3531         [ -  + ]:         30 :         else if (kthread_prio > 99)
    3532                 :          0 :                 kthread_prio = 99;
    3533                 :            : 
    3534         [ -  + ]:         30 :         if (kthread_prio != kthread_prio_in)
    3535                 :          0 :                 pr_alert("rcu_spawn_gp_kthread(): Limited prio to %d from %d\n",
    3536                 :            :                          kthread_prio, kthread_prio_in);
    3537                 :            : 
    3538                 :         30 :         rcu_scheduler_fully_active = 1;
    3539                 :         30 :         t = kthread_create(rcu_gp_kthread, NULL, "%s", rcu_state.name);
    3540   [ -  +  -  -  :         30 :         if (WARN_ONCE(IS_ERR(t), "%s: Could not start grace-period kthread, OOM is now expected behavior\n", __func__))
                   +  - ]
    3541                 :            :                 return 0;
    3542         [ -  + ]:         30 :         if (kthread_prio) {
    3543                 :          0 :                 sp.sched_priority = kthread_prio;
    3544                 :          0 :                 sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
    3545                 :            :         }
    3546                 :         30 :         rnp = rcu_get_root();
    3547                 :         30 :         raw_spin_lock_irqsave_rcu_node(rnp, flags);
    3548                 :         30 :         rcu_state.gp_kthread = t;
    3549                 :         30 :         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    3550                 :         30 :         wake_up_process(t);
    3551                 :         30 :         rcu_spawn_nocb_kthreads();
    3552                 :         30 :         rcu_spawn_boost_kthreads();
    3553                 :         30 :         return 0;
    3554                 :            : }
    3555                 :            : early_initcall(rcu_spawn_gp_kthread);
    3556                 :            : 
    3557                 :            : /*
    3558                 :            :  * This function is invoked towards the end of the scheduler's
    3559                 :            :  * initialization process.  Before this is called, the idle task might
    3560                 :            :  * contain synchronous grace-period primitives (during which time, this idle
    3561                 :            :  * task is booting the system, and such primitives are no-ops).  After this
    3562                 :            :  * function is called, any synchronous grace-period primitives are run as
    3563                 :            :  * expedited, with the requesting task driving the grace period forward.
    3564                 :            :  * A later core_initcall() rcu_set_runtime_mode() will switch to full
    3565                 :            :  * runtime RCU functionality.
    3566                 :            :  */
    3567                 :         30 : void rcu_scheduler_starting(void)
    3568                 :            : {
    3569         [ -  + ]:         30 :         WARN_ON(num_online_cpus() != 1);
    3570         [ -  + ]:         30 :         WARN_ON(nr_context_switches() > 0);
    3571                 :         30 :         rcu_test_sync_prims();
    3572                 :         30 :         rcu_scheduler_active = RCU_SCHEDULER_INIT;
    3573                 :         30 :         rcu_test_sync_prims();
    3574                 :         30 : }
    3575                 :            : 
    3576                 :            : /*
    3577                 :            :  * Helper function for rcu_init() that initializes the rcu_state structure.
    3578                 :            :  */
    3579                 :         30 : static void __init rcu_init_one(void)
    3580                 :            : {
    3581                 :         30 :         static const char * const buf[] = RCU_NODE_NAME_INIT;
    3582                 :         30 :         static const char * const fqs[] = RCU_FQS_NAME_INIT;
    3583                 :         30 :         static struct lock_class_key rcu_node_class[RCU_NUM_LVLS];
    3584                 :         30 :         static struct lock_class_key rcu_fqs_class[RCU_NUM_LVLS];
    3585                 :            : 
    3586                 :         30 :         int levelspread[RCU_NUM_LVLS];          /* kids/node in each level. */
    3587                 :         30 :         int cpustride = 1;
    3588                 :         30 :         int i;
    3589                 :         30 :         int j;
    3590                 :         30 :         struct rcu_node *rnp;
    3591                 :            : 
    3592                 :         30 :         BUILD_BUG_ON(RCU_NUM_LVLS > ARRAY_SIZE(buf));  /* Fix buf[] init! */
    3593                 :            : 
    3594                 :            :         /* Silence gcc 4.8 false positive about array index out of range. */
    3595         [ +  - ]:         30 :         if (rcu_num_lvls <= 0 || rcu_num_lvls > RCU_NUM_LVLS)
    3596                 :          0 :                 panic("rcu_init_one: rcu_num_lvls out of range");
    3597                 :            : 
    3598                 :            :         /* Initialize the level-tracking arrays. */
    3599                 :            : 
    3600         [ -  + ]:         30 :         for (i = 1; i < rcu_num_lvls; i++)
    3601                 :          0 :                 rcu_state.level[i] =
    3602                 :          0 :                         rcu_state.level[i - 1] + num_rcu_lvl[i - 1];
    3603                 :         30 :         rcu_init_levelspread(levelspread, num_rcu_lvl);
    3604                 :            : 
    3605                 :            :         /* Initialize the elements themselves, starting from the leaves. */
    3606                 :            : 
    3607         [ +  + ]:         60 :         for (i = rcu_num_lvls - 1; i >= 0; i--) {
    3608                 :         30 :                 cpustride *= levelspread[i];
    3609                 :         30 :                 rnp = rcu_state.level[i];
    3610         [ +  + ]:         60 :                 for (j = 0; j < num_rcu_lvl[i]; j++, rnp++) {
    3611                 :         30 :                         raw_spin_lock_init(&ACCESS_PRIVATE(rnp, lock));
    3612                 :         30 :                         lockdep_set_class_and_name(&ACCESS_PRIVATE(rnp, lock),
    3613                 :            :                                                    &rcu_node_class[i], buf[i]);
    3614                 :         30 :                         raw_spin_lock_init(&rnp->fqslock);
    3615                 :         30 :                         lockdep_set_class_and_name(&rnp->fqslock,
    3616                 :            :                                                    &rcu_fqs_class[i], fqs[i]);
    3617                 :         30 :                         rnp->gp_seq = rcu_state.gp_seq;
    3618                 :         30 :                         rnp->gp_seq_needed = rcu_state.gp_seq;
    3619                 :         30 :                         rnp->completedqs = rcu_state.gp_seq;
    3620                 :         30 :                         rnp->qsmask = 0;
    3621                 :         30 :                         rnp->qsmaskinit = 0;
    3622                 :         30 :                         rnp->grplo = j * cpustride;
    3623                 :         30 :                         rnp->grphi = (j + 1) * cpustride - 1;
    3624         [ -  + ]:         30 :                         if (rnp->grphi >= nr_cpu_ids)
    3625                 :          0 :                                 rnp->grphi = nr_cpu_ids - 1;
    3626         [ +  - ]:         30 :                         if (i == 0) {
    3627                 :         30 :                                 rnp->grpnum = 0;
    3628                 :         30 :                                 rnp->grpmask = 0;
    3629                 :         30 :                                 rnp->parent = NULL;
    3630                 :            :                         } else {
    3631                 :          0 :                                 rnp->grpnum = j % levelspread[i - 1];
    3632                 :          0 :                                 rnp->grpmask = BIT(rnp->grpnum);
    3633                 :          0 :                                 rnp->parent = rcu_state.level[i - 1] +
    3634                 :          0 :                                               j / levelspread[i - 1];
    3635                 :            :                         }
    3636                 :         30 :                         rnp->level = i;
    3637                 :         30 :                         INIT_LIST_HEAD(&rnp->blkd_tasks);
    3638                 :         30 :                         rcu_init_one_nocb(rnp);
    3639                 :         30 :                         init_waitqueue_head(&rnp->exp_wq[0]);
    3640                 :         30 :                         init_waitqueue_head(&rnp->exp_wq[1]);
    3641                 :         30 :                         init_waitqueue_head(&rnp->exp_wq[2]);
    3642                 :         30 :                         init_waitqueue_head(&rnp->exp_wq[3]);
    3643                 :         30 :                         spin_lock_init(&rnp->exp_lock);
    3644                 :            :                 }
    3645                 :            :         }
    3646                 :            : 
    3647                 :         30 :         init_swait_queue_head(&rcu_state.gp_wq);
    3648                 :         30 :         init_swait_queue_head(&rcu_state.expedited_wq);
    3649                 :         30 :         rnp = rcu_first_leaf_node();
    3650         [ +  + ]:         60 :         for_each_possible_cpu(i) {
    3651         [ -  + ]:         30 :                 while (i > rnp->grphi)
    3652                 :          0 :                         rnp++;
    3653                 :         30 :                 per_cpu_ptr(&rcu_data, i)->mynode = rnp;
    3654                 :         30 :                 rcu_boot_init_percpu_data(i);
    3655                 :            :         }
    3656                 :         30 : }
    3657                 :            : 
    3658                 :            : /*
    3659                 :            :  * Compute the rcu_node tree geometry from kernel parameters.  This cannot
    3660                 :            :  * replace the definitions in tree.h because those are needed to size
    3661                 :            :  * the ->node array in the rcu_state structure.
    3662                 :            :  */
    3663                 :         30 : static void __init rcu_init_geometry(void)
    3664                 :            : {
    3665                 :         30 :         ulong d;
    3666                 :         30 :         int i;
    3667                 :         30 :         int rcu_capacity[RCU_NUM_LVLS];
    3668                 :            : 
    3669                 :            :         /*
    3670                 :            :          * Initialize any unspecified boot parameters.
    3671                 :            :          * The default values of jiffies_till_first_fqs and
    3672                 :            :          * jiffies_till_next_fqs are set to the RCU_JIFFIES_TILL_FORCE_QS
    3673                 :            :          * value, which is a function of HZ, then adding one for each
    3674                 :            :          * RCU_JIFFIES_FQS_DIV CPUs that might be on the system.
    3675                 :            :          */
    3676                 :         30 :         d = RCU_JIFFIES_TILL_FORCE_QS + nr_cpu_ids / RCU_JIFFIES_FQS_DIV;
    3677         [ +  - ]:         30 :         if (jiffies_till_first_fqs == ULONG_MAX)
    3678                 :         30 :                 jiffies_till_first_fqs = d;
    3679         [ +  - ]:         30 :         if (jiffies_till_next_fqs == ULONG_MAX)
    3680                 :         30 :                 jiffies_till_next_fqs = d;
    3681                 :         30 :         adjust_jiffies_till_sched_qs();
    3682                 :            : 
    3683                 :            :         /* If the compile-time values are accurate, just leave. */
    3684         [ +  - ]:         30 :         if (rcu_fanout_leaf == RCU_FANOUT_LEAF &&
    3685         [ +  - ]:         30 :             nr_cpu_ids == NR_CPUS)
    3686                 :          0 :                 return;
    3687                 :         30 :         pr_info("Adjusting geometry for rcu_fanout_leaf=%d, nr_cpu_ids=%u\n",
    3688                 :            :                 rcu_fanout_leaf, nr_cpu_ids);
    3689                 :            : 
    3690                 :            :         /*
    3691                 :            :          * The boot-time rcu_fanout_leaf parameter must be at least two
    3692                 :            :          * and cannot exceed the number of bits in the rcu_node masks.
    3693                 :            :          * Complain and fall back to the compile-time values if this
    3694                 :            :          * limit is exceeded.
    3695                 :            :          */
    3696         [ -  + ]:         30 :         if (rcu_fanout_leaf < 2 ||
    3697                 :            :             rcu_fanout_leaf > sizeof(unsigned long) * 8) {
    3698                 :          0 :                 rcu_fanout_leaf = RCU_FANOUT_LEAF;
    3699                 :          0 :                 WARN_ON(1);
    3700                 :          0 :                 return;
    3701                 :            :         }
    3702                 :            : 
    3703                 :            :         /*
    3704                 :            :          * Compute number of nodes that can be handled an rcu_node tree
    3705                 :            :          * with the given number of levels.
    3706                 :            :          */
    3707                 :         30 :         rcu_capacity[0] = rcu_fanout_leaf;
    3708         [ +  + ]:         60 :         for (i = 1; i < RCU_NUM_LVLS; i++)
    3709                 :         30 :                 rcu_capacity[i] = rcu_capacity[i - 1] * RCU_FANOUT;
    3710                 :            : 
    3711                 :            :         /*
    3712                 :            :          * The tree must be able to accommodate the configured number of CPUs.
    3713                 :            :          * If this limit is exceeded, fall back to the compile-time values.
    3714                 :            :          */
    3715         [ -  + ]:         30 :         if (nr_cpu_ids > rcu_capacity[RCU_NUM_LVLS - 1]) {
    3716                 :          0 :                 rcu_fanout_leaf = RCU_FANOUT_LEAF;
    3717                 :          0 :                 WARN_ON(1);
    3718                 :          0 :                 return;
    3719                 :            :         }
    3720                 :            : 
    3721                 :            :         /* Calculate the number of levels in the tree. */
    3722         [ -  + ]:         30 :         for (i = 0; nr_cpu_ids > rcu_capacity[i]; i++) {
    3723                 :          0 :         }
    3724                 :         30 :         rcu_num_lvls = i + 1;
    3725                 :            : 
    3726                 :            :         /* Calculate the number of rcu_nodes at each level of the tree. */
    3727         [ +  + ]:         60 :         for (i = 0; i < rcu_num_lvls; i++) {
    3728                 :         30 :                 int cap = rcu_capacity[(rcu_num_lvls - 1) - i];
    3729                 :         30 :                 num_rcu_lvl[i] = DIV_ROUND_UP(nr_cpu_ids, cap);
    3730                 :            :         }
    3731                 :            : 
    3732                 :            :         /* Calculate the total number of rcu_node structures. */
    3733                 :         30 :         rcu_num_nodes = 0;
    3734         [ +  + ]:         60 :         for (i = 0; i < rcu_num_lvls; i++)
    3735                 :         30 :                 rcu_num_nodes += num_rcu_lvl[i];
    3736                 :            : }
    3737                 :            : 
    3738                 :            : /*
    3739                 :            :  * Dump out the structure of the rcu_node combining tree associated
    3740                 :            :  * with the rcu_state structure.
    3741                 :            :  */
    3742                 :          0 : static void __init rcu_dump_rcu_node_tree(void)
    3743                 :            : {
    3744                 :          0 :         int level = 0;
    3745                 :          0 :         struct rcu_node *rnp;
    3746                 :            : 
    3747                 :          0 :         pr_info("rcu_node tree layout dump\n");
    3748                 :          0 :         pr_info(" ");
    3749         [ #  # ]:          0 :         rcu_for_each_node_breadth_first(rnp) {
    3750         [ #  # ]:          0 :                 if (rnp->level != level) {
    3751                 :          0 :                         pr_cont("\n");
    3752                 :          0 :                         pr_info(" ");
    3753                 :          0 :                         level = rnp->level;
    3754                 :            :                 }
    3755                 :          0 :                 pr_cont("%d:%d ^%d  ", rnp->grplo, rnp->grphi, rnp->grpnum);
    3756                 :            :         }
    3757                 :          0 :         pr_cont("\n");
    3758                 :          0 : }
    3759                 :            : 
    3760                 :            : struct workqueue_struct *rcu_gp_wq;
    3761                 :            : struct workqueue_struct *rcu_par_gp_wq;
    3762                 :            : 
    3763                 :         30 : static void __init kfree_rcu_batch_init(void)
    3764                 :            : {
    3765                 :         30 :         int cpu;
    3766                 :         30 :         int i;
    3767                 :            : 
    3768         [ +  + ]:         60 :         for_each_possible_cpu(cpu) {
    3769                 :         30 :                 struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
    3770                 :            : 
    3771                 :         30 :                 spin_lock_init(&krcp->lock);
    3772         [ +  + ]:         90 :                 for (i = 0; i < KFREE_N_BATCHES; i++)
    3773                 :         60 :                         krcp->krw_arr[i].krcp = krcp;
    3774                 :         30 :                 INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
    3775                 :         30 :                 krcp->initialized = true;
    3776                 :            :         }
    3777                 :         30 : }
    3778                 :            : 
    3779                 :         30 : void __init rcu_init(void)
    3780                 :            : {
    3781                 :         30 :         int cpu;
    3782                 :            : 
    3783                 :         30 :         rcu_early_boot_tests();
    3784                 :            : 
    3785                 :         30 :         kfree_rcu_batch_init();
    3786                 :         30 :         rcu_bootup_announce();
    3787                 :         30 :         rcu_init_geometry();
    3788                 :         30 :         rcu_init_one();
    3789         [ -  + ]:         30 :         if (dump_tree)
    3790                 :          0 :                 rcu_dump_rcu_node_tree();
    3791         [ +  - ]:         30 :         if (use_softirq)
    3792                 :         30 :                 open_softirq(RCU_SOFTIRQ, rcu_core_si);
    3793                 :            : 
    3794                 :            :         /*
    3795                 :            :          * We don't need protection against CPU-hotplug here because
    3796                 :            :          * this is called early in boot, before either interrupts
    3797                 :            :          * or the scheduler are operational.
    3798                 :            :          */
    3799                 :         30 :         pm_notifier(rcu_pm_notify, 0);
    3800         [ +  + ]:         90 :         for_each_online_cpu(cpu) {
    3801                 :         30 :                 rcutree_prepare_cpu(cpu);
    3802                 :         30 :                 rcu_cpu_starting(cpu);
    3803                 :         30 :                 rcutree_online_cpu(cpu);
    3804                 :            :         }
    3805                 :            : 
    3806                 :            :         /* Create workqueue for expedited GPs and for Tree SRCU. */
    3807                 :         30 :         rcu_gp_wq = alloc_workqueue("rcu_gp", WQ_MEM_RECLAIM, 0);
    3808         [ -  + ]:         30 :         WARN_ON(!rcu_gp_wq);
    3809                 :         30 :         rcu_par_gp_wq = alloc_workqueue("rcu_par_gp", WQ_MEM_RECLAIM, 0);
    3810         [ -  + ]:         30 :         WARN_ON(!rcu_par_gp_wq);
    3811                 :         30 :         srcu_init();
    3812                 :         30 : }
    3813                 :            : 
    3814                 :            : #include "tree_stall.h"
    3815                 :            : #include "tree_exp.h"
    3816                 :            : #include "tree_plugin.h"

Generated by: LCOV version 1.14