LCOV - code coverage report
Current view: top level - kernel/rcu - tree_plugin.h (source / functions) Hit Total Coverage
Test: Real Lines: 75 91 82.4 %
Date: 2020-10-17 15:46:16 Functions: 1 15 6.7 %
Legend: Neither, QEMU, Real, Both Branches: 0 0 -

           Branch data     Line data    Source code
       1                 :            : /* SPDX-License-Identifier: GPL-2.0+ */
       2                 :            : /*
       3                 :            :  * Read-Copy Update mechanism for mutual exclusion (tree-based version)
       4                 :            :  * Internal non-public definitions that provide either classic
       5                 :            :  * or preemptible semantics.
       6                 :            :  *
       7                 :            :  * Copyright Red Hat, 2009
       8                 :            :  * Copyright IBM Corporation, 2009
       9                 :            :  *
      10                 :            :  * Author: Ingo Molnar <mingo@elte.hu>
      11                 :            :  *         Paul E. McKenney <paulmck@linux.ibm.com>
      12                 :            :  */
      13                 :            : 
      14                 :            : #include "../locking/rtmutex_common.h"
      15                 :            : 
      16                 :            : #ifdef CONFIG_RCU_NOCB_CPU
      17                 :            : static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */
      18                 :            : static bool __read_mostly rcu_nocb_poll;    /* Offload kthread are to poll. */
      19                 :            : #endif /* #ifdef CONFIG_RCU_NOCB_CPU */
      20                 :            : 
      21                 :            : /*
      22                 :            :  * Check the RCU kernel configuration parameters and print informative
      23                 :            :  * messages about anything out of the ordinary.
      24                 :            :  */
      25                 :          3 : static void __init rcu_bootup_announce_oddness(void)
      26                 :            : {
      27                 :            :         if (IS_ENABLED(CONFIG_RCU_TRACE))
      28                 :            :                 pr_info("\tRCU event tracing is enabled.\n");
      29                 :            :         if ((IS_ENABLED(CONFIG_64BIT) && RCU_FANOUT != 64) ||
      30                 :            :             (!IS_ENABLED(CONFIG_64BIT) && RCU_FANOUT != 32))
      31                 :            :                 pr_info("\tCONFIG_RCU_FANOUT set to non-default value of %d.\n",
      32                 :            :                         RCU_FANOUT);
      33                 :          3 :         if (rcu_fanout_exact)
      34                 :          0 :                 pr_info("\tHierarchical RCU autobalancing is disabled.\n");
      35                 :            :         if (IS_ENABLED(CONFIG_RCU_FAST_NO_HZ))
      36                 :            :                 pr_info("\tRCU dyntick-idle grace-period acceleration is enabled.\n");
      37                 :            :         if (IS_ENABLED(CONFIG_PROVE_RCU))
      38                 :            :                 pr_info("\tRCU lockdep checking is enabled.\n");
      39                 :            :         if (RCU_NUM_LVLS >= 4)
      40                 :            :                 pr_info("\tFour(or more)-level hierarchy is enabled.\n");
      41                 :            :         if (RCU_FANOUT_LEAF != 16)
      42                 :            :                 pr_info("\tBuild-time adjustment of leaf fanout to %d.\n",
      43                 :            :                         RCU_FANOUT_LEAF);
      44                 :          3 :         if (rcu_fanout_leaf != RCU_FANOUT_LEAF)
      45                 :          0 :                 pr_info("\tBoot-time adjustment of leaf fanout to %d.\n",
      46                 :            :                         rcu_fanout_leaf);
      47                 :          3 :         if (nr_cpu_ids != NR_CPUS)
      48                 :          0 :                 pr_info("\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%u.\n", NR_CPUS, nr_cpu_ids);
      49                 :            : #ifdef CONFIG_RCU_BOOST
      50                 :            :         pr_info("\tRCU priority boosting: priority %d delay %d ms.\n",
      51                 :            :                 kthread_prio, CONFIG_RCU_BOOST_DELAY);
      52                 :            : #endif
      53                 :          3 :         if (blimit != DEFAULT_RCU_BLIMIT)
      54                 :          0 :                 pr_info("\tBoot-time adjustment of callback invocation limit to %ld.\n", blimit);
      55                 :          3 :         if (qhimark != DEFAULT_RCU_QHIMARK)
      56                 :          0 :                 pr_info("\tBoot-time adjustment of callback high-water mark to %ld.\n", qhimark);
      57                 :          3 :         if (qlowmark != DEFAULT_RCU_QLOMARK)
      58                 :          0 :                 pr_info("\tBoot-time adjustment of callback low-water mark to %ld.\n", qlowmark);
      59                 :          3 :         if (jiffies_till_first_fqs != ULONG_MAX)
      60                 :          0 :                 pr_info("\tBoot-time adjustment of first FQS scan delay to %ld jiffies.\n", jiffies_till_first_fqs);
      61                 :          3 :         if (jiffies_till_next_fqs != ULONG_MAX)
      62                 :          0 :                 pr_info("\tBoot-time adjustment of subsequent FQS scan delay to %ld jiffies.\n", jiffies_till_next_fqs);
      63                 :          3 :         if (jiffies_till_sched_qs != ULONG_MAX)
      64                 :          0 :                 pr_info("\tBoot-time adjustment of scheduler-enlistment delay to %ld jiffies.\n", jiffies_till_sched_qs);
      65                 :          3 :         if (rcu_kick_kthreads)
      66                 :          0 :                 pr_info("\tKick kthreads if too-long grace period.\n");
      67                 :            :         if (IS_ENABLED(CONFIG_DEBUG_OBJECTS_RCU_HEAD))
      68                 :            :                 pr_info("\tRCU callback double-/use-after-free debug enabled.\n");
      69                 :          3 :         if (gp_preinit_delay)
      70                 :          0 :                 pr_info("\tRCU debug GP pre-init slowdown %d jiffies.\n", gp_preinit_delay);
      71                 :          3 :         if (gp_init_delay)
      72                 :          0 :                 pr_info("\tRCU debug GP init slowdown %d jiffies.\n", gp_init_delay);
      73                 :          3 :         if (gp_cleanup_delay)
      74                 :          0 :                 pr_info("\tRCU debug GP init slowdown %d jiffies.\n", gp_cleanup_delay);
      75                 :          3 :         if (!use_softirq)
      76                 :          0 :                 pr_info("\tRCU_SOFTIRQ processing moved to rcuc kthreads.\n");
      77                 :            :         if (IS_ENABLED(CONFIG_RCU_EQS_DEBUG))
      78                 :            :                 pr_info("\tRCU debug extended QS entry/exit.\n");
      79                 :          3 :         rcupdate_announce_bootup_oddness();
      80                 :          3 : }
      81                 :            : 
      82                 :            : #ifdef CONFIG_PREEMPT_RCU
      83                 :            : 
      84                 :            : static void rcu_report_exp_rnp(struct rcu_node *rnp, bool wake);
      85                 :            : static void rcu_read_unlock_special(struct task_struct *t);
      86                 :            : 
      87                 :            : /*
      88                 :            :  * Tell them what RCU they are running.
      89                 :            :  */
      90                 :            : static void __init rcu_bootup_announce(void)
      91                 :            : {
      92                 :            :         pr_info("Preemptible hierarchical RCU implementation.\n");
      93                 :            :         rcu_bootup_announce_oddness();
      94                 :            : }
      95                 :            : 
      96                 :            : /* Flags for rcu_preempt_ctxt_queue() decision table. */
      97                 :            : #define RCU_GP_TASKS    0x8
      98                 :            : #define RCU_EXP_TASKS   0x4
      99                 :            : #define RCU_GP_BLKD     0x2
     100                 :            : #define RCU_EXP_BLKD    0x1
     101                 :            : 
     102                 :            : /*
     103                 :            :  * Queues a task preempted within an RCU-preempt read-side critical
     104                 :            :  * section into the appropriate location within the ->blkd_tasks list,
     105                 :            :  * depending on the states of any ongoing normal and expedited grace
     106                 :            :  * periods.  The ->gp_tasks pointer indicates which element the normal
     107                 :            :  * grace period is waiting on (NULL if none), and the ->exp_tasks pointer
     108                 :            :  * indicates which element the expedited grace period is waiting on (again,
     109                 :            :  * NULL if none).  If a grace period is waiting on a given element in the
     110                 :            :  * ->blkd_tasks list, it also waits on all subsequent elements.  Thus,
     111                 :            :  * adding a task to the tail of the list blocks any grace period that is
     112                 :            :  * already waiting on one of the elements.  In contrast, adding a task
     113                 :            :  * to the head of the list won't block any grace period that is already
     114                 :            :  * waiting on one of the elements.
     115                 :            :  *
     116                 :            :  * This queuing is imprecise, and can sometimes make an ongoing grace
     117                 :            :  * period wait for a task that is not strictly speaking blocking it.
     118                 :            :  * Given the choice, we needlessly block a normal grace period rather than
     119                 :            :  * blocking an expedited grace period.
     120                 :            :  *
     121                 :            :  * Note that an endless sequence of expedited grace periods still cannot
     122                 :            :  * indefinitely postpone a normal grace period.  Eventually, all of the
     123                 :            :  * fixed number of preempted tasks blocking the normal grace period that are
     124                 :            :  * not also blocking the expedited grace period will resume and complete
     125                 :            :  * their RCU read-side critical sections.  At that point, the ->gp_tasks
     126                 :            :  * pointer will equal the ->exp_tasks pointer, at which point the end of
     127                 :            :  * the corresponding expedited grace period will also be the end of the
     128                 :            :  * normal grace period.
     129                 :            :  */
     130                 :            : static void rcu_preempt_ctxt_queue(struct rcu_node *rnp, struct rcu_data *rdp)
     131                 :            :         __releases(rnp->lock) /* But leaves rrupts disabled. */
     132                 :            : {
     133                 :            :         int blkd_state = (rnp->gp_tasks ? RCU_GP_TASKS : 0) +
     134                 :            :                          (rnp->exp_tasks ? RCU_EXP_TASKS : 0) +
     135                 :            :                          (rnp->qsmask & rdp->grpmask ? RCU_GP_BLKD : 0) +
     136                 :            :                          (rnp->expmask & rdp->grpmask ? RCU_EXP_BLKD : 0);
     137                 :            :         struct task_struct *t = current;
     138                 :            : 
     139                 :            :         raw_lockdep_assert_held_rcu_node(rnp);
     140                 :            :         WARN_ON_ONCE(rdp->mynode != rnp);
     141                 :            :         WARN_ON_ONCE(!rcu_is_leaf_node(rnp));
     142                 :            :         /* RCU better not be waiting on newly onlined CPUs! */
     143                 :            :         WARN_ON_ONCE(rnp->qsmaskinitnext & ~rnp->qsmaskinit & rnp->qsmask &
     144                 :            :                      rdp->grpmask);
     145                 :            : 
     146                 :            :         /*
     147                 :            :          * Decide where to queue the newly blocked task.  In theory,
     148                 :            :          * this could be an if-statement.  In practice, when I tried
     149                 :            :          * that, it was quite messy.
     150                 :            :          */
     151                 :            :         switch (blkd_state) {
     152                 :            :         case 0:
     153                 :            :         case                RCU_EXP_TASKS:
     154                 :            :         case                RCU_EXP_TASKS + RCU_GP_BLKD:
     155                 :            :         case RCU_GP_TASKS:
     156                 :            :         case RCU_GP_TASKS + RCU_EXP_TASKS:
     157                 :            : 
     158                 :            :                 /*
     159                 :            :                  * Blocking neither GP, or first task blocking the normal
     160                 :            :                  * GP but not blocking the already-waiting expedited GP.
     161                 :            :                  * Queue at the head of the list to avoid unnecessarily
     162                 :            :                  * blocking the already-waiting GPs.
     163                 :            :                  */
     164                 :            :                 list_add(&t->rcu_node_entry, &rnp->blkd_tasks);
     165                 :            :                 break;
     166                 :            : 
     167                 :            :         case                                              RCU_EXP_BLKD:
     168                 :            :         case                                RCU_GP_BLKD:
     169                 :            :         case                                RCU_GP_BLKD + RCU_EXP_BLKD:
     170                 :            :         case RCU_GP_TASKS +                               RCU_EXP_BLKD:
     171                 :            :         case RCU_GP_TASKS +                 RCU_GP_BLKD + RCU_EXP_BLKD:
     172                 :            :         case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
     173                 :            : 
     174                 :            :                 /*
     175                 :            :                  * First task arriving that blocks either GP, or first task
     176                 :            :                  * arriving that blocks the expedited GP (with the normal
     177                 :            :                  * GP already waiting), or a task arriving that blocks
     178                 :            :                  * both GPs with both GPs already waiting.  Queue at the
     179                 :            :                  * tail of the list to avoid any GP waiting on any of the
     180                 :            :                  * already queued tasks that are not blocking it.
     181                 :            :                  */
     182                 :            :                 list_add_tail(&t->rcu_node_entry, &rnp->blkd_tasks);
     183                 :            :                 break;
     184                 :            : 
     185                 :            :         case                RCU_EXP_TASKS +               RCU_EXP_BLKD:
     186                 :            :         case                RCU_EXP_TASKS + RCU_GP_BLKD + RCU_EXP_BLKD:
     187                 :            :         case RCU_GP_TASKS + RCU_EXP_TASKS +               RCU_EXP_BLKD:
     188                 :            : 
     189                 :            :                 /*
     190                 :            :                  * Second or subsequent task blocking the expedited GP.
     191                 :            :                  * The task either does not block the normal GP, or is the
     192                 :            :                  * first task blocking the normal GP.  Queue just after
     193                 :            :                  * the first task blocking the expedited GP.
     194                 :            :                  */
     195                 :            :                 list_add(&t->rcu_node_entry, rnp->exp_tasks);
     196                 :            :                 break;
     197                 :            : 
     198                 :            :         case RCU_GP_TASKS +                 RCU_GP_BLKD:
     199                 :            :         case RCU_GP_TASKS + RCU_EXP_TASKS + RCU_GP_BLKD:
     200                 :            : 
     201                 :            :                 /*
     202                 :            :                  * Second or subsequent task blocking the normal GP.
     203                 :            :                  * The task does not block the expedited GP. Queue just
     204                 :            :                  * after the first task blocking the normal GP.
     205                 :            :                  */
     206                 :            :                 list_add(&t->rcu_node_entry, rnp->gp_tasks);
     207                 :            :                 break;
     208                 :            : 
     209                 :            :         default:
     210                 :            : 
     211                 :            :                 /* Yet another exercise in excessive paranoia. */
     212                 :            :                 WARN_ON_ONCE(1);
     213                 :            :                 break;
     214                 :            :         }
     215                 :            : 
     216                 :            :         /*
     217                 :            :          * We have now queued the task.  If it was the first one to
     218                 :            :          * block either grace period, update the ->gp_tasks and/or
     219                 :            :          * ->exp_tasks pointers, respectively, to reference the newly
     220                 :            :          * blocked tasks.
     221                 :            :          */
     222                 :            :         if (!rnp->gp_tasks && (blkd_state & RCU_GP_BLKD)) {
     223                 :            :                 WRITE_ONCE(rnp->gp_tasks, &t->rcu_node_entry);
     224                 :            :                 WARN_ON_ONCE(rnp->completedqs == rnp->gp_seq);
     225                 :            :         }
     226                 :            :         if (!rnp->exp_tasks && (blkd_state & RCU_EXP_BLKD))
     227                 :            :                 rnp->exp_tasks = &t->rcu_node_entry;
     228                 :            :         WARN_ON_ONCE(!(blkd_state & RCU_GP_BLKD) !=
     229                 :            :                      !(rnp->qsmask & rdp->grpmask));
     230                 :            :         WARN_ON_ONCE(!(blkd_state & RCU_EXP_BLKD) !=
     231                 :            :                      !(rnp->expmask & rdp->grpmask));
     232                 :            :         raw_spin_unlock_rcu_node(rnp); /* interrupts remain disabled. */
     233                 :            : 
     234                 :            :         /*
     235                 :            :          * Report the quiescent state for the expedited GP.  This expedited
     236                 :            :          * GP should not be able to end until we report, so there should be
     237                 :            :          * no need to check for a subsequent expedited GP.  (Though we are
     238                 :            :          * still in a quiescent state in any case.)
     239                 :            :          */
     240                 :            :         if (blkd_state & RCU_EXP_BLKD && rdp->exp_deferred_qs)
     241                 :            :                 rcu_report_exp_rdp(rdp);
     242                 :            :         else
     243                 :            :                 WARN_ON_ONCE(rdp->exp_deferred_qs);
     244                 :            : }
     245                 :            : 
     246                 :            : /*
     247                 :            :  * Record a preemptible-RCU quiescent state for the specified CPU.
     248                 :            :  * Note that this does not necessarily mean that the task currently running
     249                 :            :  * on the CPU is in a quiescent state:  Instead, it means that the current
     250                 :            :  * grace period need not wait on any RCU read-side critical section that
     251                 :            :  * starts later on this CPU.  It also means that if the current task is
     252                 :            :  * in an RCU read-side critical section, it has already added itself to
     253                 :            :  * some leaf rcu_node structure's ->blkd_tasks list.  In addition to the
     254                 :            :  * current task, there might be any number of other tasks blocked while
     255                 :            :  * in an RCU read-side critical section.
     256                 :            :  *
     257                 :            :  * Callers to this function must disable preemption.
     258                 :            :  */
     259                 :            : static void rcu_qs(void)
     260                 :            : {
     261                 :            :         RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!\n");
     262                 :            :         if (__this_cpu_read(rcu_data.cpu_no_qs.s)) {
     263                 :            :                 trace_rcu_grace_period(TPS("rcu_preempt"),
     264                 :            :                                        __this_cpu_read(rcu_data.gp_seq),
     265                 :            :                                        TPS("cpuqs"));
     266                 :            :                 __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false);
     267                 :            :                 barrier(); /* Coordinate with rcu_flavor_sched_clock_irq(). */
     268                 :            :                 WRITE_ONCE(current->rcu_read_unlock_special.b.need_qs, false);
     269                 :            :         }
     270                 :            : }
     271                 :            : 
     272                 :            : /*
     273                 :            :  * We have entered the scheduler, and the current task might soon be
     274                 :            :  * context-switched away from.  If this task is in an RCU read-side
     275                 :            :  * critical section, we will no longer be able to rely on the CPU to
     276                 :            :  * record that fact, so we enqueue the task on the blkd_tasks list.
     277                 :            :  * The task will dequeue itself when it exits the outermost enclosing
     278                 :            :  * RCU read-side critical section.  Therefore, the current grace period
     279                 :            :  * cannot be permitted to complete until the blkd_tasks list entries
     280                 :            :  * predating the current grace period drain, in other words, until
     281                 :            :  * rnp->gp_tasks becomes NULL.
     282                 :            :  *
     283                 :            :  * Caller must disable interrupts.
     284                 :            :  */
     285                 :            : void rcu_note_context_switch(bool preempt)
     286                 :            : {
     287                 :            :         struct task_struct *t = current;
     288                 :            :         struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
     289                 :            :         struct rcu_node *rnp;
     290                 :            : 
     291                 :            :         trace_rcu_utilization(TPS("Start context switch"));
     292                 :            :         lockdep_assert_irqs_disabled();
     293                 :            :         WARN_ON_ONCE(!preempt && t->rcu_read_lock_nesting > 0);
     294                 :            :         if (t->rcu_read_lock_nesting > 0 &&
     295                 :            :             !t->rcu_read_unlock_special.b.blocked) {
     296                 :            : 
     297                 :            :                 /* Possibly blocking in an RCU read-side critical section. */
     298                 :            :                 rnp = rdp->mynode;
     299                 :            :                 raw_spin_lock_rcu_node(rnp);
     300                 :            :                 t->rcu_read_unlock_special.b.blocked = true;
     301                 :            :                 t->rcu_blocked_node = rnp;
     302                 :            : 
     303                 :            :                 /*
     304                 :            :                  * Verify the CPU's sanity, trace the preemption, and
     305                 :            :                  * then queue the task as required based on the states
     306                 :            :                  * of any ongoing and expedited grace periods.
     307                 :            :                  */
     308                 :            :                 WARN_ON_ONCE((rdp->grpmask & rcu_rnp_online_cpus(rnp)) == 0);
     309                 :            :                 WARN_ON_ONCE(!list_empty(&t->rcu_node_entry));
     310                 :            :                 trace_rcu_preempt_task(rcu_state.name,
     311                 :            :                                        t->pid,
     312                 :            :                                        (rnp->qsmask & rdp->grpmask)
     313                 :            :                                        ? rnp->gp_seq
     314                 :            :                                        : rcu_seq_snap(&rnp->gp_seq));
     315                 :            :                 rcu_preempt_ctxt_queue(rnp, rdp);
     316                 :            :         } else {
     317                 :            :                 rcu_preempt_deferred_qs(t);
     318                 :            :         }
     319                 :            : 
     320                 :            :         /*
     321                 :            :          * Either we were not in an RCU read-side critical section to
     322                 :            :          * begin with, or we have now recorded that critical section
     323                 :            :          * globally.  Either way, we can now note a quiescent state
     324                 :            :          * for this CPU.  Again, if we were in an RCU read-side critical
     325                 :            :          * section, and if that critical section was blocking the current
     326                 :            :          * grace period, then the fact that the task has been enqueued
     327                 :            :          * means that we continue to block the current grace period.
     328                 :            :          */
     329                 :            :         rcu_qs();
     330                 :            :         if (rdp->exp_deferred_qs)
     331                 :            :                 rcu_report_exp_rdp(rdp);
     332                 :            :         trace_rcu_utilization(TPS("End context switch"));
     333                 :            : }
     334                 :            : EXPORT_SYMBOL_GPL(rcu_note_context_switch);
     335                 :            : 
     336                 :            : /*
     337                 :            :  * Check for preempted RCU readers blocking the current grace period
     338                 :            :  * for the specified rcu_node structure.  If the caller needs a reliable
     339                 :            :  * answer, it must hold the rcu_node's ->lock.
     340                 :            :  */
     341                 :            : static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
     342                 :            : {
     343                 :            :         return READ_ONCE(rnp->gp_tasks) != NULL;
     344                 :            : }
     345                 :            : 
     346                 :            : /* Bias and limit values for ->rcu_read_lock_nesting. */
     347                 :            : #define RCU_NEST_BIAS INT_MAX
     348                 :            : #define RCU_NEST_NMAX (-INT_MAX / 2)
     349                 :            : #define RCU_NEST_PMAX (INT_MAX / 2)
     350                 :            : 
     351                 :            : /*
     352                 :            :  * Preemptible RCU implementation for rcu_read_lock().
     353                 :            :  * Just increment ->rcu_read_lock_nesting, shared state will be updated
     354                 :            :  * if we block.
     355                 :            :  */
     356                 :            : void __rcu_read_lock(void)
     357                 :            : {
     358                 :            :         current->rcu_read_lock_nesting++;
     359                 :            :         if (IS_ENABLED(CONFIG_PROVE_LOCKING))
     360                 :            :                 WARN_ON_ONCE(current->rcu_read_lock_nesting > RCU_NEST_PMAX);
     361                 :            :         barrier();  /* critical section after entry code. */
     362                 :            : }
     363                 :            : EXPORT_SYMBOL_GPL(__rcu_read_lock);
     364                 :            : 
     365                 :            : /*
     366                 :            :  * Preemptible RCU implementation for rcu_read_unlock().
     367                 :            :  * Decrement ->rcu_read_lock_nesting.  If the result is zero (outermost
     368                 :            :  * rcu_read_unlock()) and ->rcu_read_unlock_special is non-zero, then
     369                 :            :  * invoke rcu_read_unlock_special() to clean up after a context switch
     370                 :            :  * in an RCU read-side critical section and other special cases.
     371                 :            :  */
     372                 :            : void __rcu_read_unlock(void)
     373                 :            : {
     374                 :            :         struct task_struct *t = current;
     375                 :            : 
     376                 :            :         if (t->rcu_read_lock_nesting != 1) {
     377                 :            :                 --t->rcu_read_lock_nesting;
     378                 :            :         } else {
     379                 :            :                 barrier();  /* critical section before exit code. */
     380                 :            :                 t->rcu_read_lock_nesting = -RCU_NEST_BIAS;
     381                 :            :                 barrier();  /* assign before ->rcu_read_unlock_special load */
     382                 :            :                 if (unlikely(READ_ONCE(t->rcu_read_unlock_special.s)))
     383                 :            :                         rcu_read_unlock_special(t);
     384                 :            :                 barrier();  /* ->rcu_read_unlock_special load before assign */
     385                 :            :                 t->rcu_read_lock_nesting = 0;
     386                 :            :         }
     387                 :            :         if (IS_ENABLED(CONFIG_PROVE_LOCKING)) {
     388                 :            :                 int rrln = t->rcu_read_lock_nesting;
     389                 :            : 
     390                 :            :                 WARN_ON_ONCE(rrln < 0 && rrln > RCU_NEST_NMAX);
     391                 :            :         }
     392                 :            : }
     393                 :            : EXPORT_SYMBOL_GPL(__rcu_read_unlock);
     394                 :            : 
     395                 :            : /*
     396                 :            :  * Advance a ->blkd_tasks-list pointer to the next entry, instead
     397                 :            :  * returning NULL if at the end of the list.
     398                 :            :  */
     399                 :            : static struct list_head *rcu_next_node_entry(struct task_struct *t,
     400                 :            :                                              struct rcu_node *rnp)
     401                 :            : {
     402                 :            :         struct list_head *np;
     403                 :            : 
     404                 :            :         np = t->rcu_node_entry.next;
     405                 :            :         if (np == &rnp->blkd_tasks)
     406                 :            :                 np = NULL;
     407                 :            :         return np;
     408                 :            : }
     409                 :            : 
     410                 :            : /*
     411                 :            :  * Return true if the specified rcu_node structure has tasks that were
     412                 :            :  * preempted within an RCU read-side critical section.
     413                 :            :  */
     414                 :            : static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
     415                 :            : {
     416                 :            :         return !list_empty(&rnp->blkd_tasks);
     417                 :            : }
     418                 :            : 
     419                 :            : /*
     420                 :            :  * Report deferred quiescent states.  The deferral time can
     421                 :            :  * be quite short, for example, in the case of the call from
     422                 :            :  * rcu_read_unlock_special().
     423                 :            :  */
     424                 :            : static void
     425                 :            : rcu_preempt_deferred_qs_irqrestore(struct task_struct *t, unsigned long flags)
     426                 :            : {
     427                 :            :         bool empty_exp;
     428                 :            :         bool empty_norm;
     429                 :            :         bool empty_exp_now;
     430                 :            :         struct list_head *np;
     431                 :            :         bool drop_boost_mutex = false;
     432                 :            :         struct rcu_data *rdp;
     433                 :            :         struct rcu_node *rnp;
     434                 :            :         union rcu_special special;
     435                 :            : 
     436                 :            :         /*
     437                 :            :          * If RCU core is waiting for this CPU to exit its critical section,
     438                 :            :          * report the fact that it has exited.  Because irqs are disabled,
     439                 :            :          * t->rcu_read_unlock_special cannot change.
     440                 :            :          */
     441                 :            :         special = t->rcu_read_unlock_special;
     442                 :            :         rdp = this_cpu_ptr(&rcu_data);
     443                 :            :         if (!special.s && !rdp->exp_deferred_qs) {
     444                 :            :                 local_irq_restore(flags);
     445                 :            :                 return;
     446                 :            :         }
     447                 :            :         t->rcu_read_unlock_special.b.deferred_qs = false;
     448                 :            :         if (special.b.need_qs) {
     449                 :            :                 rcu_qs();
     450                 :            :                 t->rcu_read_unlock_special.b.need_qs = false;
     451                 :            :                 if (!t->rcu_read_unlock_special.s && !rdp->exp_deferred_qs) {
     452                 :            :                         local_irq_restore(flags);
     453                 :            :                         return;
     454                 :            :                 }
     455                 :            :         }
     456                 :            : 
     457                 :            :         /*
     458                 :            :          * Respond to a request by an expedited grace period for a
     459                 :            :          * quiescent state from this CPU.  Note that requests from
     460                 :            :          * tasks are handled when removing the task from the
     461                 :            :          * blocked-tasks list below.
     462                 :            :          */
     463                 :            :         if (rdp->exp_deferred_qs) {
     464                 :            :                 rcu_report_exp_rdp(rdp);
     465                 :            :                 if (!t->rcu_read_unlock_special.s) {
     466                 :            :                         local_irq_restore(flags);
     467                 :            :                         return;
     468                 :            :                 }
     469                 :            :         }
     470                 :            : 
     471                 :            :         /* Clean up if blocked during RCU read-side critical section. */
     472                 :            :         if (special.b.blocked) {
     473                 :            :                 t->rcu_read_unlock_special.b.blocked = false;
     474                 :            : 
     475                 :            :                 /*
     476                 :            :                  * Remove this task from the list it blocked on.  The task
     477                 :            :                  * now remains queued on the rcu_node corresponding to the
     478                 :            :                  * CPU it first blocked on, so there is no longer any need
     479                 :            :                  * to loop.  Retain a WARN_ON_ONCE() out of sheer paranoia.
     480                 :            :                  */
     481                 :            :                 rnp = t->rcu_blocked_node;
     482                 :            :                 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
     483                 :            :                 WARN_ON_ONCE(rnp != t->rcu_blocked_node);
     484                 :            :                 WARN_ON_ONCE(!rcu_is_leaf_node(rnp));
     485                 :            :                 empty_norm = !rcu_preempt_blocked_readers_cgp(rnp);
     486                 :            :                 WARN_ON_ONCE(rnp->completedqs == rnp->gp_seq &&
     487                 :            :                              (!empty_norm || rnp->qsmask));
     488                 :            :                 empty_exp = sync_rcu_preempt_exp_done(rnp);
     489                 :            :                 smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */
     490                 :            :                 np = rcu_next_node_entry(t, rnp);
     491                 :            :                 list_del_init(&t->rcu_node_entry);
     492                 :            :                 t->rcu_blocked_node = NULL;
     493                 :            :                 trace_rcu_unlock_preempted_task(TPS("rcu_preempt"),
     494                 :            :                                                 rnp->gp_seq, t->pid);
     495                 :            :                 if (&t->rcu_node_entry == rnp->gp_tasks)
     496                 :            :                         WRITE_ONCE(rnp->gp_tasks, np);
     497                 :            :                 if (&t->rcu_node_entry == rnp->exp_tasks)
     498                 :            :                         rnp->exp_tasks = np;
     499                 :            :                 if (IS_ENABLED(CONFIG_RCU_BOOST)) {
     500                 :            :                         /* Snapshot ->boost_mtx ownership w/rnp->lock held. */
     501                 :            :                         drop_boost_mutex = rt_mutex_owner(&rnp->boost_mtx) == t;
     502                 :            :                         if (&t->rcu_node_entry == rnp->boost_tasks)
     503                 :            :                                 rnp->boost_tasks = np;
     504                 :            :                 }
     505                 :            : 
     506                 :            :                 /*
     507                 :            :                  * If this was the last task on the current list, and if
     508                 :            :                  * we aren't waiting on any CPUs, report the quiescent state.
     509                 :            :                  * Note that rcu_report_unblock_qs_rnp() releases rnp->lock,
     510                 :            :                  * so we must take a snapshot of the expedited state.
     511                 :            :                  */
     512                 :            :                 empty_exp_now = sync_rcu_preempt_exp_done(rnp);
     513                 :            :                 if (!empty_norm && !rcu_preempt_blocked_readers_cgp(rnp)) {
     514                 :            :                         trace_rcu_quiescent_state_report(TPS("preempt_rcu"),
     515                 :            :                                                          rnp->gp_seq,
     516                 :            :                                                          0, rnp->qsmask,
     517                 :            :                                                          rnp->level,
     518                 :            :                                                          rnp->grplo,
     519                 :            :                                                          rnp->grphi,
     520                 :            :                                                          !!rnp->gp_tasks);
     521                 :            :                         rcu_report_unblock_qs_rnp(rnp, flags);
     522                 :            :                 } else {
     523                 :            :                         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
     524                 :            :                 }
     525                 :            : 
     526                 :            :                 /* Unboost if we were boosted. */
     527                 :            :                 if (IS_ENABLED(CONFIG_RCU_BOOST) && drop_boost_mutex)
     528                 :            :                         rt_mutex_futex_unlock(&rnp->boost_mtx);
     529                 :            : 
     530                 :            :                 /*
     531                 :            :                  * If this was the last task on the expedited lists,
     532                 :            :                  * then we need to report up the rcu_node hierarchy.
     533                 :            :                  */
     534                 :            :                 if (!empty_exp && empty_exp_now)
     535                 :            :                         rcu_report_exp_rnp(rnp, true);
     536                 :            :         } else {
     537                 :            :                 local_irq_restore(flags);
     538                 :            :         }
     539                 :            : }
     540                 :            : 
     541                 :            : /*
     542                 :            :  * Is a deferred quiescent-state pending, and are we also not in
     543                 :            :  * an RCU read-side critical section?  It is the caller's responsibility
     544                 :            :  * to ensure it is otherwise safe to report any deferred quiescent
     545                 :            :  * states.  The reason for this is that it is safe to report a
     546                 :            :  * quiescent state during context switch even though preemption
     547                 :            :  * is disabled.  This function cannot be expected to understand these
     548                 :            :  * nuances, so the caller must handle them.
     549                 :            :  */
     550                 :            : static bool rcu_preempt_need_deferred_qs(struct task_struct *t)
     551                 :            : {
     552                 :            :         return (__this_cpu_read(rcu_data.exp_deferred_qs) ||
     553                 :            :                 READ_ONCE(t->rcu_read_unlock_special.s)) &&
     554                 :            :                t->rcu_read_lock_nesting <= 0;
     555                 :            : }
     556                 :            : 
     557                 :            : /*
     558                 :            :  * Report a deferred quiescent state if needed and safe to do so.
     559                 :            :  * As with rcu_preempt_need_deferred_qs(), "safe" involves only
     560                 :            :  * not being in an RCU read-side critical section.  The caller must
     561                 :            :  * evaluate safety in terms of interrupt, softirq, and preemption
     562                 :            :  * disabling.
     563                 :            :  */
     564                 :            : static void rcu_preempt_deferred_qs(struct task_struct *t)
     565                 :            : {
     566                 :            :         unsigned long flags;
     567                 :            :         bool couldrecurse = t->rcu_read_lock_nesting >= 0;
     568                 :            : 
     569                 :            :         if (!rcu_preempt_need_deferred_qs(t))
     570                 :            :                 return;
     571                 :            :         if (couldrecurse)
     572                 :            :                 t->rcu_read_lock_nesting -= RCU_NEST_BIAS;
     573                 :            :         local_irq_save(flags);
     574                 :            :         rcu_preempt_deferred_qs_irqrestore(t, flags);
     575                 :            :         if (couldrecurse)
     576                 :            :                 t->rcu_read_lock_nesting += RCU_NEST_BIAS;
     577                 :            : }
     578                 :            : 
     579                 :            : /*
     580                 :            :  * Minimal handler to give the scheduler a chance to re-evaluate.
     581                 :            :  */
     582                 :            : static void rcu_preempt_deferred_qs_handler(struct irq_work *iwp)
     583                 :            : {
     584                 :            :         struct rcu_data *rdp;
     585                 :            : 
     586                 :            :         rdp = container_of(iwp, struct rcu_data, defer_qs_iw);
     587                 :            :         rdp->defer_qs_iw_pending = false;
     588                 :            : }
     589                 :            : 
     590                 :            : /*
     591                 :            :  * Handle special cases during rcu_read_unlock(), such as needing to
     592                 :            :  * notify RCU core processing or task having blocked during the RCU
     593                 :            :  * read-side critical section.
     594                 :            :  */
     595                 :            : static void rcu_read_unlock_special(struct task_struct *t)
     596                 :            : {
     597                 :            :         unsigned long flags;
     598                 :            :         bool preempt_bh_were_disabled =
     599                 :            :                         !!(preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK));
     600                 :            :         bool irqs_were_disabled;
     601                 :            : 
     602                 :            :         /* NMI handlers cannot block and cannot safely manipulate state. */
     603                 :            :         if (in_nmi())
     604                 :            :                 return;
     605                 :            : 
     606                 :            :         local_irq_save(flags);
     607                 :            :         irqs_were_disabled = irqs_disabled_flags(flags);
     608                 :            :         if (preempt_bh_were_disabled || irqs_were_disabled) {
     609                 :            :                 bool exp;
     610                 :            :                 struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
     611                 :            :                 struct rcu_node *rnp = rdp->mynode;
     612                 :            : 
     613                 :            :                 t->rcu_read_unlock_special.b.exp_hint = false;
     614                 :            :                 exp = (t->rcu_blocked_node && t->rcu_blocked_node->exp_tasks) ||
     615                 :            :                       (rdp->grpmask & READ_ONCE(rnp->expmask)) ||
     616                 :            :                       tick_nohz_full_cpu(rdp->cpu);
     617                 :            :                 // Need to defer quiescent state until everything is enabled.
     618                 :            :                 if (irqs_were_disabled && use_softirq &&
     619                 :            :                     (in_interrupt() ||
     620                 :            :                      (exp && !t->rcu_read_unlock_special.b.deferred_qs))) {
     621                 :            :                         // Using softirq, safe to awaken, and we get
     622                 :            :                         // no help from enabling irqs, unlike bh/preempt.
     623                 :            :                         raise_softirq_irqoff(RCU_SOFTIRQ);
     624                 :            :                 } else {
     625                 :            :                         // Enabling BH or preempt does reschedule, so...
     626                 :            :                         // Also if no expediting or NO_HZ_FULL, slow is OK.
     627                 :            :                         set_tsk_need_resched(current);
     628                 :            :                         set_preempt_need_resched();
     629                 :            :                         if (IS_ENABLED(CONFIG_IRQ_WORK) && irqs_were_disabled &&
     630                 :            :                             !rdp->defer_qs_iw_pending && exp) {
     631                 :            :                                 // Get scheduler to re-evaluate and call hooks.
     632                 :            :                                 // If !IRQ_WORK, FQS scan will eventually IPI.
     633                 :            :                                 init_irq_work(&rdp->defer_qs_iw,
     634                 :            :                                               rcu_preempt_deferred_qs_handler);
     635                 :            :                                 rdp->defer_qs_iw_pending = true;
     636                 :            :                                 irq_work_queue_on(&rdp->defer_qs_iw, rdp->cpu);
     637                 :            :                         }
     638                 :            :                 }
     639                 :            :                 t->rcu_read_unlock_special.b.deferred_qs = true;
     640                 :            :                 local_irq_restore(flags);
     641                 :            :                 return;
     642                 :            :         }
     643                 :            :         WRITE_ONCE(t->rcu_read_unlock_special.b.exp_hint, false);
     644                 :            :         rcu_preempt_deferred_qs_irqrestore(t, flags);
     645                 :            : }
     646                 :            : 
     647                 :            : /*
     648                 :            :  * Check that the list of blocked tasks for the newly completed grace
     649                 :            :  * period is in fact empty.  It is a serious bug to complete a grace
     650                 :            :  * period that still has RCU readers blocked!  This function must be
     651                 :            :  * invoked -before- updating this rnp's ->gp_seq, and the rnp's ->lock
     652                 :            :  * must be held by the caller.
     653                 :            :  *
     654                 :            :  * Also, if there are blocked tasks on the list, they automatically
     655                 :            :  * block the newly created grace period, so set up ->gp_tasks accordingly.
     656                 :            :  */
     657                 :            : static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
     658                 :            : {
     659                 :            :         struct task_struct *t;
     660                 :            : 
     661                 :            :         RCU_LOCKDEP_WARN(preemptible(), "rcu_preempt_check_blocked_tasks() invoked with preemption enabled!!!\n");
     662                 :            :         if (WARN_ON_ONCE(rcu_preempt_blocked_readers_cgp(rnp)))
     663                 :            :                 dump_blkd_tasks(rnp, 10);
     664                 :            :         if (rcu_preempt_has_tasks(rnp) &&
     665                 :            :             (rnp->qsmaskinit || rnp->wait_blkd_tasks)) {
     666                 :            :                 WRITE_ONCE(rnp->gp_tasks, rnp->blkd_tasks.next);
     667                 :            :                 t = container_of(rnp->gp_tasks, struct task_struct,
     668                 :            :                                  rcu_node_entry);
     669                 :            :                 trace_rcu_unlock_preempted_task(TPS("rcu_preempt-GPS"),
     670                 :            :                                                 rnp->gp_seq, t->pid);
     671                 :            :         }
     672                 :            :         WARN_ON_ONCE(rnp->qsmask);
     673                 :            : }
     674                 :            : 
     675                 :            : /*
     676                 :            :  * Check for a quiescent state from the current CPU, including voluntary
     677                 :            :  * context switches for Tasks RCU.  When a task blocks, the task is
     678                 :            :  * recorded in the corresponding CPU's rcu_node structure, which is checked
     679                 :            :  * elsewhere, hence this function need only check for quiescent states
     680                 :            :  * related to the current CPU, not to those related to tasks.
     681                 :            :  */
     682                 :            : static void rcu_flavor_sched_clock_irq(int user)
     683                 :            : {
     684                 :            :         struct task_struct *t = current;
     685                 :            : 
     686                 :            :         if (user || rcu_is_cpu_rrupt_from_idle()) {
     687                 :            :                 rcu_note_voluntary_context_switch(current);
     688                 :            :         }
     689                 :            :         if (t->rcu_read_lock_nesting > 0 ||
     690                 :            :             (preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK))) {
     691                 :            :                 /* No QS, force context switch if deferred. */
     692                 :            :                 if (rcu_preempt_need_deferred_qs(t)) {
     693                 :            :                         set_tsk_need_resched(t);
     694                 :            :                         set_preempt_need_resched();
     695                 :            :                 }
     696                 :            :         } else if (rcu_preempt_need_deferred_qs(t)) {
     697                 :            :                 rcu_preempt_deferred_qs(t); /* Report deferred QS. */
     698                 :            :                 return;
     699                 :            :         } else if (!t->rcu_read_lock_nesting) {
     700                 :            :                 rcu_qs(); /* Report immediate QS. */
     701                 :            :                 return;
     702                 :            :         }
     703                 :            : 
     704                 :            :         /* If GP is oldish, ask for help from rcu_read_unlock_special(). */
     705                 :            :         if (t->rcu_read_lock_nesting > 0 &&
     706                 :            :             __this_cpu_read(rcu_data.core_needs_qs) &&
     707                 :            :             __this_cpu_read(rcu_data.cpu_no_qs.b.norm) &&
     708                 :            :             !t->rcu_read_unlock_special.b.need_qs &&
     709                 :            :             time_after(jiffies, rcu_state.gp_start + HZ))
     710                 :            :                 t->rcu_read_unlock_special.b.need_qs = true;
     711                 :            : }
     712                 :            : 
     713                 :            : /*
     714                 :            :  * Check for a task exiting while in a preemptible-RCU read-side
     715                 :            :  * critical section, clean up if so.  No need to issue warnings, as
     716                 :            :  * debug_check_no_locks_held() already does this if lockdep is enabled.
     717                 :            :  * Besides, if this function does anything other than just immediately
     718                 :            :  * return, there was a bug of some sort.  Spewing warnings from this
     719                 :            :  * function is like as not to simply obscure important prior warnings.
     720                 :            :  */
     721                 :            : void exit_rcu(void)
     722                 :            : {
     723                 :            :         struct task_struct *t = current;
     724                 :            : 
     725                 :            :         if (unlikely(!list_empty(&current->rcu_node_entry))) {
     726                 :            :                 t->rcu_read_lock_nesting = 1;
     727                 :            :                 barrier();
     728                 :            :                 WRITE_ONCE(t->rcu_read_unlock_special.b.blocked, true);
     729                 :            :         } else if (unlikely(t->rcu_read_lock_nesting)) {
     730                 :            :                 t->rcu_read_lock_nesting = 1;
     731                 :            :         } else {
     732                 :            :                 return;
     733                 :            :         }
     734                 :            :         __rcu_read_unlock();
     735                 :            :         rcu_preempt_deferred_qs(current);
     736                 :            : }
     737                 :            : 
     738                 :            : /*
     739                 :            :  * Dump the blocked-tasks state, but limit the list dump to the
     740                 :            :  * specified number of elements.
     741                 :            :  */
     742                 :            : static void
     743                 :            : dump_blkd_tasks(struct rcu_node *rnp, int ncheck)
     744                 :            : {
     745                 :            :         int cpu;
     746                 :            :         int i;
     747                 :            :         struct list_head *lhp;
     748                 :            :         bool onl;
     749                 :            :         struct rcu_data *rdp;
     750                 :            :         struct rcu_node *rnp1;
     751                 :            : 
     752                 :            :         raw_lockdep_assert_held_rcu_node(rnp);
     753                 :            :         pr_info("%s: grp: %d-%d level: %d ->gp_seq %ld ->completedqs %ld\n",
     754                 :            :                 __func__, rnp->grplo, rnp->grphi, rnp->level,
     755                 :            :                 (long)rnp->gp_seq, (long)rnp->completedqs);
     756                 :            :         for (rnp1 = rnp; rnp1; rnp1 = rnp1->parent)
     757                 :            :                 pr_info("%s: %d:%d ->qsmask %#lx ->qsmaskinit %#lx ->qsmaskinitnext %#lx\n",
     758                 :            :                         __func__, rnp1->grplo, rnp1->grphi, rnp1->qsmask, rnp1->qsmaskinit, rnp1->qsmaskinitnext);
     759                 :            :         pr_info("%s: ->gp_tasks %p ->boost_tasks %p ->exp_tasks %p\n",
     760                 :            :                 __func__, READ_ONCE(rnp->gp_tasks), rnp->boost_tasks,
     761                 :            :                 rnp->exp_tasks);
     762                 :            :         pr_info("%s: ->blkd_tasks", __func__);
     763                 :            :         i = 0;
     764                 :            :         list_for_each(lhp, &rnp->blkd_tasks) {
     765                 :            :                 pr_cont(" %p", lhp);
     766                 :            :                 if (++i >= ncheck)
     767                 :            :                         break;
     768                 :            :         }
     769                 :            :         pr_cont("\n");
     770                 :            :         for (cpu = rnp->grplo; cpu <= rnp->grphi; cpu++) {
     771                 :            :                 rdp = per_cpu_ptr(&rcu_data, cpu);
     772                 :            :                 onl = !!(rdp->grpmask & rcu_rnp_online_cpus(rnp));
     773                 :            :                 pr_info("\t%d: %c online: %ld(%d) offline: %ld(%d)\n",
     774                 :            :                         cpu, ".o"[onl],
     775                 :            :                         (long)rdp->rcu_onl_gp_seq, rdp->rcu_onl_gp_flags,
     776                 :            :                         (long)rdp->rcu_ofl_gp_seq, rdp->rcu_ofl_gp_flags);
     777                 :            :         }
     778                 :            : }
     779                 :            : 
     780                 :            : #else /* #ifdef CONFIG_PREEMPT_RCU */
     781                 :            : 
     782                 :            : /*
     783                 :            :  * Tell them what RCU they are running.
     784                 :            :  */
     785                 :          3 : static void __init rcu_bootup_announce(void)
     786                 :            : {
     787                 :          3 :         pr_info("Hierarchical RCU implementation.\n");
     788                 :          3 :         rcu_bootup_announce_oddness();
     789                 :          3 : }
     790                 :            : 
     791                 :            : /*
     792                 :            :  * Note a quiescent state for PREEMPT=n.  Because we do not need to know
     793                 :            :  * how many quiescent states passed, just if there was at least one since
     794                 :            :  * the start of the grace period, this just sets a flag.  The caller must
     795                 :            :  * have disabled preemption.
     796                 :            :  */
     797                 :          3 : static void rcu_qs(void)
     798                 :            : {
     799                 :            :         RCU_LOCKDEP_WARN(preemptible(), "rcu_qs() invoked with preemption enabled!!!");
     800                 :          3 :         if (!__this_cpu_read(rcu_data.cpu_no_qs.s))
     801                 :            :                 return;
     802                 :            :         trace_rcu_grace_period(TPS("rcu_sched"),
     803                 :          3 :                                __this_cpu_read(rcu_data.gp_seq), TPS("cpuqs"));
     804                 :          3 :         __this_cpu_write(rcu_data.cpu_no_qs.b.norm, false);
     805                 :          3 :         if (!__this_cpu_read(rcu_data.cpu_no_qs.b.exp))
     806                 :            :                 return;
     807                 :          3 :         __this_cpu_write(rcu_data.cpu_no_qs.b.exp, false);
     808                 :          3 :         rcu_report_exp_rdp(this_cpu_ptr(&rcu_data));
     809                 :            : }
     810                 :            : 
     811                 :            : /*
     812                 :            :  * Register an urgently needed quiescent state.  If there is an
     813                 :            :  * emergency, invoke rcu_momentary_dyntick_idle() to do a heavy-weight
     814                 :            :  * dyntick-idle quiescent state visible to other CPUs, which will in
     815                 :            :  * some cases serve for expedited as well as normal grace periods.
     816                 :            :  * Either way, register a lightweight quiescent state.
     817                 :            :  */
     818                 :          3 : void rcu_all_qs(void)
     819                 :            : {
     820                 :            :         unsigned long flags;
     821                 :            : 
     822                 :          3 :         if (!raw_cpu_read(rcu_data.rcu_urgent_qs))
     823                 :            :                 return;
     824                 :          3 :         preempt_disable();
     825                 :            :         /* Load rcu_urgent_qs before other flags. */
     826                 :          3 :         if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) {
     827                 :          1 :                 preempt_enable();
     828                 :          1 :                 return;
     829                 :            :         }
     830                 :          3 :         this_cpu_write(rcu_data.rcu_urgent_qs, false);
     831                 :          3 :         if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs))) {
     832                 :          3 :                 local_irq_save(flags);
     833                 :          3 :                 rcu_momentary_dyntick_idle();
     834                 :          3 :                 local_irq_restore(flags);
     835                 :            :         }
     836                 :          3 :         rcu_qs();
     837                 :          3 :         preempt_enable();
     838                 :            : }
     839                 :            : EXPORT_SYMBOL_GPL(rcu_all_qs);
     840                 :            : 
     841                 :            : /*
     842                 :            :  * Note a PREEMPT=n context switch.  The caller must have disabled interrupts.
     843                 :            :  */
     844                 :          3 : void rcu_note_context_switch(bool preempt)
     845                 :            : {
     846                 :          3 :         trace_rcu_utilization(TPS("Start context switch"));
     847                 :          3 :         rcu_qs();
     848                 :            :         /* Load rcu_urgent_qs before other flags. */
     849                 :          3 :         if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs)))
     850                 :            :                 goto out;
     851                 :          3 :         this_cpu_write(rcu_data.rcu_urgent_qs, false);
     852                 :          3 :         if (unlikely(raw_cpu_read(rcu_data.rcu_need_heavy_qs)))
     853                 :          3 :                 rcu_momentary_dyntick_idle();
     854                 :            :         if (!preempt)
     855                 :            :                 rcu_tasks_qs(current);
     856                 :            : out:
     857                 :          3 :         trace_rcu_utilization(TPS("End context switch"));
     858                 :          3 : }
     859                 :            : EXPORT_SYMBOL_GPL(rcu_note_context_switch);
     860                 :            : 
     861                 :            : /*
     862                 :            :  * Because preemptible RCU does not exist, there are never any preempted
     863                 :            :  * RCU readers.
     864                 :            :  */
     865                 :            : static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp)
     866                 :            : {
     867                 :            :         return 0;
     868                 :            : }
     869                 :            : 
     870                 :            : /*
     871                 :            :  * Because there is no preemptible RCU, there can be no readers blocked.
     872                 :            :  */
     873                 :            : static bool rcu_preempt_has_tasks(struct rcu_node *rnp)
     874                 :            : {
     875                 :            :         return false;
     876                 :            : }
     877                 :            : 
     878                 :            : /*
     879                 :            :  * Because there is no preemptible RCU, there can be no deferred quiescent
     880                 :            :  * states.
     881                 :            :  */
     882                 :            : static bool rcu_preempt_need_deferred_qs(struct task_struct *t)
     883                 :            : {
     884                 :            :         return false;
     885                 :            : }
     886                 :            : static void rcu_preempt_deferred_qs(struct task_struct *t) { }
     887                 :            : 
     888                 :            : /*
     889                 :            :  * Because there is no preemptible RCU, there can be no readers blocked,
     890                 :            :  * so there is no need to check for blocked tasks.  So check only for
     891                 :            :  * bogus qsmask values.
     892                 :            :  */
     893                 :          3 : static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp)
     894                 :            : {
     895                 :          3 :         WARN_ON_ONCE(rnp->qsmask);
     896                 :          3 : }
     897                 :            : 
     898                 :            : /*
     899                 :            :  * Check to see if this CPU is in a non-context-switch quiescent state,
     900                 :            :  * namely user mode and idle loop.
     901                 :            :  */
     902                 :          3 : static void rcu_flavor_sched_clock_irq(int user)
     903                 :            : {
     904                 :          3 :         if (user || rcu_is_cpu_rrupt_from_idle()) {
     905                 :            : 
     906                 :            :                 /*
     907                 :            :                  * Get here if this CPU took its interrupt from user
     908                 :            :                  * mode or from the idle loop, and if this is not a
     909                 :            :                  * nested interrupt.  In this case, the CPU is in
     910                 :            :                  * a quiescent state, so note it.
     911                 :            :                  *
     912                 :            :                  * No memory barrier is required here because rcu_qs()
     913                 :            :                  * references only CPU-local variables that other CPUs
     914                 :            :                  * neither access nor modify, at least not while the
     915                 :            :                  * corresponding CPU is online.
     916                 :            :                  */
     917                 :            : 
     918                 :          3 :                 rcu_qs();
     919                 :            :         }
     920                 :          3 : }
     921                 :            : 
     922                 :            : /*
     923                 :            :  * Because preemptible RCU does not exist, tasks cannot possibly exit
     924                 :            :  * while in preemptible RCU read-side critical sections.
     925                 :            :  */
     926                 :          3 : void exit_rcu(void)
     927                 :            : {
     928                 :          3 : }
     929                 :            : 
     930                 :            : /*
     931                 :            :  * Dump the guaranteed-empty blocked-tasks state.  Trust but verify.
     932                 :            :  */
     933                 :            : static void
     934                 :            : dump_blkd_tasks(struct rcu_node *rnp, int ncheck)
     935                 :            : {
     936                 :            :         WARN_ON_ONCE(!list_empty(&rnp->blkd_tasks));
     937                 :            : }
     938                 :            : 
     939                 :            : #endif /* #else #ifdef CONFIG_PREEMPT_RCU */
     940                 :            : 
     941                 :            : /*
     942                 :            :  * If boosting, set rcuc kthreads to realtime priority.
     943                 :            :  */
     944                 :          0 : static void rcu_cpu_kthread_setup(unsigned int cpu)
     945                 :            : {
     946                 :            : #ifdef CONFIG_RCU_BOOST
     947                 :            :         struct sched_param sp;
     948                 :            : 
     949                 :            :         sp.sched_priority = kthread_prio;
     950                 :            :         sched_setscheduler_nocheck(current, SCHED_FIFO, &sp);
     951                 :            : #endif /* #ifdef CONFIG_RCU_BOOST */
     952                 :          0 : }
     953                 :            : 
     954                 :            : #ifdef CONFIG_RCU_BOOST
     955                 :            : 
     956                 :            : /*
     957                 :            :  * Carry out RCU priority boosting on the task indicated by ->exp_tasks
     958                 :            :  * or ->boost_tasks, advancing the pointer to the next task in the
     959                 :            :  * ->blkd_tasks list.
     960                 :            :  *
     961                 :            :  * Note that irqs must be enabled: boosting the task can block.
     962                 :            :  * Returns 1 if there are more tasks needing to be boosted.
     963                 :            :  */
     964                 :            : static int rcu_boost(struct rcu_node *rnp)
     965                 :            : {
     966                 :            :         unsigned long flags;
     967                 :            :         struct task_struct *t;
     968                 :            :         struct list_head *tb;
     969                 :            : 
     970                 :            :         if (READ_ONCE(rnp->exp_tasks) == NULL &&
     971                 :            :             READ_ONCE(rnp->boost_tasks) == NULL)
     972                 :            :                 return 0;  /* Nothing left to boost. */
     973                 :            : 
     974                 :            :         raw_spin_lock_irqsave_rcu_node(rnp, flags);
     975                 :            : 
     976                 :            :         /*
     977                 :            :          * Recheck under the lock: all tasks in need of boosting
     978                 :            :          * might exit their RCU read-side critical sections on their own.
     979                 :            :          */
     980                 :            :         if (rnp->exp_tasks == NULL && rnp->boost_tasks == NULL) {
     981                 :            :                 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
     982                 :            :                 return 0;
     983                 :            :         }
     984                 :            : 
     985                 :            :         /*
     986                 :            :          * Preferentially boost tasks blocking expedited grace periods.
     987                 :            :          * This cannot starve the normal grace periods because a second
     988                 :            :          * expedited grace period must boost all blocked tasks, including
     989                 :            :          * those blocking the pre-existing normal grace period.
     990                 :            :          */
     991                 :            :         if (rnp->exp_tasks != NULL)
     992                 :            :                 tb = rnp->exp_tasks;
     993                 :            :         else
     994                 :            :                 tb = rnp->boost_tasks;
     995                 :            : 
     996                 :            :         /*
     997                 :            :          * We boost task t by manufacturing an rt_mutex that appears to
     998                 :            :          * be held by task t.  We leave a pointer to that rt_mutex where
     999                 :            :          * task t can find it, and task t will release the mutex when it
    1000                 :            :          * exits its outermost RCU read-side critical section.  Then
    1001                 :            :          * simply acquiring this artificial rt_mutex will boost task
    1002                 :            :          * t's priority.  (Thanks to tglx for suggesting this approach!)
    1003                 :            :          *
    1004                 :            :          * Note that task t must acquire rnp->lock to remove itself from
    1005                 :            :          * the ->blkd_tasks list, which it will do from exit() if from
    1006                 :            :          * nowhere else.  We therefore are guaranteed that task t will
    1007                 :            :          * stay around at least until we drop rnp->lock.  Note that
    1008                 :            :          * rnp->lock also resolves races between our priority boosting
    1009                 :            :          * and task t's exiting its outermost RCU read-side critical
    1010                 :            :          * section.
    1011                 :            :          */
    1012                 :            :         t = container_of(tb, struct task_struct, rcu_node_entry);
    1013                 :            :         rt_mutex_init_proxy_locked(&rnp->boost_mtx, t);
    1014                 :            :         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    1015                 :            :         /* Lock only for side effect: boosts task t's priority. */
    1016                 :            :         rt_mutex_lock(&rnp->boost_mtx);
    1017                 :            :         rt_mutex_unlock(&rnp->boost_mtx);  /* Then keep lockdep happy. */
    1018                 :            : 
    1019                 :            :         return READ_ONCE(rnp->exp_tasks) != NULL ||
    1020                 :            :                READ_ONCE(rnp->boost_tasks) != NULL;
    1021                 :            : }
    1022                 :            : 
    1023                 :            : /*
    1024                 :            :  * Priority-boosting kthread, one per leaf rcu_node.
    1025                 :            :  */
    1026                 :            : static int rcu_boost_kthread(void *arg)
    1027                 :            : {
    1028                 :            :         struct rcu_node *rnp = (struct rcu_node *)arg;
    1029                 :            :         int spincnt = 0;
    1030                 :            :         int more2boost;
    1031                 :            : 
    1032                 :            :         trace_rcu_utilization(TPS("Start boost kthread@init"));
    1033                 :            :         for (;;) {
    1034                 :            :                 rnp->boost_kthread_status = RCU_KTHREAD_WAITING;
    1035                 :            :                 trace_rcu_utilization(TPS("End boost kthread@rcu_wait"));
    1036                 :            :                 rcu_wait(rnp->boost_tasks || rnp->exp_tasks);
    1037                 :            :                 trace_rcu_utilization(TPS("Start boost kthread@rcu_wait"));
    1038                 :            :                 rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;
    1039                 :            :                 more2boost = rcu_boost(rnp);
    1040                 :            :                 if (more2boost)
    1041                 :            :                         spincnt++;
    1042                 :            :                 else
    1043                 :            :                         spincnt = 0;
    1044                 :            :                 if (spincnt > 10) {
    1045                 :            :                         rnp->boost_kthread_status = RCU_KTHREAD_YIELDING;
    1046                 :            :                         trace_rcu_utilization(TPS("End boost kthread@rcu_yield"));
    1047                 :            :                         schedule_timeout_interruptible(2);
    1048                 :            :                         trace_rcu_utilization(TPS("Start boost kthread@rcu_yield"));
    1049                 :            :                         spincnt = 0;
    1050                 :            :                 }
    1051                 :            :         }
    1052                 :            :         /* NOTREACHED */
    1053                 :            :         trace_rcu_utilization(TPS("End boost kthread@notreached"));
    1054                 :            :         return 0;
    1055                 :            : }
    1056                 :            : 
    1057                 :            : /*
    1058                 :            :  * Check to see if it is time to start boosting RCU readers that are
    1059                 :            :  * blocking the current grace period, and, if so, tell the per-rcu_node
    1060                 :            :  * kthread to start boosting them.  If there is an expedited grace
    1061                 :            :  * period in progress, it is always time to boost.
    1062                 :            :  *
    1063                 :            :  * The caller must hold rnp->lock, which this function releases.
    1064                 :            :  * The ->boost_kthread_task is immortal, so we don't need to worry
    1065                 :            :  * about it going away.
    1066                 :            :  */
    1067                 :            : static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
    1068                 :            :         __releases(rnp->lock)
    1069                 :            : {
    1070                 :            :         raw_lockdep_assert_held_rcu_node(rnp);
    1071                 :            :         if (!rcu_preempt_blocked_readers_cgp(rnp) && rnp->exp_tasks == NULL) {
    1072                 :            :                 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    1073                 :            :                 return;
    1074                 :            :         }
    1075                 :            :         if (rnp->exp_tasks != NULL ||
    1076                 :            :             (rnp->gp_tasks != NULL &&
    1077                 :            :              rnp->boost_tasks == NULL &&
    1078                 :            :              rnp->qsmask == 0 &&
    1079                 :            :              ULONG_CMP_GE(jiffies, rnp->boost_time))) {
    1080                 :            :                 if (rnp->exp_tasks == NULL)
    1081                 :            :                         rnp->boost_tasks = rnp->gp_tasks;
    1082                 :            :                 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    1083                 :            :                 rcu_wake_cond(rnp->boost_kthread_task,
    1084                 :            :                               rnp->boost_kthread_status);
    1085                 :            :         } else {
    1086                 :            :                 raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    1087                 :            :         }
    1088                 :            : }
    1089                 :            : 
    1090                 :            : /*
    1091                 :            :  * Is the current CPU running the RCU-callbacks kthread?
    1092                 :            :  * Caller must have preemption disabled.
    1093                 :            :  */
    1094                 :            : static bool rcu_is_callbacks_kthread(void)
    1095                 :            : {
    1096                 :            :         return __this_cpu_read(rcu_data.rcu_cpu_kthread_task) == current;
    1097                 :            : }
    1098                 :            : 
    1099                 :            : #define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
    1100                 :            : 
    1101                 :            : /*
    1102                 :            :  * Do priority-boost accounting for the start of a new grace period.
    1103                 :            :  */
    1104                 :            : static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
    1105                 :            : {
    1106                 :            :         rnp->boost_time = jiffies + RCU_BOOST_DELAY_JIFFIES;
    1107                 :            : }
    1108                 :            : 
    1109                 :            : /*
    1110                 :            :  * Create an RCU-boost kthread for the specified node if one does not
    1111                 :            :  * already exist.  We only create this kthread for preemptible RCU.
    1112                 :            :  * Returns zero if all is well, a negated errno otherwise.
    1113                 :            :  */
    1114                 :            : static void rcu_spawn_one_boost_kthread(struct rcu_node *rnp)
    1115                 :            : {
    1116                 :            :         int rnp_index = rnp - rcu_get_root();
    1117                 :            :         unsigned long flags;
    1118                 :            :         struct sched_param sp;
    1119                 :            :         struct task_struct *t;
    1120                 :            : 
    1121                 :            :         if (!IS_ENABLED(CONFIG_PREEMPT_RCU))
    1122                 :            :                 return;
    1123                 :            : 
    1124                 :            :         if (!rcu_scheduler_fully_active || rcu_rnp_online_cpus(rnp) == 0)
    1125                 :            :                 return;
    1126                 :            : 
    1127                 :            :         rcu_state.boost = 1;
    1128                 :            : 
    1129                 :            :         if (rnp->boost_kthread_task != NULL)
    1130                 :            :                 return;
    1131                 :            : 
    1132                 :            :         t = kthread_create(rcu_boost_kthread, (void *)rnp,
    1133                 :            :                            "rcub/%d", rnp_index);
    1134                 :            :         if (WARN_ON_ONCE(IS_ERR(t)))
    1135                 :            :                 return;
    1136                 :            : 
    1137                 :            :         raw_spin_lock_irqsave_rcu_node(rnp, flags);
    1138                 :            :         rnp->boost_kthread_task = t;
    1139                 :            :         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    1140                 :            :         sp.sched_priority = kthread_prio;
    1141                 :            :         sched_setscheduler_nocheck(t, SCHED_FIFO, &sp);
    1142                 :            :         wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */
    1143                 :            : }
    1144                 :            : 
    1145                 :            : /*
    1146                 :            :  * Set the per-rcu_node kthread's affinity to cover all CPUs that are
    1147                 :            :  * served by the rcu_node in question.  The CPU hotplug lock is still
    1148                 :            :  * held, so the value of rnp->qsmaskinit will be stable.
    1149                 :            :  *
    1150                 :            :  * We don't include outgoingcpu in the affinity set, use -1 if there is
    1151                 :            :  * no outgoing CPU.  If there are no CPUs left in the affinity set,
    1152                 :            :  * this function allows the kthread to execute on any CPU.
    1153                 :            :  */
    1154                 :            : static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
    1155                 :            : {
    1156                 :            :         struct task_struct *t = rnp->boost_kthread_task;
    1157                 :            :         unsigned long mask = rcu_rnp_online_cpus(rnp);
    1158                 :            :         cpumask_var_t cm;
    1159                 :            :         int cpu;
    1160                 :            : 
    1161                 :            :         if (!t)
    1162                 :            :                 return;
    1163                 :            :         if (!zalloc_cpumask_var(&cm, GFP_KERNEL))
    1164                 :            :                 return;
    1165                 :            :         for_each_leaf_node_possible_cpu(rnp, cpu)
    1166                 :            :                 if ((mask & leaf_node_cpu_bit(rnp, cpu)) &&
    1167                 :            :                     cpu != outgoingcpu)
    1168                 :            :                         cpumask_set_cpu(cpu, cm);
    1169                 :            :         if (cpumask_weight(cm) == 0)
    1170                 :            :                 cpumask_setall(cm);
    1171                 :            :         set_cpus_allowed_ptr(t, cm);
    1172                 :            :         free_cpumask_var(cm);
    1173                 :            : }
    1174                 :            : 
    1175                 :            : /*
    1176                 :            :  * Spawn boost kthreads -- called as soon as the scheduler is running.
    1177                 :            :  */
    1178                 :            : static void __init rcu_spawn_boost_kthreads(void)
    1179                 :            : {
    1180                 :            :         struct rcu_node *rnp;
    1181                 :            : 
    1182                 :            :         rcu_for_each_leaf_node(rnp)
    1183                 :            :                 rcu_spawn_one_boost_kthread(rnp);
    1184                 :            : }
    1185                 :            : 
    1186                 :            : static void rcu_prepare_kthreads(int cpu)
    1187                 :            : {
    1188                 :            :         struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
    1189                 :            :         struct rcu_node *rnp = rdp->mynode;
    1190                 :            : 
    1191                 :            :         /* Fire up the incoming CPU's kthread and leaf rcu_node kthread. */
    1192                 :            :         if (rcu_scheduler_fully_active)
    1193                 :            :                 rcu_spawn_one_boost_kthread(rnp);
    1194                 :            : }
    1195                 :            : 
    1196                 :            : #else /* #ifdef CONFIG_RCU_BOOST */
    1197                 :            : 
    1198                 :            : static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags)
    1199                 :            :         __releases(rnp->lock)
    1200                 :            : {
    1201                 :          3 :         raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
    1202                 :            : }
    1203                 :            : 
    1204                 :            : static bool rcu_is_callbacks_kthread(void)
    1205                 :            : {
    1206                 :            :         return false;
    1207                 :            : }
    1208                 :            : 
    1209                 :            : static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
    1210                 :            : {
    1211                 :            : }
    1212                 :            : 
    1213                 :            : static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
    1214                 :            : {
    1215                 :            : }
    1216                 :            : 
    1217                 :            : static void __init rcu_spawn_boost_kthreads(void)
    1218                 :            : {
    1219                 :            : }
    1220                 :            : 
    1221                 :            : static void rcu_prepare_kthreads(int cpu)
    1222                 :            : {
    1223                 :            : }
    1224                 :            : 
    1225                 :            : #endif /* #else #ifdef CONFIG_RCU_BOOST */
    1226                 :            : 
    1227                 :            : #if !defined(CONFIG_RCU_FAST_NO_HZ)
    1228                 :            : 
    1229                 :            : /*
    1230                 :            :  * Check to see if any future non-offloaded RCU-related work will need
    1231                 :            :  * to be done by the current CPU, even if none need be done immediately,
    1232                 :            :  * returning 1 if so.  This function is part of the RCU implementation;
    1233                 :            :  * it is -not- an exported member of the RCU API.
    1234                 :            :  *
    1235                 :            :  * Because we not have RCU_FAST_NO_HZ, just check whether or not this
    1236                 :            :  * CPU has RCU callbacks queued.
    1237                 :            :  */
    1238                 :          3 : int rcu_needs_cpu(u64 basemono, u64 *nextevt)
    1239                 :            : {
    1240                 :          3 :         *nextevt = KTIME_MAX;
    1241                 :          3 :         return !rcu_segcblist_empty(&this_cpu_ptr(&rcu_data)->cblist) &&
    1242                 :          3 :                !rcu_segcblist_is_offloaded(&this_cpu_ptr(&rcu_data)->cblist);
    1243                 :            : }
    1244                 :            : 
    1245                 :            : /*
    1246                 :            :  * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
    1247                 :            :  * after it.
    1248                 :            :  */
    1249                 :          3 : static void rcu_cleanup_after_idle(void)
    1250                 :            : {
    1251                 :          3 : }
    1252                 :            : 
    1253                 :            : /*
    1254                 :            :  * Do the idle-entry grace-period work, which, because CONFIG_RCU_FAST_NO_HZ=n,
    1255                 :            :  * is nothing.
    1256                 :            :  */
    1257                 :          3 : static void rcu_prepare_for_idle(void)
    1258                 :            : {
    1259                 :          3 : }
    1260                 :            : 
    1261                 :            : #else /* #if !defined(CONFIG_RCU_FAST_NO_HZ) */
    1262                 :            : 
    1263                 :            : /*
    1264                 :            :  * This code is invoked when a CPU goes idle, at which point we want
    1265                 :            :  * to have the CPU do everything required for RCU so that it can enter
    1266                 :            :  * the energy-efficient dyntick-idle mode.  This is handled by a
    1267                 :            :  * state machine implemented by rcu_prepare_for_idle() below.
    1268                 :            :  *
    1269                 :            :  * The following three proprocessor symbols control this state machine:
    1270                 :            :  *
    1271                 :            :  * RCU_IDLE_GP_DELAY gives the number of jiffies that a CPU is permitted
    1272                 :            :  *      to sleep in dyntick-idle mode with RCU callbacks pending.  This
    1273                 :            :  *      is sized to be roughly one RCU grace period.  Those energy-efficiency
    1274                 :            :  *      benchmarkers who might otherwise be tempted to set this to a large
    1275                 :            :  *      number, be warned: Setting RCU_IDLE_GP_DELAY too high can hang your
    1276                 :            :  *      system.  And if you are -that- concerned about energy efficiency,
    1277                 :            :  *      just power the system down and be done with it!
    1278                 :            :  * RCU_IDLE_LAZY_GP_DELAY gives the number of jiffies that a CPU is
    1279                 :            :  *      permitted to sleep in dyntick-idle mode with only lazy RCU
    1280                 :            :  *      callbacks pending.  Setting this too high can OOM your system.
    1281                 :            :  *
    1282                 :            :  * The values below work well in practice.  If future workloads require
    1283                 :            :  * adjustment, they can be converted into kernel config parameters, though
    1284                 :            :  * making the state machine smarter might be a better option.
    1285                 :            :  */
    1286                 :            : #define RCU_IDLE_GP_DELAY 4             /* Roughly one grace period. */
    1287                 :            : #define RCU_IDLE_LAZY_GP_DELAY (6 * HZ) /* Roughly six seconds. */
    1288                 :            : 
    1289                 :            : static int rcu_idle_gp_delay = RCU_IDLE_GP_DELAY;
    1290                 :            : module_param(rcu_idle_gp_delay, int, 0644);
    1291                 :            : static int rcu_idle_lazy_gp_delay = RCU_IDLE_LAZY_GP_DELAY;
    1292                 :            : module_param(rcu_idle_lazy_gp_delay, int, 0644);
    1293                 :            : 
    1294                 :            : /*
    1295                 :            :  * Try to advance callbacks on the current CPU, but only if it has been
    1296                 :            :  * awhile since the last time we did so.  Afterwards, if there are any
    1297                 :            :  * callbacks ready for immediate invocation, return true.
    1298                 :            :  */
    1299                 :            : static bool __maybe_unused rcu_try_advance_all_cbs(void)
    1300                 :            : {
    1301                 :            :         bool cbs_ready = false;
    1302                 :            :         struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
    1303                 :            :         struct rcu_node *rnp;
    1304                 :            : 
    1305                 :            :         /* Exit early if we advanced recently. */
    1306                 :            :         if (jiffies == rdp->last_advance_all)
    1307                 :            :                 return false;
    1308                 :            :         rdp->last_advance_all = jiffies;
    1309                 :            : 
    1310                 :            :         rnp = rdp->mynode;
    1311                 :            : 
    1312                 :            :         /*
    1313                 :            :          * Don't bother checking unless a grace period has
    1314                 :            :          * completed since we last checked and there are
    1315                 :            :          * callbacks not yet ready to invoke.
    1316                 :            :          */
    1317                 :            :         if ((rcu_seq_completed_gp(rdp->gp_seq,
    1318                 :            :                                   rcu_seq_current(&rnp->gp_seq)) ||
    1319                 :            :              unlikely(READ_ONCE(rdp->gpwrap))) &&
    1320                 :            :             rcu_segcblist_pend_cbs(&rdp->cblist))
    1321                 :            :                 note_gp_changes(rdp);
    1322                 :            : 
    1323                 :            :         if (rcu_segcblist_ready_cbs(&rdp->cblist))
    1324                 :            :                 cbs_ready = true;
    1325                 :            :         return cbs_ready;
    1326                 :            : }
    1327                 :            : 
    1328                 :            : /*
    1329                 :            :  * Allow the CPU to enter dyntick-idle mode unless it has callbacks ready
    1330                 :            :  * to invoke.  If the CPU has callbacks, try to advance them.  Tell the
    1331                 :            :  * caller to set the timeout based on whether or not there are non-lazy
    1332                 :            :  * callbacks.
    1333                 :            :  *
    1334                 :            :  * The caller must have disabled interrupts.
    1335                 :            :  */
    1336                 :            : int rcu_needs_cpu(u64 basemono, u64 *nextevt)
    1337                 :            : {
    1338                 :            :         struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
    1339                 :            :         unsigned long dj;
    1340                 :            : 
    1341                 :            :         lockdep_assert_irqs_disabled();
    1342                 :            : 
    1343                 :            :         /* If no non-offloaded callbacks, RCU doesn't need the CPU. */
    1344                 :            :         if (rcu_segcblist_empty(&rdp->cblist) ||
    1345                 :            :             rcu_segcblist_is_offloaded(&this_cpu_ptr(&rcu_data)->cblist)) {
    1346                 :            :                 *nextevt = KTIME_MAX;
    1347                 :            :                 return 0;
    1348                 :            :         }
    1349                 :            : 
    1350                 :            :         /* Attempt to advance callbacks. */
    1351                 :            :         if (rcu_try_advance_all_cbs()) {
    1352                 :            :                 /* Some ready to invoke, so initiate later invocation. */
    1353                 :            :                 invoke_rcu_core();
    1354                 :            :                 return 1;
    1355                 :            :         }
    1356                 :            :         rdp->last_accelerate = jiffies;
    1357                 :            : 
    1358                 :            :         /* Request timer delay depending on laziness, and round. */
    1359                 :            :         rdp->all_lazy = !rcu_segcblist_n_nonlazy_cbs(&rdp->cblist);
    1360                 :            :         if (rdp->all_lazy) {
    1361                 :            :                 dj = round_jiffies(rcu_idle_lazy_gp_delay + jiffies) - jiffies;
    1362                 :            :         } else {
    1363                 :            :                 dj = round_up(rcu_idle_gp_delay + jiffies,
    1364                 :            :                                rcu_idle_gp_delay) - jiffies;
    1365                 :            :         }
    1366                 :            :         *nextevt = basemono + dj * TICK_NSEC;
    1367                 :            :         return 0;
    1368                 :            : }
    1369                 :            : 
    1370                 :            : /*
    1371                 :            :  * Prepare a CPU for idle from an RCU perspective.  The first major task
    1372                 :            :  * is to sense whether nohz mode has been enabled or disabled via sysfs.
    1373                 :            :  * The second major task is to check to see if a non-lazy callback has
    1374                 :            :  * arrived at a CPU that previously had only lazy callbacks.  The third
    1375                 :            :  * major task is to accelerate (that is, assign grace-period numbers to)
    1376                 :            :  * any recently arrived callbacks.
    1377                 :            :  *
    1378                 :            :  * The caller must have disabled interrupts.
    1379                 :            :  */
    1380                 :            : static void rcu_prepare_for_idle(void)
    1381                 :            : {
    1382                 :            :         bool needwake;
    1383                 :            :         struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
    1384                 :            :         struct rcu_node *rnp;
    1385                 :            :         int tne;
    1386                 :            : 
    1387                 :            :         lockdep_assert_irqs_disabled();
    1388                 :            :         if (rcu_segcblist_is_offloaded(&rdp->cblist))
    1389                 :            :                 return;
    1390                 :            : 
    1391                 :            :         /* Handle nohz enablement switches conservatively. */
    1392                 :            :         tne = READ_ONCE(tick_nohz_active);
    1393                 :            :         if (tne != rdp->tick_nohz_enabled_snap) {
    1394                 :            :                 if (!rcu_segcblist_empty(&rdp->cblist))
    1395                 :            :                         invoke_rcu_core(); /* force nohz to see update. */
    1396                 :            :                 rdp->tick_nohz_enabled_snap = tne;
    1397                 :            :                 return;
    1398                 :            :         }
    1399                 :            :         if (!tne)
    1400                 :            :                 return;
    1401                 :            : 
    1402                 :            :         /*
    1403                 :            :          * If a non-lazy callback arrived at a CPU having only lazy
    1404                 :            :          * callbacks, invoke RCU core for the side-effect of recalculating
    1405                 :            :          * idle duration on re-entry to idle.
    1406                 :            :          */
    1407                 :            :         if (rdp->all_lazy && rcu_segcblist_n_nonlazy_cbs(&rdp->cblist)) {
    1408                 :            :                 rdp->all_lazy = false;
    1409                 :            :                 invoke_rcu_core();
    1410                 :            :                 return;
    1411                 :            :         }
    1412                 :            : 
    1413                 :            :         /*
    1414                 :            :          * If we have not yet accelerated this jiffy, accelerate all
    1415                 :            :          * callbacks on this CPU.
    1416                 :            :          */
    1417                 :            :         if (rdp->last_accelerate == jiffies)
    1418                 :            :                 return;
    1419                 :            :         rdp->last_accelerate = jiffies;
    1420                 :            :         if (rcu_segcblist_pend_cbs(&rdp->cblist)) {
    1421                 :            :                 rnp = rdp->mynode;
    1422                 :            :                 raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
    1423                 :            :                 needwake = rcu_accelerate_cbs(rnp, rdp);
    1424                 :            :                 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
    1425                 :            :                 if (needwake)
    1426                 :            :                         rcu_gp_kthread_wake();
    1427                 :            :         }
    1428                 :            : }
    1429                 :            : 
    1430                 :            : /*
    1431                 :            :  * Clean up for exit from idle.  Attempt to advance callbacks based on
    1432                 :            :  * any grace periods that elapsed while the CPU was idle, and if any
    1433                 :            :  * callbacks are now ready to invoke, initiate invocation.
    1434                 :            :  */
    1435                 :            : static void rcu_cleanup_after_idle(void)
    1436                 :            : {
    1437                 :            :         struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
    1438                 :            : 
    1439                 :            :         lockdep_assert_irqs_disabled();
    1440                 :            :         if (rcu_segcblist_is_offloaded(&rdp->cblist))
    1441                 :            :                 return;
    1442                 :            :         if (rcu_try_advance_all_cbs())
    1443                 :            :                 invoke_rcu_core();
    1444                 :            : }
    1445                 :            : 
    1446                 :            : #endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */
    1447                 :            : 
    1448                 :            : #ifdef CONFIG_RCU_NOCB_CPU
    1449                 :            : 
    1450                 :            : /*
    1451                 :            :  * Offload callback processing from the boot-time-specified set of CPUs
    1452                 :            :  * specified by rcu_nocb_mask.  For the CPUs in the set, there are kthreads
    1453                 :            :  * created that pull the callbacks from the corresponding CPU, wait for
    1454                 :            :  * a grace period to elapse, and invoke the callbacks.  These kthreads
    1455                 :            :  * are organized into GP kthreads, which manage incoming callbacks, wait for
    1456                 :            :  * grace periods, and awaken CB kthreads, and the CB kthreads, which only
    1457                 :            :  * invoke callbacks.  Each GP kthread invokes its own CBs.  The no-CBs CPUs
    1458                 :            :  * do a wake_up() on their GP kthread when they insert a callback into any
    1459                 :            :  * empty list, unless the rcu_nocb_poll boot parameter has been specified,
    1460                 :            :  * in which case each kthread actively polls its CPU.  (Which isn't so great
    1461                 :            :  * for energy efficiency, but which does reduce RCU's overhead on that CPU.)
    1462                 :            :  *
    1463                 :            :  * This is intended to be used in conjunction with Frederic Weisbecker's
    1464                 :            :  * adaptive-idle work, which would seriously reduce OS jitter on CPUs
    1465                 :            :  * running CPU-bound user-mode computations.
    1466                 :            :  *
    1467                 :            :  * Offloading of callbacks can also be used as an energy-efficiency
    1468                 :            :  * measure because CPUs with no RCU callbacks queued are more aggressive
    1469                 :            :  * about entering dyntick-idle mode.
    1470                 :            :  */
    1471                 :            : 
    1472                 :            : 
    1473                 :            : /*
    1474                 :            :  * Parse the boot-time rcu_nocb_mask CPU list from the kernel parameters.
    1475                 :            :  * The string after the "rcu_nocbs=" is either "all" for all CPUs, or a
    1476                 :            :  * comma-separated list of CPUs and/or CPU ranges.  If an invalid list is
    1477                 :            :  * given, a warning is emitted and all CPUs are offloaded.
    1478                 :            :  */
    1479                 :            : static int __init rcu_nocb_setup(char *str)
    1480                 :            : {
    1481                 :            :         alloc_bootmem_cpumask_var(&rcu_nocb_mask);
    1482                 :            :         if (!strcasecmp(str, "all"))
    1483                 :            :                 cpumask_setall(rcu_nocb_mask);
    1484                 :            :         else
    1485                 :            :                 if (cpulist_parse(str, rcu_nocb_mask)) {
    1486                 :            :                         pr_warn("rcu_nocbs= bad CPU range, all CPUs set\n");
    1487                 :            :                         cpumask_setall(rcu_nocb_mask);
    1488                 :            :                 }
    1489                 :            :         return 1;
    1490                 :            : }
    1491                 :            : __setup("rcu_nocbs=", rcu_nocb_setup);
    1492                 :            : 
    1493                 :            : static int __init parse_rcu_nocb_poll(char *arg)
    1494                 :            : {
    1495                 :            :         rcu_nocb_poll = true;
    1496                 :            :         return 0;
    1497                 :            : }
    1498                 :            : early_param("rcu_nocb_poll", parse_rcu_nocb_poll);
    1499                 :            : 
    1500                 :            : /*
    1501                 :            :  * Don't bother bypassing ->cblist if the call_rcu() rate is low.
    1502                 :            :  * After all, the main point of bypassing is to avoid lock contention
    1503                 :            :  * on ->nocb_lock, which only can happen at high call_rcu() rates.
    1504                 :            :  */
    1505                 :            : int nocb_nobypass_lim_per_jiffy = 16 * 1000 / HZ;
    1506                 :            : module_param(nocb_nobypass_lim_per_jiffy, int, 0);
    1507                 :            : 
    1508                 :            : /*
    1509                 :            :  * Acquire the specified rcu_data structure's ->nocb_bypass_lock.  If the
    1510                 :            :  * lock isn't immediately available, increment ->nocb_lock_contended to
    1511                 :            :  * flag the contention.
    1512                 :            :  */
    1513                 :            : static void rcu_nocb_bypass_lock(struct rcu_data *rdp)
    1514                 :            : {
    1515                 :            :         lockdep_assert_irqs_disabled();
    1516                 :            :         if (raw_spin_trylock(&rdp->nocb_bypass_lock))
    1517                 :            :                 return;
    1518                 :            :         atomic_inc(&rdp->nocb_lock_contended);
    1519                 :            :         WARN_ON_ONCE(smp_processor_id() != rdp->cpu);
    1520                 :            :         smp_mb__after_atomic(); /* atomic_inc() before lock. */
    1521                 :            :         raw_spin_lock(&rdp->nocb_bypass_lock);
    1522                 :            :         smp_mb__before_atomic(); /* atomic_dec() after lock. */
    1523                 :            :         atomic_dec(&rdp->nocb_lock_contended);
    1524                 :            : }
    1525                 :            : 
    1526                 :            : /*
    1527                 :            :  * Spinwait until the specified rcu_data structure's ->nocb_lock is
    1528                 :            :  * not contended.  Please note that this is extremely special-purpose,
    1529                 :            :  * relying on the fact that at most two kthreads and one CPU contend for
    1530                 :            :  * this lock, and also that the two kthreads are guaranteed to have frequent
    1531                 :            :  * grace-period-duration time intervals between successive acquisitions
    1532                 :            :  * of the lock.  This allows us to use an extremely simple throttling
    1533                 :            :  * mechanism, and further to apply it only to the CPU doing floods of
    1534                 :            :  * call_rcu() invocations.  Don't try this at home!
    1535                 :            :  */
    1536                 :            : static void rcu_nocb_wait_contended(struct rcu_data *rdp)
    1537                 :            : {
    1538                 :            :         WARN_ON_ONCE(smp_processor_id() != rdp->cpu);
    1539                 :            :         while (WARN_ON_ONCE(atomic_read(&rdp->nocb_lock_contended)))
    1540                 :            :                 cpu_relax();
    1541                 :            : }
    1542                 :            : 
    1543                 :            : /*
    1544                 :            :  * Conditionally acquire the specified rcu_data structure's
    1545                 :            :  * ->nocb_bypass_lock.
    1546                 :            :  */
    1547                 :            : static bool rcu_nocb_bypass_trylock(struct rcu_data *rdp)
    1548                 :            : {
    1549                 :            :         lockdep_assert_irqs_disabled();
    1550                 :            :         return raw_spin_trylock(&rdp->nocb_bypass_lock);
    1551                 :            : }
    1552                 :            : 
    1553                 :            : /*
    1554                 :            :  * Release the specified rcu_data structure's ->nocb_bypass_lock.
    1555                 :            :  */
    1556                 :            : static void rcu_nocb_bypass_unlock(struct rcu_data *rdp)
    1557                 :            : {
    1558                 :            :         lockdep_assert_irqs_disabled();
    1559                 :            :         raw_spin_unlock(&rdp->nocb_bypass_lock);
    1560                 :            : }
    1561                 :            : 
    1562                 :            : /*
    1563                 :            :  * Acquire the specified rcu_data structure's ->nocb_lock, but only
    1564                 :            :  * if it corresponds to a no-CBs CPU.
    1565                 :            :  */
    1566                 :            : static void rcu_nocb_lock(struct rcu_data *rdp)
    1567                 :            : {
    1568                 :            :         lockdep_assert_irqs_disabled();
    1569                 :            :         if (!rcu_segcblist_is_offloaded(&rdp->cblist))
    1570                 :            :                 return;
    1571                 :            :         raw_spin_lock(&rdp->nocb_lock);
    1572                 :            : }
    1573                 :            : 
    1574                 :            : /*
    1575                 :            :  * Release the specified rcu_data structure's ->nocb_lock, but only
    1576                 :            :  * if it corresponds to a no-CBs CPU.
    1577                 :            :  */
    1578                 :            : static void rcu_nocb_unlock(struct rcu_data *rdp)
    1579                 :            : {
    1580                 :            :         if (rcu_segcblist_is_offloaded(&rdp->cblist)) {
    1581                 :            :                 lockdep_assert_irqs_disabled();
    1582                 :            :                 raw_spin_unlock(&rdp->nocb_lock);
    1583                 :            :         }
    1584                 :            : }
    1585                 :            : 
    1586                 :            : /*
    1587                 :            :  * Release the specified rcu_data structure's ->nocb_lock and restore
    1588                 :            :  * interrupts, but only if it corresponds to a no-CBs CPU.
    1589                 :            :  */
    1590                 :            : static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,
    1591                 :            :                                        unsigned long flags)
    1592                 :            : {
    1593                 :            :         if (rcu_segcblist_is_offloaded(&rdp->cblist)) {
    1594                 :            :                 lockdep_assert_irqs_disabled();
    1595                 :            :                 raw_spin_unlock_irqrestore(&rdp->nocb_lock, flags);
    1596                 :            :         } else {
    1597                 :            :                 local_irq_restore(flags);
    1598                 :            :         }
    1599                 :            : }
    1600                 :            : 
    1601                 :            : /* Lockdep check that ->cblist may be safely accessed. */
    1602                 :            : static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)
    1603                 :            : {
    1604                 :            :         lockdep_assert_irqs_disabled();
    1605                 :            :         if (rcu_segcblist_is_offloaded(&rdp->cblist) &&
    1606                 :            :             cpu_online(rdp->cpu))
    1607                 :            :                 lockdep_assert_held(&rdp->nocb_lock);
    1608                 :            : }
    1609                 :            : 
    1610                 :            : /*
    1611                 :            :  * Wake up any no-CBs CPUs' kthreads that were waiting on the just-ended
    1612                 :            :  * grace period.
    1613                 :            :  */
    1614                 :            : static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq)
    1615                 :            : {
    1616                 :            :         swake_up_all(sq);
    1617                 :            : }
    1618                 :            : 
    1619                 :            : static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
    1620                 :            : {
    1621                 :            :         return &rnp->nocb_gp_wq[rcu_seq_ctr(rnp->gp_seq) & 0x1];
    1622                 :            : }
    1623                 :            : 
    1624                 :            : static void rcu_init_one_nocb(struct rcu_node *rnp)
    1625                 :            : {
    1626                 :            :         init_swait_queue_head(&rnp->nocb_gp_wq[0]);
    1627                 :            :         init_swait_queue_head(&rnp->nocb_gp_wq[1]);
    1628                 :            : }
    1629                 :            : 
    1630                 :            : /* Is the specified CPU a no-CBs CPU? */
    1631                 :            : bool rcu_is_nocb_cpu(int cpu)
    1632                 :            : {
    1633                 :            :         if (cpumask_available(rcu_nocb_mask))
    1634                 :            :                 return cpumask_test_cpu(cpu, rcu_nocb_mask);
    1635                 :            :         return false;
    1636                 :            : }
    1637                 :            : 
    1638                 :            : /*
    1639                 :            :  * Kick the GP kthread for this NOCB group.  Caller holds ->nocb_lock
    1640                 :            :  * and this function releases it.
    1641                 :            :  */
    1642                 :            : static void wake_nocb_gp(struct rcu_data *rdp, bool force,
    1643                 :            :                            unsigned long flags)
    1644                 :            :         __releases(rdp->nocb_lock)
    1645                 :            : {
    1646                 :            :         bool needwake = false;
    1647                 :            :         struct rcu_data *rdp_gp = rdp->nocb_gp_rdp;
    1648                 :            : 
    1649                 :            :         lockdep_assert_held(&rdp->nocb_lock);
    1650                 :            :         if (!READ_ONCE(rdp_gp->nocb_gp_kthread)) {
    1651                 :            :                 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
    1652                 :            :                                     TPS("AlreadyAwake"));
    1653                 :            :                 rcu_nocb_unlock_irqrestore(rdp, flags);
    1654                 :            :                 return;
    1655                 :            :         }
    1656                 :            :         del_timer(&rdp->nocb_timer);
    1657                 :            :         rcu_nocb_unlock_irqrestore(rdp, flags);
    1658                 :            :         raw_spin_lock_irqsave(&rdp_gp->nocb_gp_lock, flags);
    1659                 :            :         if (force || READ_ONCE(rdp_gp->nocb_gp_sleep)) {
    1660                 :            :                 WRITE_ONCE(rdp_gp->nocb_gp_sleep, false);
    1661                 :            :                 needwake = true;
    1662                 :            :                 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake"));
    1663                 :            :         }
    1664                 :            :         raw_spin_unlock_irqrestore(&rdp_gp->nocb_gp_lock, flags);
    1665                 :            :         if (needwake)
    1666                 :            :                 wake_up_process(rdp_gp->nocb_gp_kthread);
    1667                 :            : }
    1668                 :            : 
    1669                 :            : /*
    1670                 :            :  * Arrange to wake the GP kthread for this NOCB group at some future
    1671                 :            :  * time when it is safe to do so.
    1672                 :            :  */
    1673                 :            : static void wake_nocb_gp_defer(struct rcu_data *rdp, int waketype,
    1674                 :            :                                const char *reason)
    1675                 :            : {
    1676                 :            :         if (rdp->nocb_defer_wakeup == RCU_NOCB_WAKE_NOT)
    1677                 :            :                 mod_timer(&rdp->nocb_timer, jiffies + 1);
    1678                 :            :         if (rdp->nocb_defer_wakeup < waketype)
    1679                 :            :                 WRITE_ONCE(rdp->nocb_defer_wakeup, waketype);
    1680                 :            :         trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason);
    1681                 :            : }
    1682                 :            : 
    1683                 :            : /*
    1684                 :            :  * Flush the ->nocb_bypass queue into ->cblist, enqueuing rhp if non-NULL.
    1685                 :            :  * However, if there is a callback to be enqueued and if ->nocb_bypass
    1686                 :            :  * proves to be initially empty, just return false because the no-CB GP
    1687                 :            :  * kthread may need to be awakened in this case.
    1688                 :            :  *
    1689                 :            :  * Note that this function always returns true if rhp is NULL.
    1690                 :            :  */
    1691                 :            : static bool rcu_nocb_do_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
    1692                 :            :                                      unsigned long j)
    1693                 :            : {
    1694                 :            :         struct rcu_cblist rcl;
    1695                 :            : 
    1696                 :            :         WARN_ON_ONCE(!rcu_segcblist_is_offloaded(&rdp->cblist));
    1697                 :            :         rcu_lockdep_assert_cblist_protected(rdp);
    1698                 :            :         lockdep_assert_held(&rdp->nocb_bypass_lock);
    1699                 :            :         if (rhp && !rcu_cblist_n_cbs(&rdp->nocb_bypass)) {
    1700                 :            :                 raw_spin_unlock(&rdp->nocb_bypass_lock);
    1701                 :            :                 return false;
    1702                 :            :         }
    1703                 :            :         /* Note: ->cblist.len already accounts for ->nocb_bypass contents. */
    1704                 :            :         if (rhp)
    1705                 :            :                 rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */
    1706                 :            :         rcu_cblist_flush_enqueue(&rcl, &rdp->nocb_bypass, rhp);
    1707                 :            :         rcu_segcblist_insert_pend_cbs(&rdp->cblist, &rcl);
    1708                 :            :         WRITE_ONCE(rdp->nocb_bypass_first, j);
    1709                 :            :         rcu_nocb_bypass_unlock(rdp);
    1710                 :            :         return true;
    1711                 :            : }
    1712                 :            : 
    1713                 :            : /*
    1714                 :            :  * Flush the ->nocb_bypass queue into ->cblist, enqueuing rhp if non-NULL.
    1715                 :            :  * However, if there is a callback to be enqueued and if ->nocb_bypass
    1716                 :            :  * proves to be initially empty, just return false because the no-CB GP
    1717                 :            :  * kthread may need to be awakened in this case.
    1718                 :            :  *
    1719                 :            :  * Note that this function always returns true if rhp is NULL.
    1720                 :            :  */
    1721                 :            : static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
    1722                 :            :                                   unsigned long j)
    1723                 :            : {
    1724                 :            :         if (!rcu_segcblist_is_offloaded(&rdp->cblist))
    1725                 :            :                 return true;
    1726                 :            :         rcu_lockdep_assert_cblist_protected(rdp);
    1727                 :            :         rcu_nocb_bypass_lock(rdp);
    1728                 :            :         return rcu_nocb_do_flush_bypass(rdp, rhp, j);
    1729                 :            : }
    1730                 :            : 
    1731                 :            : /*
    1732                 :            :  * If the ->nocb_bypass_lock is immediately available, flush the
    1733                 :            :  * ->nocb_bypass queue into ->cblist.
    1734                 :            :  */
    1735                 :            : static void rcu_nocb_try_flush_bypass(struct rcu_data *rdp, unsigned long j)
    1736                 :            : {
    1737                 :            :         rcu_lockdep_assert_cblist_protected(rdp);
    1738                 :            :         if (!rcu_segcblist_is_offloaded(&rdp->cblist) ||
    1739                 :            :             !rcu_nocb_bypass_trylock(rdp))
    1740                 :            :                 return;
    1741                 :            :         WARN_ON_ONCE(!rcu_nocb_do_flush_bypass(rdp, NULL, j));
    1742                 :            : }
    1743                 :            : 
    1744                 :            : /*
    1745                 :            :  * See whether it is appropriate to use the ->nocb_bypass list in order
    1746                 :            :  * to control contention on ->nocb_lock.  A limited number of direct
    1747                 :            :  * enqueues are permitted into ->cblist per jiffy.  If ->nocb_bypass
    1748                 :            :  * is non-empty, further callbacks must be placed into ->nocb_bypass,
    1749                 :            :  * otherwise rcu_barrier() breaks.  Use rcu_nocb_flush_bypass() to switch
    1750                 :            :  * back to direct use of ->cblist.  However, ->nocb_bypass should not be
    1751                 :            :  * used if ->cblist is empty, because otherwise callbacks can be stranded
    1752                 :            :  * on ->nocb_bypass because we cannot count on the current CPU ever again
    1753                 :            :  * invoking call_rcu().  The general rule is that if ->nocb_bypass is
    1754                 :            :  * non-empty, the corresponding no-CBs grace-period kthread must not be
    1755                 :            :  * in an indefinite sleep state.
    1756                 :            :  *
    1757                 :            :  * Finally, it is not permitted to use the bypass during early boot,
    1758                 :            :  * as doing so would confuse the auto-initialization code.  Besides
    1759                 :            :  * which, there is no point in worrying about lock contention while
    1760                 :            :  * there is only one CPU in operation.
    1761                 :            :  */
    1762                 :            : static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
    1763                 :            :                                 bool *was_alldone, unsigned long flags)
    1764                 :            : {
    1765                 :            :         unsigned long c;
    1766                 :            :         unsigned long cur_gp_seq;
    1767                 :            :         unsigned long j = jiffies;
    1768                 :            :         long ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
    1769                 :            : 
    1770                 :            :         if (!rcu_segcblist_is_offloaded(&rdp->cblist)) {
    1771                 :            :                 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
    1772                 :            :                 return false; /* Not offloaded, no bypassing. */
    1773                 :            :         }
    1774                 :            :         lockdep_assert_irqs_disabled();
    1775                 :            : 
    1776                 :            :         // Don't use ->nocb_bypass during early boot.
    1777                 :            :         if (rcu_scheduler_active != RCU_SCHEDULER_RUNNING) {
    1778                 :            :                 rcu_nocb_lock(rdp);
    1779                 :            :                 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
    1780                 :            :                 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
    1781                 :            :                 return false;
    1782                 :            :         }
    1783                 :            : 
    1784                 :            :         // If we have advanced to a new jiffy, reset counts to allow
    1785                 :            :         // moving back from ->nocb_bypass to ->cblist.
    1786                 :            :         if (j == rdp->nocb_nobypass_last) {
    1787                 :            :                 c = rdp->nocb_nobypass_count + 1;
    1788                 :            :         } else {
    1789                 :            :                 WRITE_ONCE(rdp->nocb_nobypass_last, j);
    1790                 :            :                 c = rdp->nocb_nobypass_count - nocb_nobypass_lim_per_jiffy;
    1791                 :            :                 if (ULONG_CMP_LT(rdp->nocb_nobypass_count,
    1792                 :            :                                  nocb_nobypass_lim_per_jiffy))
    1793                 :            :                         c = 0;
    1794                 :            :                 else if (c > nocb_nobypass_lim_per_jiffy)
    1795                 :            :                         c = nocb_nobypass_lim_per_jiffy;
    1796                 :            :         }
    1797                 :            :         WRITE_ONCE(rdp->nocb_nobypass_count, c);
    1798                 :            : 
    1799                 :            :         // If there hasn't yet been all that many ->cblist enqueues
    1800                 :            :         // this jiffy, tell the caller to enqueue onto ->cblist.  But flush
    1801                 :            :         // ->nocb_bypass first.
    1802                 :            :         if (rdp->nocb_nobypass_count < nocb_nobypass_lim_per_jiffy) {
    1803                 :            :                 rcu_nocb_lock(rdp);
    1804                 :            :                 *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
    1805                 :            :                 if (*was_alldone)
    1806                 :            :                         trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
    1807                 :            :                                             TPS("FirstQ"));
    1808                 :            :                 WARN_ON_ONCE(!rcu_nocb_flush_bypass(rdp, NULL, j));
    1809                 :            :                 WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
    1810                 :            :                 return false; // Caller must enqueue the callback.
    1811                 :            :         }
    1812                 :            : 
    1813                 :            :         // If ->nocb_bypass has been used too long or is too full,
    1814                 :            :         // flush ->nocb_bypass to ->cblist.
    1815                 :            :         if ((ncbs && j != READ_ONCE(rdp->nocb_bypass_first)) ||
    1816                 :            :             ncbs >= qhimark) {
    1817                 :            :                 rcu_nocb_lock(rdp);
    1818                 :            :                 if (!rcu_nocb_flush_bypass(rdp, rhp, j)) {
    1819                 :            :                         *was_alldone = !rcu_segcblist_pend_cbs(&rdp->cblist);
    1820                 :            :                         if (*was_alldone)
    1821                 :            :                                 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
    1822                 :            :                                                     TPS("FirstQ"));
    1823                 :            :                         WARN_ON_ONCE(rcu_cblist_n_cbs(&rdp->nocb_bypass));
    1824                 :            :                         return false; // Caller must enqueue the callback.
    1825                 :            :                 }
    1826                 :            :                 if (j != rdp->nocb_gp_adv_time &&
    1827                 :            :                     rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&
    1828                 :            :                     rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) {
    1829                 :            :                         rcu_advance_cbs_nowake(rdp->mynode, rdp);
    1830                 :            :                         rdp->nocb_gp_adv_time = j;
    1831                 :            :                 }
    1832                 :            :                 rcu_nocb_unlock_irqrestore(rdp, flags);
    1833                 :            :                 return true; // Callback already enqueued.
    1834                 :            :         }
    1835                 :            : 
    1836                 :            :         // We need to use the bypass.
    1837                 :            :         rcu_nocb_wait_contended(rdp);
    1838                 :            :         rcu_nocb_bypass_lock(rdp);
    1839                 :            :         ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
    1840                 :            :         rcu_segcblist_inc_len(&rdp->cblist); /* Must precede enqueue. */
    1841                 :            :         rcu_cblist_enqueue(&rdp->nocb_bypass, rhp);
    1842                 :            :         if (!ncbs) {
    1843                 :            :                 WRITE_ONCE(rdp->nocb_bypass_first, j);
    1844                 :            :                 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ"));
    1845                 :            :         }
    1846                 :            :         rcu_nocb_bypass_unlock(rdp);
    1847                 :            :         smp_mb(); /* Order enqueue before wake. */
    1848                 :            :         if (ncbs) {
    1849                 :            :                 local_irq_restore(flags);
    1850                 :            :         } else {
    1851                 :            :                 // No-CBs GP kthread might be indefinitely asleep, if so, wake.
    1852                 :            :                 rcu_nocb_lock(rdp); // Rare during call_rcu() flood.
    1853                 :            :                 if (!rcu_segcblist_pend_cbs(&rdp->cblist)) {
    1854                 :            :                         trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
    1855                 :            :                                             TPS("FirstBQwake"));
    1856                 :            :                         __call_rcu_nocb_wake(rdp, true, flags);
    1857                 :            :                 } else {
    1858                 :            :                         trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
    1859                 :            :                                             TPS("FirstBQnoWake"));
    1860                 :            :                         rcu_nocb_unlock_irqrestore(rdp, flags);
    1861                 :            :                 }
    1862                 :            :         }
    1863                 :            :         return true; // Callback already enqueued.
    1864                 :            : }
    1865                 :            : 
    1866                 :            : /*
    1867                 :            :  * Awaken the no-CBs grace-period kthead if needed, either due to it
    1868                 :            :  * legitimately being asleep or due to overload conditions.
    1869                 :            :  *
    1870                 :            :  * If warranted, also wake up the kthread servicing this CPUs queues.
    1871                 :            :  */
    1872                 :            : static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone,
    1873                 :            :                                  unsigned long flags)
    1874                 :            :                                  __releases(rdp->nocb_lock)
    1875                 :            : {
    1876                 :            :         unsigned long cur_gp_seq;
    1877                 :            :         unsigned long j;
    1878                 :            :         long len;
    1879                 :            :         struct task_struct *t;
    1880                 :            : 
    1881                 :            :         // If we are being polled or there is no kthread, just leave.
    1882                 :            :         t = READ_ONCE(rdp->nocb_gp_kthread);
    1883                 :            :         if (rcu_nocb_poll || !t) {
    1884                 :            :                 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
    1885                 :            :                                     TPS("WakeNotPoll"));
    1886                 :            :                 rcu_nocb_unlock_irqrestore(rdp, flags);
    1887                 :            :                 return;
    1888                 :            :         }
    1889                 :            :         // Need to actually to a wakeup.
    1890                 :            :         len = rcu_segcblist_n_cbs(&rdp->cblist);
    1891                 :            :         if (was_alldone) {
    1892                 :            :                 rdp->qlen_last_fqs_check = len;
    1893                 :            :                 if (!irqs_disabled_flags(flags)) {
    1894                 :            :                         /* ... if queue was empty ... */
    1895                 :            :                         wake_nocb_gp(rdp, false, flags);
    1896                 :            :                         trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
    1897                 :            :                                             TPS("WakeEmpty"));
    1898                 :            :                 } else {
    1899                 :            :                         wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE,
    1900                 :            :                                            TPS("WakeEmptyIsDeferred"));
    1901                 :            :                         rcu_nocb_unlock_irqrestore(rdp, flags);
    1902                 :            :                 }
    1903                 :            :         } else if (len > rdp->qlen_last_fqs_check + qhimark) {
    1904                 :            :                 /* ... or if many callbacks queued. */
    1905                 :            :                 rdp->qlen_last_fqs_check = len;
    1906                 :            :                 j = jiffies;
    1907                 :            :                 if (j != rdp->nocb_gp_adv_time &&
    1908                 :            :                     rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&
    1909                 :            :                     rcu_seq_done(&rdp->mynode->gp_seq, cur_gp_seq)) {
    1910                 :            :                         rcu_advance_cbs_nowake(rdp->mynode, rdp);
    1911                 :            :                         rdp->nocb_gp_adv_time = j;
    1912                 :            :                 }
    1913                 :            :                 smp_mb(); /* Enqueue before timer_pending(). */
    1914                 :            :                 if ((rdp->nocb_cb_sleep ||
    1915                 :            :                      !rcu_segcblist_ready_cbs(&rdp->cblist)) &&
    1916                 :            :                     !timer_pending(&rdp->nocb_bypass_timer))
    1917                 :            :                         wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_FORCE,
    1918                 :            :                                            TPS("WakeOvfIsDeferred"));
    1919                 :            :                 rcu_nocb_unlock_irqrestore(rdp, flags);
    1920                 :            :         } else {
    1921                 :            :                 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot"));
    1922                 :            :                 rcu_nocb_unlock_irqrestore(rdp, flags);
    1923                 :            :         }
    1924                 :            :         return;
    1925                 :            : }
    1926                 :            : 
    1927                 :            : /* Wake up the no-CBs GP kthread to flush ->nocb_bypass. */
    1928                 :            : static void do_nocb_bypass_wakeup_timer(struct timer_list *t)
    1929                 :            : {
    1930                 :            :         unsigned long flags;
    1931                 :            :         struct rcu_data *rdp = from_timer(rdp, t, nocb_bypass_timer);
    1932                 :            : 
    1933                 :            :         trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer"));
    1934                 :            :         rcu_nocb_lock_irqsave(rdp, flags);
    1935                 :            :         smp_mb__after_spinlock(); /* Timer expire before wakeup. */
    1936                 :            :         __call_rcu_nocb_wake(rdp, true, flags);
    1937                 :            : }
    1938                 :            : 
    1939                 :            : /*
    1940                 :            :  * No-CBs GP kthreads come here to wait for additional callbacks to show up
    1941                 :            :  * or for grace periods to end.
    1942                 :            :  */
    1943                 :            : static void nocb_gp_wait(struct rcu_data *my_rdp)
    1944                 :            : {
    1945                 :            :         bool bypass = false;
    1946                 :            :         long bypass_ncbs;
    1947                 :            :         int __maybe_unused cpu = my_rdp->cpu;
    1948                 :            :         unsigned long cur_gp_seq;
    1949                 :            :         unsigned long flags;
    1950                 :            :         bool gotcbs = false;
    1951                 :            :         unsigned long j = jiffies;
    1952                 :            :         bool needwait_gp = false; // This prevents actual uninitialized use.
    1953                 :            :         bool needwake;
    1954                 :            :         bool needwake_gp;
    1955                 :            :         struct rcu_data *rdp;
    1956                 :            :         struct rcu_node *rnp;
    1957                 :            :         unsigned long wait_gp_seq = 0; // Suppress "use uninitialized" warning.
    1958                 :            : 
    1959                 :            :         /*
    1960                 :            :          * Each pass through the following loop checks for CBs and for the
    1961                 :            :          * nearest grace period (if any) to wait for next.  The CB kthreads
    1962                 :            :          * and the global grace-period kthread are awakened if needed.
    1963                 :            :          */
    1964                 :            :         for (rdp = my_rdp; rdp; rdp = rdp->nocb_next_cb_rdp) {
    1965                 :            :                 trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check"));
    1966                 :            :                 rcu_nocb_lock_irqsave(rdp, flags);
    1967                 :            :                 bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
    1968                 :            :                 if (bypass_ncbs &&
    1969                 :            :                     (time_after(j, READ_ONCE(rdp->nocb_bypass_first) + 1) ||
    1970                 :            :                      bypass_ncbs > 2 * qhimark)) {
    1971                 :            :                         // Bypass full or old, so flush it.
    1972                 :            :                         (void)rcu_nocb_try_flush_bypass(rdp, j);
    1973                 :            :                         bypass_ncbs = rcu_cblist_n_cbs(&rdp->nocb_bypass);
    1974                 :            :                 } else if (!bypass_ncbs && rcu_segcblist_empty(&rdp->cblist)) {
    1975                 :            :                         rcu_nocb_unlock_irqrestore(rdp, flags);
    1976                 :            :                         continue; /* No callbacks here, try next. */
    1977                 :            :                 }
    1978                 :            :                 if (bypass_ncbs) {
    1979                 :            :                         trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
    1980                 :            :                                             TPS("Bypass"));
    1981                 :            :                         bypass = true;
    1982                 :            :                 }
    1983                 :            :                 rnp = rdp->mynode;
    1984                 :            :                 if (bypass) {  // Avoid race with first bypass CB.
    1985                 :            :                         WRITE_ONCE(my_rdp->nocb_defer_wakeup,
    1986                 :            :                                    RCU_NOCB_WAKE_NOT);
    1987                 :            :                         del_timer(&my_rdp->nocb_timer);
    1988                 :            :                 }
    1989                 :            :                 // Advance callbacks if helpful and low contention.
    1990                 :            :                 needwake_gp = false;
    1991                 :            :                 if (!rcu_segcblist_restempty(&rdp->cblist,
    1992                 :            :                                              RCU_NEXT_READY_TAIL) ||
    1993                 :            :                     (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&
    1994                 :            :                      rcu_seq_done(&rnp->gp_seq, cur_gp_seq))) {
    1995                 :            :                         raw_spin_lock_rcu_node(rnp); /* irqs disabled. */
    1996                 :            :                         needwake_gp = rcu_advance_cbs(rnp, rdp);
    1997                 :            :                         raw_spin_unlock_rcu_node(rnp); /* irqs disabled. */
    1998                 :            :                 }
    1999                 :            :                 // Need to wait on some grace period?
    2000                 :            :                 WARN_ON_ONCE(!rcu_segcblist_restempty(&rdp->cblist,
    2001                 :            :                                                       RCU_NEXT_READY_TAIL));
    2002                 :            :                 if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) {
    2003                 :            :                         if (!needwait_gp ||
    2004                 :            :                             ULONG_CMP_LT(cur_gp_seq, wait_gp_seq))
    2005                 :            :                                 wait_gp_seq = cur_gp_seq;
    2006                 :            :                         needwait_gp = true;
    2007                 :            :                         trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,
    2008                 :            :                                             TPS("NeedWaitGP"));
    2009                 :            :                 }
    2010                 :            :                 if (rcu_segcblist_ready_cbs(&rdp->cblist)) {
    2011                 :            :                         needwake = rdp->nocb_cb_sleep;
    2012                 :            :                         WRITE_ONCE(rdp->nocb_cb_sleep, false);
    2013                 :            :                         smp_mb(); /* CB invocation -after- GP end. */
    2014                 :            :                 } else {
    2015                 :            :                         needwake = false;
    2016                 :            :                 }
    2017                 :            :                 rcu_nocb_unlock_irqrestore(rdp, flags);
    2018                 :            :                 if (needwake) {
    2019                 :            :                         swake_up_one(&rdp->nocb_cb_wq);
    2020                 :            :                         gotcbs = true;
    2021                 :            :                 }
    2022                 :            :                 if (needwake_gp)
    2023                 :            :                         rcu_gp_kthread_wake();
    2024                 :            :         }
    2025                 :            : 
    2026                 :            :         my_rdp->nocb_gp_bypass = bypass;
    2027                 :            :         my_rdp->nocb_gp_gp = needwait_gp;
    2028                 :            :         my_rdp->nocb_gp_seq = needwait_gp ? wait_gp_seq : 0;
    2029                 :            :         if (bypass && !rcu_nocb_poll) {
    2030                 :            :                 // At least one child with non-empty ->nocb_bypass, so set
    2031                 :            :                 // timer in order to avoid stranding its callbacks.
    2032                 :            :                 raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags);
    2033                 :            :                 mod_timer(&my_rdp->nocb_bypass_timer, j + 2);
    2034                 :            :                 raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags);
    2035                 :            :         }
    2036                 :            :         if (rcu_nocb_poll) {
    2037                 :            :                 /* Polling, so trace if first poll in the series. */
    2038                 :            :                 if (gotcbs)
    2039                 :            :                         trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Poll"));
    2040                 :            :                 schedule_timeout_interruptible(1);
    2041                 :            :         } else if (!needwait_gp) {
    2042                 :            :                 /* Wait for callbacks to appear. */
    2043                 :            :                 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep"));
    2044                 :            :                 swait_event_interruptible_exclusive(my_rdp->nocb_gp_wq,
    2045                 :            :                                 !READ_ONCE(my_rdp->nocb_gp_sleep));
    2046                 :            :                 trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep"));
    2047                 :            :         } else {
    2048                 :            :                 rnp = my_rdp->mynode;
    2049                 :            :                 trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("StartWait"));
    2050                 :            :                 swait_event_interruptible_exclusive(
    2051                 :            :                         rnp->nocb_gp_wq[rcu_seq_ctr(wait_gp_seq) & 0x1],
    2052                 :            :                         rcu_seq_done(&rnp->gp_seq, wait_gp_seq) ||
    2053                 :            :                         !READ_ONCE(my_rdp->nocb_gp_sleep));
    2054                 :            :                 trace_rcu_this_gp(rnp, my_rdp, wait_gp_seq, TPS("EndWait"));
    2055                 :            :         }
    2056                 :            :         if (!rcu_nocb_poll) {
    2057                 :            :                 raw_spin_lock_irqsave(&my_rdp->nocb_gp_lock, flags);
    2058                 :            :                 if (bypass)
    2059                 :            :                         del_timer(&my_rdp->nocb_bypass_timer);
    2060                 :            :                 WRITE_ONCE(my_rdp->nocb_gp_sleep, true);
    2061                 :            :                 raw_spin_unlock_irqrestore(&my_rdp->nocb_gp_lock, flags);
    2062                 :            :         }
    2063                 :            :         my_rdp->nocb_gp_seq = -1;
    2064                 :            :         WARN_ON(signal_pending(current));
    2065                 :            : }
    2066                 :            : 
    2067                 :            : /*
    2068                 :            :  * No-CBs grace-period-wait kthread.  There is one of these per group
    2069                 :            :  * of CPUs, but only once at least one CPU in that group has come online
    2070                 :            :  * at least once since boot.  This kthread checks for newly posted
    2071                 :            :  * callbacks from any of the CPUs it is responsible for, waits for a
    2072                 :            :  * grace period, then awakens all of the rcu_nocb_cb_kthread() instances
    2073                 :            :  * that then have callback-invocation work to do.
    2074                 :            :  */
    2075                 :            : static int rcu_nocb_gp_kthread(void *arg)
    2076                 :            : {
    2077                 :            :         struct rcu_data *rdp = arg;
    2078                 :            : 
    2079                 :            :         for (;;) {
    2080                 :            :                 WRITE_ONCE(rdp->nocb_gp_loops, rdp->nocb_gp_loops + 1);
    2081                 :            :                 nocb_gp_wait(rdp);
    2082                 :            :                 cond_resched_tasks_rcu_qs();
    2083                 :            :         }
    2084                 :            :         return 0;
    2085                 :            : }
    2086                 :            : 
    2087                 :            : /*
    2088                 :            :  * Invoke any ready callbacks from the corresponding no-CBs CPU,
    2089                 :            :  * then, if there are no more, wait for more to appear.
    2090                 :            :  */
    2091                 :            : static void nocb_cb_wait(struct rcu_data *rdp)
    2092                 :            : {
    2093                 :            :         unsigned long cur_gp_seq;
    2094                 :            :         unsigned long flags;
    2095                 :            :         bool needwake_gp = false;
    2096                 :            :         struct rcu_node *rnp = rdp->mynode;
    2097                 :            : 
    2098                 :            :         local_irq_save(flags);
    2099                 :            :         rcu_momentary_dyntick_idle();
    2100                 :            :         local_irq_restore(flags);
    2101                 :            :         local_bh_disable();
    2102                 :            :         rcu_do_batch(rdp);
    2103                 :            :         local_bh_enable();
    2104                 :            :         lockdep_assert_irqs_enabled();
    2105                 :            :         rcu_nocb_lock_irqsave(rdp, flags);
    2106                 :            :         if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq) &&
    2107                 :            :             rcu_seq_done(&rnp->gp_seq, cur_gp_seq) &&
    2108                 :            :             raw_spin_trylock_rcu_node(rnp)) { /* irqs already disabled. */
    2109                 :            :                 needwake_gp = rcu_advance_cbs(rdp->mynode, rdp);
    2110                 :            :                 raw_spin_unlock_rcu_node(rnp); /* irqs remain disabled. */
    2111                 :            :         }
    2112                 :            :         if (rcu_segcblist_ready_cbs(&rdp->cblist)) {
    2113                 :            :                 rcu_nocb_unlock_irqrestore(rdp, flags);
    2114                 :            :                 if (needwake_gp)
    2115                 :            :                         rcu_gp_kthread_wake();
    2116                 :            :                 return;
    2117                 :            :         }
    2118                 :            : 
    2119                 :            :         trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep"));
    2120                 :            :         WRITE_ONCE(rdp->nocb_cb_sleep, true);
    2121                 :            :         rcu_nocb_unlock_irqrestore(rdp, flags);
    2122                 :            :         if (needwake_gp)
    2123                 :            :                 rcu_gp_kthread_wake();
    2124                 :            :         swait_event_interruptible_exclusive(rdp->nocb_cb_wq,
    2125                 :            :                                  !READ_ONCE(rdp->nocb_cb_sleep));
    2126                 :            :         if (!smp_load_acquire(&rdp->nocb_cb_sleep)) { /* VVV */
    2127                 :            :                 /* ^^^ Ensure CB invocation follows _sleep test. */
    2128                 :            :                 return;
    2129                 :            :         }
    2130                 :            :         WARN_ON(signal_pending(current));
    2131                 :            :         trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty"));
    2132                 :            : }
    2133                 :            : 
    2134                 :            : /*
    2135                 :            :  * Per-rcu_data kthread, but only for no-CBs CPUs.  Repeatedly invoke
    2136                 :            :  * nocb_cb_wait() to do the dirty work.
    2137                 :            :  */
    2138                 :            : static int rcu_nocb_cb_kthread(void *arg)
    2139                 :            : {
    2140                 :            :         struct rcu_data *rdp = arg;
    2141                 :            : 
    2142                 :            :         // Each pass through this loop does one callback batch, and,
    2143                 :            :         // if there are no more ready callbacks, waits for them.
    2144                 :            :         for (;;) {
    2145                 :            :                 nocb_cb_wait(rdp);
    2146                 :            :                 cond_resched_tasks_rcu_qs();
    2147                 :            :         }
    2148                 :            :         return 0;
    2149                 :            : }
    2150                 :            : 
    2151                 :            : /* Is a deferred wakeup of rcu_nocb_kthread() required? */
    2152                 :            : static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
    2153                 :            : {
    2154                 :            :         return READ_ONCE(rdp->nocb_defer_wakeup);
    2155                 :            : }
    2156                 :            : 
    2157                 :            : /* Do a deferred wakeup of rcu_nocb_kthread(). */
    2158                 :            : static void do_nocb_deferred_wakeup_common(struct rcu_data *rdp)
    2159                 :            : {
    2160                 :            :         unsigned long flags;
    2161                 :            :         int ndw;
    2162                 :            : 
    2163                 :            :         rcu_nocb_lock_irqsave(rdp, flags);
    2164                 :            :         if (!rcu_nocb_need_deferred_wakeup(rdp)) {
    2165                 :            :                 rcu_nocb_unlock_irqrestore(rdp, flags);
    2166                 :            :                 return;
    2167                 :            :         }
    2168                 :            :         ndw = READ_ONCE(rdp->nocb_defer_wakeup);
    2169                 :            :         WRITE_ONCE(rdp->nocb_defer_wakeup, RCU_NOCB_WAKE_NOT);
    2170                 :            :         wake_nocb_gp(rdp, ndw == RCU_NOCB_WAKE_FORCE, flags);
    2171                 :            :         trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake"));
    2172                 :            : }
    2173                 :            : 
    2174                 :            : /* Do a deferred wakeup of rcu_nocb_kthread() from a timer handler. */
    2175                 :            : static void do_nocb_deferred_wakeup_timer(struct timer_list *t)
    2176                 :            : {
    2177                 :            :         struct rcu_data *rdp = from_timer(rdp, t, nocb_timer);
    2178                 :            : 
    2179                 :            :         do_nocb_deferred_wakeup_common(rdp);
    2180                 :            : }
    2181                 :            : 
    2182                 :            : /*
    2183                 :            :  * Do a deferred wakeup of rcu_nocb_kthread() from fastpath.
    2184                 :            :  * This means we do an inexact common-case check.  Note that if
    2185                 :            :  * we miss, ->nocb_timer will eventually clean things up.
    2186                 :            :  */
    2187                 :            : static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
    2188                 :            : {
    2189                 :            :         if (rcu_nocb_need_deferred_wakeup(rdp))
    2190                 :            :                 do_nocb_deferred_wakeup_common(rdp);
    2191                 :            : }
    2192                 :            : 
    2193                 :            : void __init rcu_init_nohz(void)
    2194                 :            : {
    2195                 :            :         int cpu;
    2196                 :            :         bool need_rcu_nocb_mask = false;
    2197                 :            :         struct rcu_data *rdp;
    2198                 :            : 
    2199                 :            : #if defined(CONFIG_NO_HZ_FULL)
    2200                 :            :         if (tick_nohz_full_running && cpumask_weight(tick_nohz_full_mask))
    2201                 :            :                 need_rcu_nocb_mask = true;
    2202                 :            : #endif /* #if defined(CONFIG_NO_HZ_FULL) */
    2203                 :            : 
    2204                 :            :         if (!cpumask_available(rcu_nocb_mask) && need_rcu_nocb_mask) {
    2205                 :            :                 if (!zalloc_cpumask_var(&rcu_nocb_mask, GFP_KERNEL)) {
    2206                 :            :                         pr_info("rcu_nocb_mask allocation failed, callback offloading disabled.\n");
    2207                 :            :                         return;
    2208                 :            :                 }
    2209                 :            :         }
    2210                 :            :         if (!cpumask_available(rcu_nocb_mask))
    2211                 :            :                 return;
    2212                 :            : 
    2213                 :            : #if defined(CONFIG_NO_HZ_FULL)
    2214                 :            :         if (tick_nohz_full_running)
    2215                 :            :                 cpumask_or(rcu_nocb_mask, rcu_nocb_mask, tick_nohz_full_mask);
    2216                 :            : #endif /* #if defined(CONFIG_NO_HZ_FULL) */
    2217                 :            : 
    2218                 :            :         if (!cpumask_subset(rcu_nocb_mask, cpu_possible_mask)) {
    2219                 :            :                 pr_info("\tNote: kernel parameter 'rcu_nocbs=', 'nohz_full', or 'isolcpus=' contains nonexistent CPUs.\n");
    2220                 :            :                 cpumask_and(rcu_nocb_mask, cpu_possible_mask,
    2221                 :            :                             rcu_nocb_mask);
    2222                 :            :         }
    2223                 :            :         if (cpumask_empty(rcu_nocb_mask))
    2224                 :            :                 pr_info("\tOffload RCU callbacks from CPUs: (none).\n");
    2225                 :            :         else
    2226                 :            :                 pr_info("\tOffload RCU callbacks from CPUs: %*pbl.\n",
    2227                 :            :                         cpumask_pr_args(rcu_nocb_mask));
    2228                 :            :         if (rcu_nocb_poll)
    2229                 :            :                 pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
    2230                 :            : 
    2231                 :            :         for_each_cpu(cpu, rcu_nocb_mask) {
    2232                 :            :                 rdp = per_cpu_ptr(&rcu_data, cpu);
    2233                 :            :                 if (rcu_segcblist_empty(&rdp->cblist))
    2234                 :            :                         rcu_segcblist_init(&rdp->cblist);
    2235                 :            :                 rcu_segcblist_offload(&rdp->cblist);
    2236                 :            :         }
    2237                 :            :         rcu_organize_nocb_kthreads();
    2238                 :            : }
    2239                 :            : 
    2240                 :            : /* Initialize per-rcu_data variables for no-CBs CPUs. */
    2241                 :            : static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
    2242                 :            : {
    2243                 :            :         init_swait_queue_head(&rdp->nocb_cb_wq);
    2244                 :            :         init_swait_queue_head(&rdp->nocb_gp_wq);
    2245                 :            :         raw_spin_lock_init(&rdp->nocb_lock);
    2246                 :            :         raw_spin_lock_init(&rdp->nocb_bypass_lock);
    2247                 :            :         raw_spin_lock_init(&rdp->nocb_gp_lock);
    2248                 :            :         timer_setup(&rdp->nocb_timer, do_nocb_deferred_wakeup_timer, 0);
    2249                 :            :         timer_setup(&rdp->nocb_bypass_timer, do_nocb_bypass_wakeup_timer, 0);
    2250                 :            :         rcu_cblist_init(&rdp->nocb_bypass);
    2251                 :            : }
    2252                 :            : 
    2253                 :            : /*
    2254                 :            :  * If the specified CPU is a no-CBs CPU that does not already have its
    2255                 :            :  * rcuo CB kthread, spawn it.  Additionally, if the rcuo GP kthread
    2256                 :            :  * for this CPU's group has not yet been created, spawn it as well.
    2257                 :            :  */
    2258                 :            : static void rcu_spawn_one_nocb_kthread(int cpu)
    2259                 :            : {
    2260                 :            :         struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
    2261                 :            :         struct rcu_data *rdp_gp;
    2262                 :            :         struct task_struct *t;
    2263                 :            : 
    2264                 :            :         /*
    2265                 :            :          * If this isn't a no-CBs CPU or if it already has an rcuo kthread,
    2266                 :            :          * then nothing to do.
    2267                 :            :          */
    2268                 :            :         if (!rcu_is_nocb_cpu(cpu) || rdp->nocb_cb_kthread)
    2269                 :            :                 return;
    2270                 :            : 
    2271                 :            :         /* If we didn't spawn the GP kthread first, reorganize! */
    2272                 :            :         rdp_gp = rdp->nocb_gp_rdp;
    2273                 :            :         if (!rdp_gp->nocb_gp_kthread) {
    2274                 :            :                 t = kthread_run(rcu_nocb_gp_kthread, rdp_gp,
    2275                 :            :                                 "rcuog/%d", rdp_gp->cpu);
    2276                 :            :                 if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo GP kthread, OOM is now expected behavior\n", __func__))
    2277                 :            :                         return;
    2278                 :            :                 WRITE_ONCE(rdp_gp->nocb_gp_kthread, t);
    2279                 :            :         }
    2280                 :            : 
    2281                 :            :         /* Spawn the kthread for this CPU. */
    2282                 :            :         t = kthread_run(rcu_nocb_cb_kthread, rdp,
    2283                 :            :                         "rcuo%c/%d", rcu_state.abbr, cpu);
    2284                 :            :         if (WARN_ONCE(IS_ERR(t), "%s: Could not start rcuo CB kthread, OOM is now expected behavior\n", __func__))
    2285                 :            :                 return;
    2286                 :            :         WRITE_ONCE(rdp->nocb_cb_kthread, t);
    2287                 :            :         WRITE_ONCE(rdp->nocb_gp_kthread, rdp_gp->nocb_gp_kthread);
    2288                 :            : }
    2289                 :            : 
    2290                 :            : /*
    2291                 :            :  * If the specified CPU is a no-CBs CPU that does not already have its
    2292                 :            :  * rcuo kthread, spawn it.
    2293                 :            :  */
    2294                 :            : static void rcu_spawn_cpu_nocb_kthread(int cpu)
    2295                 :            : {
    2296                 :            :         if (rcu_scheduler_fully_active)
    2297                 :            :                 rcu_spawn_one_nocb_kthread(cpu);
    2298                 :            : }
    2299                 :            : 
    2300                 :            : /*
    2301                 :            :  * Once the scheduler is running, spawn rcuo kthreads for all online
    2302                 :            :  * no-CBs CPUs.  This assumes that the early_initcall()s happen before
    2303                 :            :  * non-boot CPUs come online -- if this changes, we will need to add
    2304                 :            :  * some mutual exclusion.
    2305                 :            :  */
    2306                 :            : static void __init rcu_spawn_nocb_kthreads(void)
    2307                 :            : {
    2308                 :            :         int cpu;
    2309                 :            : 
    2310                 :            :         for_each_online_cpu(cpu)
    2311                 :            :                 rcu_spawn_cpu_nocb_kthread(cpu);
    2312                 :            : }
    2313                 :            : 
    2314                 :            : /* How many CB CPU IDs per GP kthread?  Default of -1 for sqrt(nr_cpu_ids). */
    2315                 :            : static int rcu_nocb_gp_stride = -1;
    2316                 :            : module_param(rcu_nocb_gp_stride, int, 0444);
    2317                 :            : 
    2318                 :            : /*
    2319                 :            :  * Initialize GP-CB relationships for all no-CBs CPU.
    2320                 :            :  */
    2321                 :            : static void __init rcu_organize_nocb_kthreads(void)
    2322                 :            : {
    2323                 :            :         int cpu;
    2324                 :            :         bool firsttime = true;
    2325                 :            :         bool gotnocbs = false;
    2326                 :            :         bool gotnocbscbs = true;
    2327                 :            :         int ls = rcu_nocb_gp_stride;
    2328                 :            :         int nl = 0;  /* Next GP kthread. */
    2329                 :            :         struct rcu_data *rdp;
    2330                 :            :         struct rcu_data *rdp_gp = NULL;  /* Suppress misguided gcc warn. */
    2331                 :            :         struct rcu_data *rdp_prev = NULL;
    2332                 :            : 
    2333                 :            :         if (!cpumask_available(rcu_nocb_mask))
    2334                 :            :                 return;
    2335                 :            :         if (ls == -1) {
    2336                 :            :                 ls = nr_cpu_ids / int_sqrt(nr_cpu_ids);
    2337                 :            :                 rcu_nocb_gp_stride = ls;
    2338                 :            :         }
    2339                 :            : 
    2340                 :            :         /*
    2341                 :            :          * Each pass through this loop sets up one rcu_data structure.
    2342                 :            :          * Should the corresponding CPU come online in the future, then
    2343                 :            :          * we will spawn the needed set of rcu_nocb_kthread() kthreads.
    2344                 :            :          */
    2345                 :            :         for_each_cpu(cpu, rcu_nocb_mask) {
    2346                 :            :                 rdp = per_cpu_ptr(&rcu_data, cpu);
    2347                 :            :                 if (rdp->cpu >= nl) {
    2348                 :            :                         /* New GP kthread, set up for CBs & next GP. */
    2349                 :            :                         gotnocbs = true;
    2350                 :            :                         nl = DIV_ROUND_UP(rdp->cpu + 1, ls) * ls;
    2351                 :            :                         rdp->nocb_gp_rdp = rdp;
    2352                 :            :                         rdp_gp = rdp;
    2353                 :            :                         if (dump_tree) {
    2354                 :            :                                 if (!firsttime)
    2355                 :            :                                         pr_cont("%s\n", gotnocbscbs
    2356                 :            :                                                         ? "" : " (self only)");
    2357                 :            :                                 gotnocbscbs = false;
    2358                 :            :                                 firsttime = false;
    2359                 :            :                                 pr_alert("%s: No-CB GP kthread CPU %d:",
    2360                 :            :                                          __func__, cpu);
    2361                 :            :                         }
    2362                 :            :                 } else {
    2363                 :            :                         /* Another CB kthread, link to previous GP kthread. */
    2364                 :            :                         gotnocbscbs = true;
    2365                 :            :                         rdp->nocb_gp_rdp = rdp_gp;
    2366                 :            :                         rdp_prev->nocb_next_cb_rdp = rdp;
    2367                 :            :                         if (dump_tree)
    2368                 :            :                                 pr_cont(" %d", cpu);
    2369                 :            :                 }
    2370                 :            :                 rdp_prev = rdp;
    2371                 :            :         }
    2372                 :            :         if (gotnocbs && dump_tree)
    2373                 :            :                 pr_cont("%s\n", gotnocbscbs ? "" : " (self only)");
    2374                 :            : }
    2375                 :            : 
    2376                 :            : /*
    2377                 :            :  * Bind the current task to the offloaded CPUs.  If there are no offloaded
    2378                 :            :  * CPUs, leave the task unbound.  Splat if the bind attempt fails.
    2379                 :            :  */
    2380                 :            : void rcu_bind_current_to_nocb(void)
    2381                 :            : {
    2382                 :            :         if (cpumask_available(rcu_nocb_mask) && cpumask_weight(rcu_nocb_mask))
    2383                 :            :                 WARN_ON(sched_setaffinity(current->pid, rcu_nocb_mask));
    2384                 :            : }
    2385                 :            : EXPORT_SYMBOL_GPL(rcu_bind_current_to_nocb);
    2386                 :            : 
    2387                 :            : /*
    2388                 :            :  * Dump out nocb grace-period kthread state for the specified rcu_data
    2389                 :            :  * structure.
    2390                 :            :  */
    2391                 :            : static void show_rcu_nocb_gp_state(struct rcu_data *rdp)
    2392                 :            : {
    2393                 :            :         struct rcu_node *rnp = rdp->mynode;
    2394                 :            : 
    2395                 :            :         pr_info("nocb GP %d %c%c%c%c%c%c %c[%c%c] %c%c:%ld rnp %d:%d %lu\n",
    2396                 :            :                 rdp->cpu,
    2397                 :            :                 "kK"[!!rdp->nocb_gp_kthread],
    2398                 :            :                 "lL"[raw_spin_is_locked(&rdp->nocb_gp_lock)],
    2399                 :            :                 "dD"[!!rdp->nocb_defer_wakeup],
    2400                 :            :                 "tT"[timer_pending(&rdp->nocb_timer)],
    2401                 :            :                 "bB"[timer_pending(&rdp->nocb_bypass_timer)],
    2402                 :            :                 "sS"[!!rdp->nocb_gp_sleep],
    2403                 :            :                 ".W"[swait_active(&rdp->nocb_gp_wq)],
    2404                 :            :                 ".W"[swait_active(&rnp->nocb_gp_wq[0])],
    2405                 :            :                 ".W"[swait_active(&rnp->nocb_gp_wq[1])],
    2406                 :            :                 ".B"[!!rdp->nocb_gp_bypass],
    2407                 :            :                 ".G"[!!rdp->nocb_gp_gp],
    2408                 :            :                 (long)rdp->nocb_gp_seq,
    2409                 :            :                 rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops));
    2410                 :            : }
    2411                 :            : 
    2412                 :            : /* Dump out nocb kthread state for the specified rcu_data structure. */
    2413                 :            : static void show_rcu_nocb_state(struct rcu_data *rdp)
    2414                 :            : {
    2415                 :            :         struct rcu_segcblist *rsclp = &rdp->cblist;
    2416                 :            :         bool waslocked;
    2417                 :            :         bool wastimer;
    2418                 :            :         bool wassleep;
    2419                 :            : 
    2420                 :            :         if (rdp->nocb_gp_rdp == rdp)
    2421                 :            :                 show_rcu_nocb_gp_state(rdp);
    2422                 :            : 
    2423                 :            :         pr_info("   CB %d->%d %c%c%c%c%c%c F%ld L%ld C%d %c%c%c%c%c q%ld\n",
    2424                 :            :                 rdp->cpu, rdp->nocb_gp_rdp->cpu,
    2425                 :            :                 "kK"[!!rdp->nocb_cb_kthread],
    2426                 :            :                 "bB"[raw_spin_is_locked(&rdp->nocb_bypass_lock)],
    2427                 :            :                 "cC"[!!atomic_read(&rdp->nocb_lock_contended)],
    2428                 :            :                 "lL"[raw_spin_is_locked(&rdp->nocb_lock)],
    2429                 :            :                 "sS"[!!rdp->nocb_cb_sleep],
    2430                 :            :                 ".W"[swait_active(&rdp->nocb_cb_wq)],
    2431                 :            :                 jiffies - rdp->nocb_bypass_first,
    2432                 :            :                 jiffies - rdp->nocb_nobypass_last,
    2433                 :            :                 rdp->nocb_nobypass_count,
    2434                 :            :                 ".D"[rcu_segcblist_ready_cbs(rsclp)],
    2435                 :            :                 ".W"[!rcu_segcblist_restempty(rsclp, RCU_DONE_TAIL)],
    2436                 :            :                 ".R"[!rcu_segcblist_restempty(rsclp, RCU_WAIT_TAIL)],
    2437                 :            :                 ".N"[!rcu_segcblist_restempty(rsclp, RCU_NEXT_READY_TAIL)],
    2438                 :            :                 ".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)],
    2439                 :            :                 rcu_segcblist_n_cbs(&rdp->cblist));
    2440                 :            : 
    2441                 :            :         /* It is OK for GP kthreads to have GP state. */
    2442                 :            :         if (rdp->nocb_gp_rdp == rdp)
    2443                 :            :                 return;
    2444                 :            : 
    2445                 :            :         waslocked = raw_spin_is_locked(&rdp->nocb_gp_lock);
    2446                 :            :         wastimer = timer_pending(&rdp->nocb_timer);
    2447                 :            :         wassleep = swait_active(&rdp->nocb_gp_wq);
    2448                 :            :         if (!rdp->nocb_defer_wakeup && !rdp->nocb_gp_sleep &&
    2449                 :            :             !waslocked && !wastimer && !wassleep)
    2450                 :            :                 return;  /* Nothing untowards. */
    2451                 :            : 
    2452                 :            :         pr_info("   !!! %c%c%c%c %c\n",
    2453                 :            :                 "lL"[waslocked],
    2454                 :            :                 "dD"[!!rdp->nocb_defer_wakeup],
    2455                 :            :                 "tT"[wastimer],
    2456                 :            :                 "sS"[!!rdp->nocb_gp_sleep],
    2457                 :            :                 ".W"[wassleep]);
    2458                 :            : }
    2459                 :            : 
    2460                 :            : #else /* #ifdef CONFIG_RCU_NOCB_CPU */
    2461                 :            : 
    2462                 :            : /* No ->nocb_lock to acquire.  */
    2463                 :            : static void rcu_nocb_lock(struct rcu_data *rdp)
    2464                 :            : {
    2465                 :            : }
    2466                 :            : 
    2467                 :            : /* No ->nocb_lock to release.  */
    2468                 :            : static void rcu_nocb_unlock(struct rcu_data *rdp)
    2469                 :            : {
    2470                 :            : }
    2471                 :            : 
    2472                 :            : /* No ->nocb_lock to release.  */
    2473                 :          3 : static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp,
    2474                 :            :                                        unsigned long flags)
    2475                 :            : {
    2476                 :          3 :         local_irq_restore(flags);
    2477                 :          3 : }
    2478                 :            : 
    2479                 :            : /* Lockdep check that ->cblist may be safely accessed. */
    2480                 :            : static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)
    2481                 :            : {
    2482                 :            :         lockdep_assert_irqs_disabled();
    2483                 :            : }
    2484                 :            : 
    2485                 :            : static void rcu_nocb_gp_cleanup(struct swait_queue_head *sq)
    2486                 :            : {
    2487                 :            : }
    2488                 :            : 
    2489                 :            : static struct swait_queue_head *rcu_nocb_gp_get(struct rcu_node *rnp)
    2490                 :            : {
    2491                 :            :         return NULL;
    2492                 :            : }
    2493                 :            : 
    2494                 :            : static void rcu_init_one_nocb(struct rcu_node *rnp)
    2495                 :            : {
    2496                 :            : }
    2497                 :            : 
    2498                 :            : static bool rcu_nocb_flush_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
    2499                 :            :                                   unsigned long j)
    2500                 :            : {
    2501                 :            :         return true;
    2502                 :            : }
    2503                 :            : 
    2504                 :            : static bool rcu_nocb_try_bypass(struct rcu_data *rdp, struct rcu_head *rhp,
    2505                 :            :                                 bool *was_alldone, unsigned long flags)
    2506                 :            : {
    2507                 :            :         return false;
    2508                 :            : }
    2509                 :            : 
    2510                 :            : static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_empty,
    2511                 :            :                                  unsigned long flags)
    2512                 :            : {
    2513                 :            :         WARN_ON_ONCE(1);  /* Should be dead code! */
    2514                 :            : }
    2515                 :            : 
    2516                 :            : static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp)
    2517                 :            : {
    2518                 :            : }
    2519                 :            : 
    2520                 :            : static int rcu_nocb_need_deferred_wakeup(struct rcu_data *rdp)
    2521                 :            : {
    2522                 :            :         return false;
    2523                 :            : }
    2524                 :            : 
    2525                 :            : static void do_nocb_deferred_wakeup(struct rcu_data *rdp)
    2526                 :            : {
    2527                 :            : }
    2528                 :            : 
    2529                 :            : static void rcu_spawn_cpu_nocb_kthread(int cpu)
    2530                 :            : {
    2531                 :            : }
    2532                 :            : 
    2533                 :            : static void __init rcu_spawn_nocb_kthreads(void)
    2534                 :            : {
    2535                 :            : }
    2536                 :            : 
    2537                 :            : static void show_rcu_nocb_state(struct rcu_data *rdp)
    2538                 :            : {
    2539                 :            : }
    2540                 :            : 
    2541                 :            : #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
    2542                 :            : 
    2543                 :            : /*
    2544                 :            :  * Is this CPU a NO_HZ_FULL CPU that should ignore RCU so that the
    2545                 :            :  * grace-period kthread will do force_quiescent_state() processing?
    2546                 :            :  * The idea is to avoid waking up RCU core processing on such a
    2547                 :            :  * CPU unless the grace period has extended for too long.
    2548                 :            :  *
    2549                 :            :  * This code relies on the fact that all NO_HZ_FULL CPUs are also
    2550                 :            :  * CONFIG_RCU_NOCB_CPU CPUs.
    2551                 :            :  */
    2552                 :            : static bool rcu_nohz_full_cpu(void)
    2553                 :            : {
    2554                 :            : #ifdef CONFIG_NO_HZ_FULL
    2555                 :            :         if (tick_nohz_full_cpu(smp_processor_id()) &&
    2556                 :            :             (!rcu_gp_in_progress() ||
    2557                 :            :              ULONG_CMP_LT(jiffies, READ_ONCE(rcu_state.gp_start) + HZ)))
    2558                 :            :                 return true;
    2559                 :            : #endif /* #ifdef CONFIG_NO_HZ_FULL */
    2560                 :            :         return false;
    2561                 :            : }
    2562                 :            : 
    2563                 :            : /*
    2564                 :            :  * Bind the RCU grace-period kthreads to the housekeeping CPU.
    2565                 :            :  */
    2566                 :            : static void rcu_bind_gp_kthread(void)
    2567                 :            : {
    2568                 :            :         if (!tick_nohz_full_enabled())
    2569                 :            :                 return;
    2570                 :            :         housekeeping_affine(current, HK_FLAG_RCU);
    2571                 :            : }
    2572                 :            : 
    2573                 :            : /* Record the current task on dyntick-idle entry. */
    2574                 :          3 : static void rcu_dynticks_task_enter(void)
    2575                 :            : {
    2576                 :            : #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
    2577                 :            :         WRITE_ONCE(current->rcu_tasks_idle_cpu, smp_processor_id());
    2578                 :            : #endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
    2579                 :          3 : }
    2580                 :            : 
    2581                 :            : /* Record no current task on dyntick-idle exit. */
    2582                 :          3 : static void rcu_dynticks_task_exit(void)
    2583                 :            : {
    2584                 :            : #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL)
    2585                 :            :         WRITE_ONCE(current->rcu_tasks_idle_cpu, -1);
    2586                 :            : #endif /* #if defined(CONFIG_TASKS_RCU) && defined(CONFIG_NO_HZ_FULL) */
    2587                 :          3 : }
    

Generated by: LCOV version 1.14