Branch data Line data Source code
1 : : // SPDX-License-Identifier: GPL-2.0+ 2 : : /* 3 : : * RCU-based infrastructure for lightweight reader-writer locking 4 : : * 5 : : * Copyright (c) 2015, Red Hat, Inc. 6 : : * 7 : : * Author: Oleg Nesterov <oleg@redhat.com> 8 : : */ 9 : : 10 : : #include <linux/rcu_sync.h> 11 : : #include <linux/sched.h> 12 : : 13 : : enum { GP_IDLE = 0, GP_ENTER, GP_PASSED, GP_EXIT, GP_REPLAY }; 14 : : 15 : : #define rss_lock gp_wait.lock 16 : : 17 : : /** 18 : : * rcu_sync_init() - Initialize an rcu_sync structure 19 : : * @rsp: Pointer to rcu_sync structure to be initialized 20 : : */ 21 : 3420 : void rcu_sync_init(struct rcu_sync *rsp) 22 : : { 23 : 3420 : memset(rsp, 0, sizeof(*rsp)); 24 : 3420 : init_waitqueue_head(&rsp->gp_wait); 25 : 3420 : } 26 : : 27 : : /** 28 : : * rcu_sync_enter_start - Force readers onto slow path for multiple updates 29 : : * @rsp: Pointer to rcu_sync structure to use for synchronization 30 : : * 31 : : * Must be called after rcu_sync_init() and before first use. 32 : : * 33 : : * Ensures rcu_sync_is_idle() returns false and rcu_sync_{enter,exit}() 34 : : * pairs turn into NO-OPs. 35 : : */ 36 : 30 : void rcu_sync_enter_start(struct rcu_sync *rsp) 37 : : { 38 : 30 : rsp->gp_count++; 39 : 30 : rsp->gp_state = GP_PASSED; 40 : 30 : } 41 : : 42 : : 43 : : static void rcu_sync_func(struct rcu_head *rhp); 44 : : 45 : 60 : static void rcu_sync_call(struct rcu_sync *rsp) 46 : : { 47 : 60 : call_rcu(&rsp->cb_head, rcu_sync_func); 48 : 60 : } 49 : : 50 : : /** 51 : : * rcu_sync_func() - Callback function managing reader access to fastpath 52 : : * @rhp: Pointer to rcu_head in rcu_sync structure to use for synchronization 53 : : * 54 : : * This function is passed to call_rcu() function by rcu_sync_enter() and 55 : : * rcu_sync_exit(), so that it is invoked after a grace period following the 56 : : * that invocation of enter/exit. 57 : : * 58 : : * If it is called by rcu_sync_enter() it signals that all the readers were 59 : : * switched onto slow path. 60 : : * 61 : : * If it is called by rcu_sync_exit() it takes action based on events that 62 : : * have taken place in the meantime, so that closely spaced rcu_sync_enter() 63 : : * and rcu_sync_exit() pairs need not wait for a grace period. 64 : : * 65 : : * If another rcu_sync_enter() is invoked before the grace period 66 : : * ended, reset state to allow the next rcu_sync_exit() to let the 67 : : * readers back onto their fastpaths (after a grace period). If both 68 : : * another rcu_sync_enter() and its matching rcu_sync_exit() are invoked 69 : : * before the grace period ended, re-invoke call_rcu() on behalf of that 70 : : * rcu_sync_exit(). Otherwise, set all state back to idle so that readers 71 : : * can again use their fastpaths. 72 : : */ 73 : 120 : static void rcu_sync_func(struct rcu_head *rhp) 74 : : { 75 : 120 : struct rcu_sync *rsp = container_of(rhp, struct rcu_sync, cb_head); 76 : 120 : unsigned long flags; 77 : : 78 [ - + ]: 120 : WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_IDLE); 79 [ - + ]: 120 : WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_PASSED); 80 : : 81 : 120 : spin_lock_irqsave(&rsp->rss_lock, flags); 82 [ + + ]: 120 : if (rsp->gp_count) { 83 : : /* 84 : : * We're at least a GP after the GP_IDLE->GP_ENTER transition. 85 : : */ 86 : 60 : WRITE_ONCE(rsp->gp_state, GP_PASSED); 87 : 60 : wake_up_locked(&rsp->gp_wait); 88 [ - + ]: 60 : } else if (rsp->gp_state == GP_REPLAY) { 89 : : /* 90 : : * A new rcu_sync_exit() has happened; requeue the callback to 91 : : * catch a later GP. 92 : : */ 93 : 0 : WRITE_ONCE(rsp->gp_state, GP_EXIT); 94 : 0 : rcu_sync_call(rsp); 95 : : } else { 96 : : /* 97 : : * We're at least a GP after the last rcu_sync_exit(); eveybody 98 : : * will now have observed the write side critical section. 99 : : * Let 'em rip!. 100 : : */ 101 : 60 : WRITE_ONCE(rsp->gp_state, GP_IDLE); 102 : : } 103 : 120 : spin_unlock_irqrestore(&rsp->rss_lock, flags); 104 : 120 : } 105 : : 106 : : /** 107 : : * rcu_sync_enter() - Force readers onto slowpath 108 : : * @rsp: Pointer to rcu_sync structure to use for synchronization 109 : : * 110 : : * This function is used by updaters who need readers to make use of 111 : : * a slowpath during the update. After this function returns, all 112 : : * subsequent calls to rcu_sync_is_idle() will return false, which 113 : : * tells readers to stay off their fastpaths. A later call to 114 : : * rcu_sync_exit() re-enables reader slowpaths. 115 : : * 116 : : * When called in isolation, rcu_sync_enter() must wait for a grace 117 : : * period, however, closely spaced calls to rcu_sync_enter() can 118 : : * optimize away the grace-period wait via a state machine implemented 119 : : * by rcu_sync_enter(), rcu_sync_exit(), and rcu_sync_func(). 120 : : */ 121 : 4590 : void rcu_sync_enter(struct rcu_sync *rsp) 122 : : { 123 : 4590 : int gp_state; 124 : : 125 : 4590 : spin_lock_irq(&rsp->rss_lock); 126 : 4590 : gp_state = rsp->gp_state; 127 [ + + ]: 4590 : if (gp_state == GP_IDLE) { 128 [ - + ]: 60 : WRITE_ONCE(rsp->gp_state, GP_ENTER); 129 [ - + ]: 60 : WARN_ON_ONCE(rsp->gp_count); 130 : : /* 131 : : * Note that we could simply do rcu_sync_call(rsp) here and 132 : : * avoid the "if (gp_state == GP_IDLE)" block below. 133 : : * 134 : : * However, synchronize_rcu() can be faster if rcu_expedited 135 : : * or rcu_blocking_is_gp() is true. 136 : : * 137 : : * Another reason is that we can't wait for rcu callback if 138 : : * we are called at early boot time but this shouldn't happen. 139 : : */ 140 : : } 141 : 4590 : rsp->gp_count++; 142 : 4590 : spin_unlock_irq(&rsp->rss_lock); 143 : : 144 [ + + ]: 4590 : if (gp_state == GP_IDLE) { 145 : : /* 146 : : * See the comment above, this simply does the "synchronous" 147 : : * call_rcu(rcu_sync_func) which does GP_ENTER -> GP_PASSED. 148 : : */ 149 : 60 : synchronize_rcu(); 150 : 60 : rcu_sync_func(&rsp->cb_head); 151 : : /* Not really needed, wait_event() would see GP_PASSED. */ 152 : 60 : return; 153 : : } 154 : : 155 [ - + - - ]: 4530 : wait_event(rsp->gp_wait, READ_ONCE(rsp->gp_state) >= GP_PASSED); 156 : : } 157 : : 158 : : /** 159 : : * rcu_sync_exit() - Allow readers back onto fast path after grace period 160 : : * @rsp: Pointer to rcu_sync structure to use for synchronization 161 : : * 162 : : * This function is used by updaters who have completed, and can therefore 163 : : * now allow readers to make use of their fastpaths after a grace period 164 : : * has elapsed. After this grace period has completed, all subsequent 165 : : * calls to rcu_sync_is_idle() will return true, which tells readers that 166 : : * they can once again use their fastpaths. 167 : : */ 168 : 4590 : void rcu_sync_exit(struct rcu_sync *rsp) 169 : : { 170 [ - + ]: 4590 : WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_IDLE); 171 [ - + ]: 4590 : WARN_ON_ONCE(READ_ONCE(rsp->gp_count) == 0); 172 : : 173 : 4590 : spin_lock_irq(&rsp->rss_lock); 174 [ + + ]: 4590 : if (!--rsp->gp_count) { 175 [ + - ]: 60 : if (rsp->gp_state == GP_PASSED) { 176 : 60 : WRITE_ONCE(rsp->gp_state, GP_EXIT); 177 : 60 : rcu_sync_call(rsp); 178 [ # # ]: 0 : } else if (rsp->gp_state == GP_EXIT) { 179 : 0 : WRITE_ONCE(rsp->gp_state, GP_REPLAY); 180 : : } 181 : : } 182 : 4590 : spin_unlock_irq(&rsp->rss_lock); 183 : 4590 : } 184 : : 185 : : /** 186 : : * rcu_sync_dtor() - Clean up an rcu_sync structure 187 : : * @rsp: Pointer to rcu_sync structure to be cleaned up 188 : : */ 189 : 180 : void rcu_sync_dtor(struct rcu_sync *rsp) 190 : : { 191 : 180 : int gp_state; 192 : : 193 [ - + ]: 180 : WARN_ON_ONCE(READ_ONCE(rsp->gp_count)); 194 [ - + ]: 180 : WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_PASSED); 195 : : 196 : 180 : spin_lock_irq(&rsp->rss_lock); 197 [ - + ]: 180 : if (rsp->gp_state == GP_REPLAY) 198 : 0 : WRITE_ONCE(rsp->gp_state, GP_EXIT); 199 : 180 : gp_state = rsp->gp_state; 200 : 180 : spin_unlock_irq(&rsp->rss_lock); 201 : : 202 [ - + ]: 180 : if (gp_state != GP_IDLE) { 203 : 0 : rcu_barrier(); 204 [ # # ]: 0 : WARN_ON_ONCE(rsp->gp_state != GP_IDLE); 205 : : } 206 : 180 : }