Branch data Line data Source code
1 : : // SPDX-License-Identifier: GPL-2.0
2 : : /*
3 : : * Generic wait-for-completion handler;
4 : : *
5 : : * It differs from semaphores in that their default case is the opposite,
6 : : * wait_for_completion default blocks whereas semaphore default non-block. The
7 : : * interface also makes it easy to 'complete' multiple waiting threads,
8 : : * something which isn't entirely natural for semaphores.
9 : : *
10 : : * But more importantly, the primitive documents the usage. Semaphores would
11 : : * typically be used for exclusion which gives rise to priority inversion.
12 : : * Waiting for completion is a typically sync point, but not an exclusion point.
13 : : */
14 : : #include "sched.h"
15 : :
16 : : /**
17 : : * complete: - signals a single thread waiting on this completion
18 : : * @x: holds the state of this particular completion
19 : : *
20 : : * This will wake up a single thread waiting on this completion. Threads will be
21 : : * awakened in the same order in which they were queued.
22 : : *
23 : : * See also complete_all(), wait_for_completion() and related routines.
24 : : *
25 : : * If this function wakes up a task, it executes a full memory barrier before
26 : : * accessing the task state.
27 : : */
28 : 821942 : void complete(struct completion *x)
29 : : {
30 : : unsigned long flags;
31 : :
32 : 821942 : spin_lock_irqsave(&x->wait.lock, flags);
33 : :
34 [ + + ]: 821948 : if (x->done != UINT_MAX)
35 : 821954 : x->done++;
36 : 821948 : __wake_up_locked(&x->wait, TASK_NORMAL, 1);
37 : : spin_unlock_irqrestore(&x->wait.lock, flags);
38 : 821974 : }
39 : : EXPORT_SYMBOL(complete);
40 : :
41 : : /**
42 : : * complete_all: - signals all threads waiting on this completion
43 : : * @x: holds the state of this particular completion
44 : : *
45 : : * This will wake up all threads waiting on this particular completion event.
46 : : *
47 : : * If this function wakes up a task, it executes a full memory barrier before
48 : : * accessing the task state.
49 : : *
50 : : * Since complete_all() sets the completion of @x permanently to done
51 : : * to allow multiple waiters to finish, a call to reinit_completion()
52 : : * must be used on @x if @x is to be used again. The code must make
53 : : * sure that all waiters have woken and finished before reinitializing
54 : : * @x. Also note that the function completion_done() can not be used
55 : : * to know if there are still waiters after complete_all() has been called.
56 : : */
57 : 18988 : void complete_all(struct completion *x)
58 : : {
59 : : unsigned long flags;
60 : :
61 : 18988 : spin_lock_irqsave(&x->wait.lock, flags);
62 : 18988 : x->done = UINT_MAX;
63 : 18988 : __wake_up_locked(&x->wait, TASK_NORMAL, 0);
64 : : spin_unlock_irqrestore(&x->wait.lock, flags);
65 : 18988 : }
66 : : EXPORT_SYMBOL(complete_all);
67 : :
68 : : static inline long __sched
69 : 552816 : do_wait_for_common(struct completion *x,
70 : : long (*action)(long), long timeout, int state)
71 : : {
72 [ + + ]: 552816 : if (!x->done) {
73 : 745988 : DECLARE_WAITQUEUE(wait, current);
74 : :
75 : : __add_wait_queue_entry_tail_exclusive(&x->wait, &wait);
76 : : do {
77 [ + + ]: 745988 : if (signal_pending_state(state, current)) {
78 : : timeout = -ERESTARTSYS;
79 : : break;
80 : : }
81 : 372992 : __set_current_state(state);
82 : : spin_unlock_irq(&x->wait.lock);
83 : 372986 : timeout = action(timeout);
84 : : spin_lock_irq(&x->wait.lock);
85 [ - + # # ]: 372992 : } while (!x->done && timeout);
86 : : __remove_wait_queue(&x->wait, &wait);
87 [ - + ]: 372994 : if (!x->done)
88 : 0 : return timeout;
89 : : }
90 [ + + ]: 552816 : if (x->done != UINT_MAX)
91 : 544334 : x->done--;
92 [ + - ]: 552816 : return timeout ?: 1;
93 : : }
94 : :
95 : : static inline long __sched
96 : 552820 : __wait_for_common(struct completion *x,
97 : : long (*action)(long), long timeout, int state)
98 : : {
99 : 552820 : might_sleep();
100 : :
101 : : complete_acquire(x);
102 : :
103 : : spin_lock_irq(&x->wait.lock);
104 : 552814 : timeout = do_wait_for_common(x, action, timeout, state);
105 : : spin_unlock_irq(&x->wait.lock);
106 : :
107 : : complete_release(x);
108 : :
109 : 552808 : return timeout;
110 : : }
111 : :
112 : : static long __sched
113 : : wait_for_common(struct completion *x, long timeout, int state)
114 : : {
115 : 551184 : return __wait_for_common(x, schedule_timeout, timeout, state);
116 : : }
117 : :
118 : : static long __sched
119 : : wait_for_common_io(struct completion *x, long timeout, int state)
120 : : {
121 : 1636 : return __wait_for_common(x, io_schedule_timeout, timeout, state);
122 : : }
123 : :
124 : : /**
125 : : * wait_for_completion: - waits for completion of a task
126 : : * @x: holds the state of this particular completion
127 : : *
128 : : * This waits to be signaled for completion of a specific task. It is NOT
129 : : * interruptible and there is no timeout.
130 : : *
131 : : * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
132 : : * and interrupt capability. Also see complete().
133 : : */
134 : 378186 : void __sched wait_for_completion(struct completion *x)
135 : : {
136 : : wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
137 : 378182 : }
138 : : EXPORT_SYMBOL(wait_for_completion);
139 : :
140 : : /**
141 : : * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
142 : : * @x: holds the state of this particular completion
143 : : * @timeout: timeout value in jiffies
144 : : *
145 : : * This waits for either a completion of a specific task to be signaled or for a
146 : : * specified timeout to expire. The timeout is in jiffies. It is not
147 : : * interruptible.
148 : : *
149 : : * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
150 : : * till timeout) if completed.
151 : : */
152 : : unsigned long __sched
153 : 119062 : wait_for_completion_timeout(struct completion *x, unsigned long timeout)
154 : : {
155 : 238124 : return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
156 : : }
157 : : EXPORT_SYMBOL(wait_for_completion_timeout);
158 : :
159 : : /**
160 : : * wait_for_completion_io: - waits for completion of a task
161 : : * @x: holds the state of this particular completion
162 : : *
163 : : * This waits to be signaled for completion of a specific task. It is NOT
164 : : * interruptible and there is no timeout. The caller is accounted as waiting
165 : : * for IO (which traditionally means blkio only).
166 : : */
167 : 1636 : void __sched wait_for_completion_io(struct completion *x)
168 : : {
169 : : wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
170 : 1636 : }
171 : : EXPORT_SYMBOL(wait_for_completion_io);
172 : :
173 : : /**
174 : : * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
175 : : * @x: holds the state of this particular completion
176 : : * @timeout: timeout value in jiffies
177 : : *
178 : : * This waits for either a completion of a specific task to be signaled or for a
179 : : * specified timeout to expire. The timeout is in jiffies. It is not
180 : : * interruptible. The caller is accounted as waiting for IO (which traditionally
181 : : * means blkio only).
182 : : *
183 : : * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
184 : : * till timeout) if completed.
185 : : */
186 : : unsigned long __sched
187 : 0 : wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
188 : : {
189 : 0 : return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
190 : : }
191 : : EXPORT_SYMBOL(wait_for_completion_io_timeout);
192 : :
193 : : /**
194 : : * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
195 : : * @x: holds the state of this particular completion
196 : : *
197 : : * This waits for completion of a specific task to be signaled. It is
198 : : * interruptible.
199 : : *
200 : : * Return: -ERESTARTSYS if interrupted, 0 if completed.
201 : : */
202 : 0 : int __sched wait_for_completion_interruptible(struct completion *x)
203 : : {
204 : : long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
205 [ # # ]: 0 : if (t == -ERESTARTSYS)
206 : 0 : return t;
207 : : return 0;
208 : : }
209 : : EXPORT_SYMBOL(wait_for_completion_interruptible);
210 : :
211 : : /**
212 : : * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
213 : : * @x: holds the state of this particular completion
214 : : * @timeout: timeout value in jiffies
215 : : *
216 : : * This waits for either a completion of a specific task to be signaled or for a
217 : : * specified timeout to expire. It is interruptible. The timeout is in jiffies.
218 : : *
219 : : * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
220 : : * or number of jiffies left till timeout) if completed.
221 : : */
222 : : long __sched
223 : 0 : wait_for_completion_interruptible_timeout(struct completion *x,
224 : : unsigned long timeout)
225 : : {
226 : 0 : return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
227 : : }
228 : : EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
229 : :
230 : : /**
231 : : * wait_for_completion_killable: - waits for completion of a task (killable)
232 : : * @x: holds the state of this particular completion
233 : : *
234 : : * This waits to be signaled for completion of a specific task. It can be
235 : : * interrupted by a kill signal.
236 : : *
237 : : * Return: -ERESTARTSYS if interrupted, 0 if completed.
238 : : */
239 : 53532 : int __sched wait_for_completion_killable(struct completion *x)
240 : : {
241 : : long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
242 [ - + ]: 53526 : if (t == -ERESTARTSYS)
243 : 0 : return t;
244 : : return 0;
245 : : }
246 : : EXPORT_SYMBOL(wait_for_completion_killable);
247 : :
248 : : /**
249 : : * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
250 : : * @x: holds the state of this particular completion
251 : : * @timeout: timeout value in jiffies
252 : : *
253 : : * This waits for either a completion of a specific task to be
254 : : * signaled or for a specified timeout to expire. It can be
255 : : * interrupted by a kill signal. The timeout is in jiffies.
256 : : *
257 : : * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
258 : : * or number of jiffies left till timeout) if completed.
259 : : */
260 : : long __sched
261 : 404 : wait_for_completion_killable_timeout(struct completion *x,
262 : : unsigned long timeout)
263 : : {
264 : 808 : return wait_for_common(x, timeout, TASK_KILLABLE);
265 : : }
266 : : EXPORT_SYMBOL(wait_for_completion_killable_timeout);
267 : :
268 : : /**
269 : : * try_wait_for_completion - try to decrement a completion without blocking
270 : : * @x: completion structure
271 : : *
272 : : * Return: 0 if a decrement cannot be done without blocking
273 : : * 1 if a decrement succeeded.
274 : : *
275 : : * If a completion is being used as a counting completion,
276 : : * attempt to decrement the counter without blocking. This
277 : : * enables us to avoid waiting if the resource the completion
278 : : * is protecting is not available.
279 : : */
280 : 0 : bool try_wait_for_completion(struct completion *x)
281 : : {
282 : : unsigned long flags;
283 : : bool ret = true;
284 : :
285 : : /*
286 : : * Since x->done will need to be locked only
287 : : * in the non-blocking case, we check x->done
288 : : * first without taking the lock so we can
289 : : * return early in the blocking case.
290 : : */
291 [ # # ]: 0 : if (!READ_ONCE(x->done))
292 : : return false;
293 : :
294 : 0 : spin_lock_irqsave(&x->wait.lock, flags);
295 [ # # ]: 0 : if (!x->done)
296 : : ret = false;
297 [ # # ]: 0 : else if (x->done != UINT_MAX)
298 : 0 : x->done--;
299 : : spin_unlock_irqrestore(&x->wait.lock, flags);
300 : 0 : return ret;
301 : : }
302 : : EXPORT_SYMBOL(try_wait_for_completion);
303 : :
304 : : /**
305 : : * completion_done - Test to see if a completion has any waiters
306 : : * @x: completion structure
307 : : *
308 : : * Return: 0 if there are waiters (wait_for_completion() in progress)
309 : : * 1 if there are no waiters.
310 : : *
311 : : * Note, this will always return true if complete_all() was called on @X.
312 : : */
313 : 0 : bool completion_done(struct completion *x)
314 : : {
315 : : unsigned long flags;
316 : :
317 [ # # ]: 0 : if (!READ_ONCE(x->done))
318 : : return false;
319 : :
320 : : /*
321 : : * If ->done, we need to wait for complete() to release ->wait.lock
322 : : * otherwise we can end up freeing the completion before complete()
323 : : * is done referencing it.
324 : : */
325 : 0 : spin_lock_irqsave(&x->wait.lock, flags);
326 : : spin_unlock_irqrestore(&x->wait.lock, flags);
327 : 0 : return true;
328 : : }
329 : : EXPORT_SYMBOL(completion_done);
|
