Branch data Line data Source code
1 : : // SPDX-License-Identifier: GPL-2.0
2 : : /*
3 : : * Variant of atomic_t specialized for reference counts.
4 : : *
5 : : * The interface matches the atomic_t interface (to aid in porting) but only
6 : : * provides the few functions one should use for reference counting.
7 : : *
8 : : * It differs in that the counter saturates at UINT_MAX and will not move once
9 : : * there. This avoids wrapping the counter and causing 'spurious'
10 : : * use-after-free issues.
11 : : *
12 : : * Memory ordering rules are slightly relaxed wrt regular atomic_t functions
13 : : * and provide only what is strictly required for refcounts.
14 : : *
15 : : * The increments are fully relaxed; these will not provide ordering. The
16 : : * rationale is that whatever is used to obtain the object we're increasing the
17 : : * reference count on will provide the ordering. For locked data structures,
18 : : * its the lock acquire, for RCU/lockless data structures its the dependent
19 : : * load.
20 : : *
21 : : * Do note that inc_not_zero() provides a control dependency which will order
22 : : * future stores against the inc, this ensures we'll never modify the object
23 : : * if we did not in fact acquire a reference.
24 : : *
25 : : * The decrements will provide release order, such that all the prior loads and
26 : : * stores will be issued before, it also provides a control dependency, which
27 : : * will order us against the subsequent free().
28 : : *
29 : : * The control dependency is against the load of the cmpxchg (ll/sc) that
30 : : * succeeded. This means the stores aren't fully ordered, but this is fine
31 : : * because the 1->0 transition indicates no concurrency.
32 : : *
33 : : * Note that the allocator is responsible for ordering things between free()
34 : : * and alloc().
35 : : *
36 : : * The decrements dec_and_test() and sub_and_test() also provide acquire
37 : : * ordering on success.
38 : : *
39 : : */
40 : :
41 : : #include <linux/mutex.h>
42 : : #include <linux/refcount.h>
43 : : #include <linux/spinlock.h>
44 : : #include <linux/bug.h>
45 : :
46 : : /**
47 : : * refcount_add_not_zero_checked - add a value to a refcount unless it is 0
48 : : * @i: the value to add to the refcount
49 : : * @r: the refcount
50 : : *
51 : : * Will saturate at UINT_MAX and WARN.
52 : : *
53 : : * Provides no memory ordering, it is assumed the caller has guaranteed the
54 : : * object memory to be stable (RCU, etc.). It does provide a control dependency
55 : : * and thereby orders future stores. See the comment on top.
56 : : *
57 : : * Use of this function is not recommended for the normal reference counting
58 : : * use case in which references are taken and released one at a time. In these
59 : : * cases, refcount_inc(), or one of its variants, should instead be used to
60 : : * increment a reference count.
61 : : *
62 : : * Return: false if the passed refcount is 0, true otherwise
63 : : */
64 : 1377980 : bool refcount_add_not_zero_checked(unsigned int i, refcount_t *r)
65 : : {
66 : 1377980 : unsigned int new, val = atomic_read(&r->refs);
67 : :
68 : : do {
69 [ + - ]: 1381409 : if (!val)
70 : : return false;
71 : :
72 [ + + ]: 1384649 : if (unlikely(val == UINT_MAX))
73 : : return true;
74 : :
75 : 1382693 : new = val + i;
76 [ - + ]: 1382693 : if (new < val)
77 : : new = UINT_MAX;
78 : :
79 [ + + ]: 1382693 : } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new));
80 : :
81 [ - + # # ]: 1381242 : WARN_ONCE(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n");
82 : :
83 : : return true;
84 : : }
85 : : EXPORT_SYMBOL(refcount_add_not_zero_checked);
86 : :
87 : : /**
88 : : * refcount_add_checked - add a value to a refcount
89 : : * @i: the value to add to the refcount
90 : : * @r: the refcount
91 : : *
92 : : * Similar to atomic_add(), but will saturate at UINT_MAX and WARN.
93 : : *
94 : : * Provides no memory ordering, it is assumed the caller has guaranteed the
95 : : * object memory to be stable (RCU, etc.). It does provide a control dependency
96 : : * and thereby orders future stores. See the comment on top.
97 : : *
98 : : * Use of this function is not recommended for the normal reference counting
99 : : * use case in which references are taken and released one at a time. In these
100 : : * cases, refcount_inc(), or one of its variants, should instead be used to
101 : : * increment a reference count.
102 : : */
103 : 1381933 : void refcount_add_checked(unsigned int i, refcount_t *r)
104 : : {
105 [ - + # # ]: 1381933 : WARN_ONCE(!refcount_add_not_zero_checked(i, r), "refcount_t: addition on 0; use-after-free.\n");
106 : 1382457 : }
107 : : EXPORT_SYMBOL(refcount_add_checked);
108 : :
109 : : /**
110 : : * refcount_inc_not_zero_checked - increment a refcount unless it is 0
111 : : * @r: the refcount to increment
112 : : *
113 : : * Similar to atomic_inc_not_zero(), but will saturate at UINT_MAX and WARN.
114 : : *
115 : : * Provides no memory ordering, it is assumed the caller has guaranteed the
116 : : * object memory to be stable (RCU, etc.). It does provide a control dependency
117 : : * and thereby orders future stores. See the comment on top.
118 : : *
119 : : * Return: true if the increment was successful, false otherwise
120 : : */
121 : 30536742 : bool refcount_inc_not_zero_checked(refcount_t *r)
122 : : {
123 : 30536742 : unsigned int new, val = atomic_read(&r->refs);
124 : :
125 : : do {
126 : 30550722 : new = val + 1;
127 : :
128 [ + - ]: 30550722 : if (!val)
129 : : return false;
130 : :
131 [ + + ]: 30570717 : if (unlikely(!new))
132 : : return true;
133 : :
134 [ + + ]: 30560050 : } while (!atomic_try_cmpxchg_relaxed(&r->refs, &val, new));
135 : :
136 [ - + # # ]: 30547925 : WARN_ONCE(new == UINT_MAX, "refcount_t: saturated; leaking memory.\n");
137 : :
138 : : return true;
139 : : }
140 : : EXPORT_SYMBOL(refcount_inc_not_zero_checked);
141 : :
142 : : /**
143 : : * refcount_inc_checked - increment a refcount
144 : : * @r: the refcount to increment
145 : : *
146 : : * Similar to atomic_inc(), but will saturate at UINT_MAX and WARN.
147 : : *
148 : : * Provides no memory ordering, it is assumed the caller already has a
149 : : * reference on the object.
150 : : *
151 : : * Will WARN if the refcount is 0, as this represents a possible use-after-free
152 : : * condition.
153 : : */
154 : 29138648 : void refcount_inc_checked(refcount_t *r)
155 : : {
156 [ - + # # ]: 29138648 : WARN_ONCE(!refcount_inc_not_zero_checked(r), "refcount_t: increment on 0; use-after-free.\n");
157 : 29137032 : }
158 : : EXPORT_SYMBOL(refcount_inc_checked);
159 : :
160 : : /**
161 : : * refcount_sub_and_test_checked - subtract from a refcount and test if it is 0
162 : : * @i: amount to subtract from the refcount
163 : : * @r: the refcount
164 : : *
165 : : * Similar to atomic_dec_and_test(), but it will WARN, return false and
166 : : * ultimately leak on underflow and will fail to decrement when saturated
167 : : * at UINT_MAX.
168 : : *
169 : : * Provides release memory ordering, such that prior loads and stores are done
170 : : * before, and provides an acquire ordering on success such that free()
171 : : * must come after.
172 : : *
173 : : * Use of this function is not recommended for the normal reference counting
174 : : * use case in which references are taken and released one at a time. In these
175 : : * cases, refcount_dec(), or one of its variants, should instead be used to
176 : : * decrement a reference count.
177 : : *
178 : : * Return: true if the resulting refcount is 0, false otherwise
179 : : */
180 : 32625084 : bool refcount_sub_and_test_checked(unsigned int i, refcount_t *r)
181 : : {
182 : 32625084 : unsigned int new, val = atomic_read(&r->refs);
183 : :
184 : : do {
185 [ + - ]: 32682444 : if (unlikely(val == UINT_MAX))
186 : : return false;
187 : :
188 : 32746250 : new = val - i;
189 [ - + ]: 32746250 : if (new > val) {
190 [ # # # # ]: 0 : WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n");
191 : : return false;
192 : : }
193 : :
194 [ + + ]: 32746250 : } while (!atomic_try_cmpxchg_release(&r->refs, &val, new));
195 : :
196 [ + + ]: 32702977 : if (!new) {
197 : 6117124 : smp_acquire__after_ctrl_dep();
198 : 6116885 : return true;
199 : : }
200 : : return false;
201 : :
202 : : }
203 : : EXPORT_SYMBOL(refcount_sub_and_test_checked);
204 : :
205 : : /**
206 : : * refcount_dec_and_test_checked - decrement a refcount and test if it is 0
207 : : * @r: the refcount
208 : : *
209 : : * Similar to atomic_dec_and_test(), it will WARN on underflow and fail to
210 : : * decrement when saturated at UINT_MAX.
211 : : *
212 : : * Provides release memory ordering, such that prior loads and stores are done
213 : : * before, and provides an acquire ordering on success such that free()
214 : : * must come after.
215 : : *
216 : : * Return: true if the resulting refcount is 0, false otherwise
217 : : */
218 : 29738567 : bool refcount_dec_and_test_checked(refcount_t *r)
219 : : {
220 : 29924773 : return refcount_sub_and_test_checked(1, r);
221 : : }
222 : : EXPORT_SYMBOL(refcount_dec_and_test_checked);
223 : :
224 : : /**
225 : : * refcount_dec_checked - decrement a refcount
226 : : * @r: the refcount
227 : : *
228 : : * Similar to atomic_dec(), it will WARN on underflow and fail to decrement
229 : : * when saturated at UINT_MAX.
230 : : *
231 : : * Provides release memory ordering, such that prior loads and stores are done
232 : : * before.
233 : : */
234 : 175596 : void refcount_dec_checked(refcount_t *r)
235 : : {
236 [ - + # # ]: 175598 : WARN_ONCE(refcount_dec_and_test_checked(r), "refcount_t: decrement hit 0; leaking memory.\n");
237 : 175598 : }
238 : : EXPORT_SYMBOL(refcount_dec_checked);
239 : :
240 : : /**
241 : : * refcount_dec_if_one - decrement a refcount if it is 1
242 : : * @r: the refcount
243 : : *
244 : : * No atomic_t counterpart, it attempts a 1 -> 0 transition and returns the
245 : : * success thereof.
246 : : *
247 : : * Like all decrement operations, it provides release memory order and provides
248 : : * a control dependency.
249 : : *
250 : : * It can be used like a try-delete operator; this explicit case is provided
251 : : * and not cmpxchg in generic, because that would allow implementing unsafe
252 : : * operations.
253 : : *
254 : : * Return: true if the resulting refcount is 0, false otherwise
255 : : */
256 : 0 : bool refcount_dec_if_one(refcount_t *r)
257 : : {
258 : 0 : int val = 1;
259 : :
260 : 0 : return atomic_try_cmpxchg_release(&r->refs, &val, 0);
261 : : }
262 : : EXPORT_SYMBOL(refcount_dec_if_one);
263 : :
264 : : /**
265 : : * refcount_dec_not_one - decrement a refcount if it is not 1
266 : : * @r: the refcount
267 : : *
268 : : * No atomic_t counterpart, it decrements unless the value is 1, in which case
269 : : * it will return false.
270 : : *
271 : : * Was often done like: atomic_add_unless(&var, -1, 1)
272 : : *
273 : : * Return: true if the decrement operation was successful, false otherwise
274 : : */
275 : 6017515 : bool refcount_dec_not_one(refcount_t *r)
276 : : {
277 : 6017515 : unsigned int new, val = atomic_read(&r->refs);
278 : :
279 : : do {
280 [ + - ]: 6065750 : if (unlikely(val == UINT_MAX))
281 : : return true;
282 : :
283 [ + + ]: 6082315 : if (val == 1)
284 : : return false;
285 : :
286 : 6038826 : new = val - 1;
287 [ - + ]: 6038826 : if (new > val) {
288 [ # # # # ]: 0 : WARN_ONCE(new > val, "refcount_t: underflow; use-after-free.\n");
289 : : return true;
290 : : }
291 : :
292 [ + + ]: 6038826 : } while (!atomic_try_cmpxchg_release(&r->refs, &val, new));
293 : :
294 : : return true;
295 : : }
296 : : EXPORT_SYMBOL(refcount_dec_not_one);
297 : :
298 : : /**
299 : : * refcount_dec_and_mutex_lock - return holding mutex if able to decrement
300 : : * refcount to 0
301 : : * @r: the refcount
302 : : * @lock: the mutex to be locked
303 : : *
304 : : * Similar to atomic_dec_and_mutex_lock(), it will WARN on underflow and fail
305 : : * to decrement when saturated at UINT_MAX.
306 : : *
307 : : * Provides release memory ordering, such that prior loads and stores are done
308 : : * before, and provides a control dependency such that free() must come after.
309 : : * See the comment on top.
310 : : *
311 : : * Return: true and hold mutex if able to decrement refcount to 0, false
312 : : * otherwise
313 : : */
314 : 71826 : bool refcount_dec_and_mutex_lock(refcount_t *r, struct mutex *lock)
315 : : {
316 [ - + ]: 71826 : if (refcount_dec_not_one(r))
317 : : return false;
318 : :
319 : 0 : mutex_lock(lock);
320 [ # # ]: 0 : if (!refcount_dec_and_test(r)) {
321 : 0 : mutex_unlock(lock);
322 : 0 : return false;
323 : : }
324 : :
325 : : return true;
326 : : }
327 : : EXPORT_SYMBOL(refcount_dec_and_mutex_lock);
328 : :
329 : : /**
330 : : * refcount_dec_and_lock - return holding spinlock if able to decrement
331 : : * refcount to 0
332 : : * @r: the refcount
333 : : * @lock: the spinlock to be locked
334 : : *
335 : : * Similar to atomic_dec_and_lock(), it will WARN on underflow and fail to
336 : : * decrement when saturated at UINT_MAX.
337 : : *
338 : : * Provides release memory ordering, such that prior loads and stores are done
339 : : * before, and provides a control dependency such that free() must come after.
340 : : * See the comment on top.
341 : : *
342 : : * Return: true and hold spinlock if able to decrement refcount to 0, false
343 : : * otherwise
344 : : */
345 : 98669 : bool refcount_dec_and_lock(refcount_t *r, spinlock_t *lock)
346 : : {
347 [ + + ]: 98669 : if (refcount_dec_not_one(r))
348 : : return false;
349 : :
350 : : spin_lock(lock);
351 [ - + ]: 10171 : if (!refcount_dec_and_test(r)) {
352 : : spin_unlock(lock);
353 : 0 : return false;
354 : : }
355 : :
356 : : return true;
357 : : }
358 : : EXPORT_SYMBOL(refcount_dec_and_lock);
359 : :
360 : : /**
361 : : * refcount_dec_and_lock_irqsave - return holding spinlock with disabled
362 : : * interrupts if able to decrement refcount to 0
363 : : * @r: the refcount
364 : : * @lock: the spinlock to be locked
365 : : * @flags: saved IRQ-flags if the is acquired
366 : : *
367 : : * Same as refcount_dec_and_lock() above except that the spinlock is acquired
368 : : * with disabled interupts.
369 : : *
370 : : * Return: true and hold spinlock if able to decrement refcount to 0, false
371 : : * otherwise
372 : : */
373 : 4873684 : bool refcount_dec_and_lock_irqsave(refcount_t *r, spinlock_t *lock,
374 : : unsigned long *flags)
375 : : {
376 [ + + ]: 4873684 : if (refcount_dec_not_one(r))
377 : : return false;
378 : :
379 : 439 : spin_lock_irqsave(lock, *flags);
380 [ - + ]: 439 : if (!refcount_dec_and_test(r)) {
381 : 0 : spin_unlock_irqrestore(lock, *flags);
382 : 0 : return false;
383 : : }
384 : :
385 : : return true;
386 : : }
387 : : EXPORT_SYMBOL(refcount_dec_and_lock_irqsave);
|
