Branch data Line data Source code
1 : : // SPDX-License-Identifier: GPL-2.0
2 : : /*
3 : : * Kernel internal timers
4 : : *
5 : : * Copyright (C) 1991, 1992 Linus Torvalds
6 : : *
7 : : * 1997-01-28 Modified by Finn Arne Gangstad to make timers scale better.
8 : : *
9 : : * 1997-09-10 Updated NTP code according to technical memorandum Jan '96
10 : : * "A Kernel Model for Precision Timekeeping" by Dave Mills
11 : : * 1998-12-24 Fixed a xtime SMP race (we need the xtime_lock rw spinlock to
12 : : * serialize accesses to xtime/lost_ticks).
13 : : * Copyright (C) 1998 Andrea Arcangeli
14 : : * 1999-03-10 Improved NTP compatibility by Ulrich Windl
15 : : * 2002-05-31 Move sys_sysinfo here and make its locking sane, Robert Love
16 : : * 2000-10-05 Implemented scalable SMP per-CPU timer handling.
17 : : * Copyright (C) 2000, 2001, 2002 Ingo Molnar
18 : : * Designed by David S. Miller, Alexey Kuznetsov and Ingo Molnar
19 : : */
20 : :
21 : : #include <linux/kernel_stat.h>
22 : : #include <linux/export.h>
23 : : #include <linux/interrupt.h>
24 : : #include <linux/percpu.h>
25 : : #include <linux/init.h>
26 : : #include <linux/mm.h>
27 : : #include <linux/swap.h>
28 : : #include <linux/pid_namespace.h>
29 : : #include <linux/notifier.h>
30 : : #include <linux/thread_info.h>
31 : : #include <linux/time.h>
32 : : #include <linux/jiffies.h>
33 : : #include <linux/posix-timers.h>
34 : : #include <linux/cpu.h>
35 : : #include <linux/syscalls.h>
36 : : #include <linux/delay.h>
37 : : #include <linux/tick.h>
38 : : #include <linux/kallsyms.h>
39 : : #include <linux/irq_work.h>
40 : : #include <linux/sched/signal.h>
41 : : #include <linux/sched/sysctl.h>
42 : : #include <linux/sched/nohz.h>
43 : : #include <linux/sched/debug.h>
44 : : #include <linux/slab.h>
45 : : #include <linux/compat.h>
46 : :
47 : : #include <linux/uaccess.h>
48 : : #include <asm/unistd.h>
49 : : #include <asm/div64.h>
50 : : #include <asm/timex.h>
51 : : #include <asm/io.h>
52 : :
53 : : #include "tick-internal.h"
54 : :
55 : : #define CREATE_TRACE_POINTS
56 : : #include <trace/events/timer.h>
57 : :
58 : : __visible u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES;
59 : :
60 : : EXPORT_SYMBOL(jiffies_64);
61 : :
62 : : /*
63 : : * The timer wheel has LVL_DEPTH array levels. Each level provides an array of
64 : : * LVL_SIZE buckets. Each level is driven by its own clock and therefor each
65 : : * level has a different granularity.
66 : : *
67 : : * The level granularity is: LVL_CLK_DIV ^ lvl
68 : : * The level clock frequency is: HZ / (LVL_CLK_DIV ^ level)
69 : : *
70 : : * The array level of a newly armed timer depends on the relative expiry
71 : : * time. The farther the expiry time is away the higher the array level and
72 : : * therefor the granularity becomes.
73 : : *
74 : : * Contrary to the original timer wheel implementation, which aims for 'exact'
75 : : * expiry of the timers, this implementation removes the need for recascading
76 : : * the timers into the lower array levels. The previous 'classic' timer wheel
77 : : * implementation of the kernel already violated the 'exact' expiry by adding
78 : : * slack to the expiry time to provide batched expiration. The granularity
79 : : * levels provide implicit batching.
80 : : *
81 : : * This is an optimization of the original timer wheel implementation for the
82 : : * majority of the timer wheel use cases: timeouts. The vast majority of
83 : : * timeout timers (networking, disk I/O ...) are canceled before expiry. If
84 : : * the timeout expires it indicates that normal operation is disturbed, so it
85 : : * does not matter much whether the timeout comes with a slight delay.
86 : : *
87 : : * The only exception to this are networking timers with a small expiry
88 : : * time. They rely on the granularity. Those fit into the first wheel level,
89 : : * which has HZ granularity.
90 : : *
91 : : * We don't have cascading anymore. timers with a expiry time above the
92 : : * capacity of the last wheel level are force expired at the maximum timeout
93 : : * value of the last wheel level. From data sampling we know that the maximum
94 : : * value observed is 5 days (network connection tracking), so this should not
95 : : * be an issue.
96 : : *
97 : : * The currently chosen array constants values are a good compromise between
98 : : * array size and granularity.
99 : : *
100 : : * This results in the following granularity and range levels:
101 : : *
102 : : * HZ 1000 steps
103 : : * Level Offset Granularity Range
104 : : * 0 0 1 ms 0 ms - 63 ms
105 : : * 1 64 8 ms 64 ms - 511 ms
106 : : * 2 128 64 ms 512 ms - 4095 ms (512ms - ~4s)
107 : : * 3 192 512 ms 4096 ms - 32767 ms (~4s - ~32s)
108 : : * 4 256 4096 ms (~4s) 32768 ms - 262143 ms (~32s - ~4m)
109 : : * 5 320 32768 ms (~32s) 262144 ms - 2097151 ms (~4m - ~34m)
110 : : * 6 384 262144 ms (~4m) 2097152 ms - 16777215 ms (~34m - ~4h)
111 : : * 7 448 2097152 ms (~34m) 16777216 ms - 134217727 ms (~4h - ~1d)
112 : : * 8 512 16777216 ms (~4h) 134217728 ms - 1073741822 ms (~1d - ~12d)
113 : : *
114 : : * HZ 300
115 : : * Level Offset Granularity Range
116 : : * 0 0 3 ms 0 ms - 210 ms
117 : : * 1 64 26 ms 213 ms - 1703 ms (213ms - ~1s)
118 : : * 2 128 213 ms 1706 ms - 13650 ms (~1s - ~13s)
119 : : * 3 192 1706 ms (~1s) 13653 ms - 109223 ms (~13s - ~1m)
120 : : * 4 256 13653 ms (~13s) 109226 ms - 873810 ms (~1m - ~14m)
121 : : * 5 320 109226 ms (~1m) 873813 ms - 6990503 ms (~14m - ~1h)
122 : : * 6 384 873813 ms (~14m) 6990506 ms - 55924050 ms (~1h - ~15h)
123 : : * 7 448 6990506 ms (~1h) 55924053 ms - 447392423 ms (~15h - ~5d)
124 : : * 8 512 55924053 ms (~15h) 447392426 ms - 3579139406 ms (~5d - ~41d)
125 : : *
126 : : * HZ 250
127 : : * Level Offset Granularity Range
128 : : * 0 0 4 ms 0 ms - 255 ms
129 : : * 1 64 32 ms 256 ms - 2047 ms (256ms - ~2s)
130 : : * 2 128 256 ms 2048 ms - 16383 ms (~2s - ~16s)
131 : : * 3 192 2048 ms (~2s) 16384 ms - 131071 ms (~16s - ~2m)
132 : : * 4 256 16384 ms (~16s) 131072 ms - 1048575 ms (~2m - ~17m)
133 : : * 5 320 131072 ms (~2m) 1048576 ms - 8388607 ms (~17m - ~2h)
134 : : * 6 384 1048576 ms (~17m) 8388608 ms - 67108863 ms (~2h - ~18h)
135 : : * 7 448 8388608 ms (~2h) 67108864 ms - 536870911 ms (~18h - ~6d)
136 : : * 8 512 67108864 ms (~18h) 536870912 ms - 4294967288 ms (~6d - ~49d)
137 : : *
138 : : * HZ 100
139 : : * Level Offset Granularity Range
140 : : * 0 0 10 ms 0 ms - 630 ms
141 : : * 1 64 80 ms 640 ms - 5110 ms (640ms - ~5s)
142 : : * 2 128 640 ms 5120 ms - 40950 ms (~5s - ~40s)
143 : : * 3 192 5120 ms (~5s) 40960 ms - 327670 ms (~40s - ~5m)
144 : : * 4 256 40960 ms (~40s) 327680 ms - 2621430 ms (~5m - ~43m)
145 : : * 5 320 327680 ms (~5m) 2621440 ms - 20971510 ms (~43m - ~5h)
146 : : * 6 384 2621440 ms (~43m) 20971520 ms - 167772150 ms (~5h - ~1d)
147 : : * 7 448 20971520 ms (~5h) 167772160 ms - 1342177270 ms (~1d - ~15d)
148 : : */
149 : :
150 : : /* Clock divisor for the next level */
151 : : #define LVL_CLK_SHIFT 3
152 : : #define LVL_CLK_DIV (1UL << LVL_CLK_SHIFT)
153 : : #define LVL_CLK_MASK (LVL_CLK_DIV - 1)
154 : : #define LVL_SHIFT(n) ((n) * LVL_CLK_SHIFT)
155 : : #define LVL_GRAN(n) (1UL << LVL_SHIFT(n))
156 : :
157 : : /*
158 : : * The time start value for each level to select the bucket at enqueue
159 : : * time.
160 : : */
161 : : #define LVL_START(n) ((LVL_SIZE - 1) << (((n) - 1) * LVL_CLK_SHIFT))
162 : :
163 : : /* Size of each clock level */
164 : : #define LVL_BITS 6
165 : : #define LVL_SIZE (1UL << LVL_BITS)
166 : : #define LVL_MASK (LVL_SIZE - 1)
167 : : #define LVL_OFFS(n) ((n) * LVL_SIZE)
168 : :
169 : : /* Level depth */
170 : : #if HZ > 100
171 : : # define LVL_DEPTH 9
172 : : # else
173 : : # define LVL_DEPTH 8
174 : : #endif
175 : :
176 : : /* The cutoff (max. capacity of the wheel) */
177 : : #define WHEEL_TIMEOUT_CUTOFF (LVL_START(LVL_DEPTH))
178 : : #define WHEEL_TIMEOUT_MAX (WHEEL_TIMEOUT_CUTOFF - LVL_GRAN(LVL_DEPTH - 1))
179 : :
180 : : /*
181 : : * The resulting wheel size. If NOHZ is configured we allocate two
182 : : * wheels so we have a separate storage for the deferrable timers.
183 : : */
184 : : #define WHEEL_SIZE (LVL_SIZE * LVL_DEPTH)
185 : :
186 : : #ifdef CONFIG_NO_HZ_COMMON
187 : : # define NR_BASES 2
188 : : # define BASE_STD 0
189 : : # define BASE_DEF 1
190 : : #else
191 : : # define NR_BASES 1
192 : : # define BASE_STD 0
193 : : # define BASE_DEF 0
194 : : #endif
195 : :
196 : : struct timer_base {
197 : : raw_spinlock_t lock;
198 : : struct timer_list *running_timer;
199 : : #ifdef CONFIG_PREEMPT_RT
200 : : spinlock_t expiry_lock;
201 : : atomic_t timer_waiters;
202 : : #endif
203 : : unsigned long clk;
204 : : unsigned long next_expiry;
205 : : unsigned int cpu;
206 : : bool is_idle;
207 : : bool must_forward_clk;
208 : : DECLARE_BITMAP(pending_map, WHEEL_SIZE);
209 : : struct hlist_head vectors[WHEEL_SIZE];
210 : : } ____cacheline_aligned;
211 : :
212 : : static DEFINE_PER_CPU(struct timer_base, timer_bases[NR_BASES]);
213 : :
214 : : #ifdef CONFIG_NO_HZ_COMMON
215 : :
216 : : static DEFINE_STATIC_KEY_FALSE(timers_nohz_active);
217 : : static DEFINE_MUTEX(timer_keys_mutex);
218 : :
219 : : static void timer_update_keys(struct work_struct *work);
220 : : static DECLARE_WORK(timer_update_work, timer_update_keys);
221 : :
222 : : #ifdef CONFIG_SMP
223 : : unsigned int sysctl_timer_migration = 1;
224 : :
225 : : DEFINE_STATIC_KEY_FALSE(timers_migration_enabled);
226 : :
227 : 11 : static void timers_update_migration(void)
228 : : {
229 [ + - + - ]: 11 : if (sysctl_timer_migration && tick_nohz_active)
230 : 11 : static_branch_enable(&timers_migration_enabled);
231 : : else
232 : 0 : static_branch_disable(&timers_migration_enabled);
233 : 11 : }
234 : : #else
235 : : static inline void timers_update_migration(void) { }
236 : : #endif /* !CONFIG_SMP */
237 : :
238 : 11 : static void timer_update_keys(struct work_struct *work)
239 : : {
240 : 11 : mutex_lock(&timer_keys_mutex);
241 : 11 : timers_update_migration();
242 : 11 : static_branch_enable(&timers_nohz_active);
243 : 11 : mutex_unlock(&timer_keys_mutex);
244 : 11 : }
245 : :
246 : 11 : void timers_update_nohz(void)
247 : : {
248 : 11 : schedule_work(&timer_update_work);
249 : 11 : }
250 : :
251 : 0 : int timer_migration_handler(struct ctl_table *table, int write,
252 : : void __user *buffer, size_t *lenp,
253 : : loff_t *ppos)
254 : : {
255 : 0 : int ret;
256 : :
257 : 0 : mutex_lock(&timer_keys_mutex);
258 : 0 : ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
259 [ # # ]: 0 : if (!ret && write)
260 : 0 : timers_update_migration();
261 : 0 : mutex_unlock(&timer_keys_mutex);
262 : 0 : return ret;
263 : : }
264 : :
265 : : static inline bool is_timers_nohz_active(void)
266 : : {
267 : : return static_branch_unlikely(&timers_nohz_active);
268 : : }
269 : : #else
270 : : static inline bool is_timers_nohz_active(void) { return false; }
271 : : #endif /* NO_HZ_COMMON */
272 : :
273 : 24696 : static unsigned long round_jiffies_common(unsigned long j, int cpu,
274 : : bool force_up)
275 : : {
276 : 24696 : int rem;
277 : 24696 : unsigned long original = j;
278 : :
279 : : /*
280 : : * We don't want all cpus firing their timers at once hitting the
281 : : * same lock or cachelines, so we skew each extra cpu with an extra
282 : : * 3 jiffies. This 3 jiffies came originally from the mm/ code which
283 : : * already did this.
284 : : * The skew is done by adding 3*cpunr, then round, then subtract this
285 : : * extra offset again.
286 : : */
287 : 24696 : j += cpu * 3;
288 : :
289 : 24696 : rem = j % HZ;
290 : :
291 : : /*
292 : : * If the target jiffie is just after a whole second (which can happen
293 : : * due to delays of the timer irq, long irq off times etc etc) then
294 : : * we should round down to the whole second, not up. Use 1/4th second
295 : : * as cutoff for this rounding as an extreme upper bound for this.
296 : : * But never round down if @force_up is set.
297 : : */
298 : 24696 : if (rem < HZ/4 && !force_up) /* round down */
299 : 36 : j = j - rem;
300 : : else /* round up */
301 : 43 : j = j - rem + HZ;
302 : :
303 : : /* now that we have rounded, subtract the extra skew again */
304 : 24696 : j -= cpu * 3;
305 : :
306 : : /*
307 : : * Make sure j is still in the future. Otherwise return the
308 : : * unmodified value.
309 : : */
310 [ - + - + : 79 : return time_is_after_jiffies(j) ? j : original;
- - - - ]
311 : : }
312 : :
313 : : /**
314 : : * __round_jiffies - function to round jiffies to a full second
315 : : * @j: the time in (absolute) jiffies that should be rounded
316 : : * @cpu: the processor number on which the timeout will happen
317 : : *
318 : : * __round_jiffies() rounds an absolute time in the future (in jiffies)
319 : : * up or down to (approximately) full seconds. This is useful for timers
320 : : * for which the exact time they fire does not matter too much, as long as
321 : : * they fire approximately every X seconds.
322 : : *
323 : : * By rounding these timers to whole seconds, all such timers will fire
324 : : * at the same time, rather than at various times spread out. The goal
325 : : * of this is to have the CPU wake up less, which saves power.
326 : : *
327 : : * The exact rounding is skewed for each processor to avoid all
328 : : * processors firing at the exact same time, which could lead
329 : : * to lock contention or spurious cache line bouncing.
330 : : *
331 : : * The return value is the rounded version of the @j parameter.
332 : : */
333 : 0 : unsigned long __round_jiffies(unsigned long j, int cpu)
334 : : {
335 [ # # ]: 0 : return round_jiffies_common(j, cpu, false);
336 : : }
337 : : EXPORT_SYMBOL_GPL(__round_jiffies);
338 : :
339 : : /**
340 : : * __round_jiffies_relative - function to round jiffies to a full second
341 : : * @j: the time in (relative) jiffies that should be rounded
342 : : * @cpu: the processor number on which the timeout will happen
343 : : *
344 : : * __round_jiffies_relative() rounds a time delta in the future (in jiffies)
345 : : * up or down to (approximately) full seconds. This is useful for timers
346 : : * for which the exact time they fire does not matter too much, as long as
347 : : * they fire approximately every X seconds.
348 : : *
349 : : * By rounding these timers to whole seconds, all such timers will fire
350 : : * at the same time, rather than at various times spread out. The goal
351 : : * of this is to have the CPU wake up less, which saves power.
352 : : *
353 : : * The exact rounding is skewed for each processor to avoid all
354 : : * processors firing at the exact same time, which could lead
355 : : * to lock contention or spurious cache line bouncing.
356 : : *
357 : : * The return value is the rounded version of the @j parameter.
358 : : */
359 : 68 : unsigned long __round_jiffies_relative(unsigned long j, int cpu)
360 : : {
361 : 68 : unsigned long j0 = jiffies;
362 : :
363 : : /* Use j0 because jiffies might change while we run */
364 [ # # ]: 0 : return round_jiffies_common(j + j0, cpu, false) - j0;
365 : : }
366 : : EXPORT_SYMBOL_GPL(__round_jiffies_relative);
367 : :
368 : : /**
369 : : * round_jiffies - function to round jiffies to a full second
370 : : * @j: the time in (absolute) jiffies that should be rounded
371 : : *
372 : : * round_jiffies() rounds an absolute time in the future (in jiffies)
373 : : * up or down to (approximately) full seconds. This is useful for timers
374 : : * for which the exact time they fire does not matter too much, as long as
375 : : * they fire approximately every X seconds.
376 : : *
377 : : * By rounding these timers to whole seconds, all such timers will fire
378 : : * at the same time, rather than at various times spread out. The goal
379 : : * of this is to have the CPU wake up less, which saves power.
380 : : *
381 : : * The return value is the rounded version of the @j parameter.
382 : : */
383 : 11 : unsigned long round_jiffies(unsigned long j)
384 : : {
385 [ + - ]: 11 : return round_jiffies_common(j, raw_smp_processor_id(), false);
386 : : }
387 : : EXPORT_SYMBOL_GPL(round_jiffies);
388 : :
389 : : /**
390 : : * round_jiffies_relative - function to round jiffies to a full second
391 : : * @j: the time in (relative) jiffies that should be rounded
392 : : *
393 : : * round_jiffies_relative() rounds a time delta in the future (in jiffies)
394 : : * up or down to (approximately) full seconds. This is useful for timers
395 : : * for which the exact time they fire does not matter too much, as long as
396 : : * they fire approximately every X seconds.
397 : : *
398 : : * By rounding these timers to whole seconds, all such timers will fire
399 : : * at the same time, rather than at various times spread out. The goal
400 : : * of this is to have the CPU wake up less, which saves power.
401 : : *
402 : : * The return value is the rounded version of the @j parameter.
403 : : */
404 : 68 : unsigned long round_jiffies_relative(unsigned long j)
405 : : {
406 [ + + ]: 68 : return __round_jiffies_relative(j, raw_smp_processor_id());
407 : : }
408 : : EXPORT_SYMBOL_GPL(round_jiffies_relative);
409 : :
410 : : /**
411 : : * __round_jiffies_up - function to round jiffies up to a full second
412 : : * @j: the time in (absolute) jiffies that should be rounded
413 : : * @cpu: the processor number on which the timeout will happen
414 : : *
415 : : * This is the same as __round_jiffies() except that it will never
416 : : * round down. This is useful for timeouts for which the exact time
417 : : * of firing does not matter too much, as long as they don't fire too
418 : : * early.
419 : : */
420 : 0 : unsigned long __round_jiffies_up(unsigned long j, int cpu)
421 : : {
422 [ # # ]: 0 : return round_jiffies_common(j, cpu, true);
423 : : }
424 : : EXPORT_SYMBOL_GPL(__round_jiffies_up);
425 : :
426 : : /**
427 : : * __round_jiffies_up_relative - function to round jiffies up to a full second
428 : : * @j: the time in (relative) jiffies that should be rounded
429 : : * @cpu: the processor number on which the timeout will happen
430 : : *
431 : : * This is the same as __round_jiffies_relative() except that it will never
432 : : * round down. This is useful for timeouts for which the exact time
433 : : * of firing does not matter too much, as long as they don't fire too
434 : : * early.
435 : : */
436 : 0 : unsigned long __round_jiffies_up_relative(unsigned long j, int cpu)
437 : : {
438 : 0 : unsigned long j0 = jiffies;
439 : :
440 : : /* Use j0 because jiffies might change while we run */
441 [ # # ]: 0 : return round_jiffies_common(j + j0, cpu, true) - j0;
442 : : }
443 : : EXPORT_SYMBOL_GPL(__round_jiffies_up_relative);
444 : :
445 : : /**
446 : : * round_jiffies_up - function to round jiffies up to a full second
447 : : * @j: the time in (absolute) jiffies that should be rounded
448 : : *
449 : : * This is the same as round_jiffies() except that it will never
450 : : * round down. This is useful for timeouts for which the exact time
451 : : * of firing does not matter too much, as long as they don't fire too
452 : : * early.
453 : : */
454 : 24617 : unsigned long round_jiffies_up(unsigned long j)
455 : : {
456 [ - + ]: 24617 : return round_jiffies_common(j, raw_smp_processor_id(), true);
457 : : }
458 : : EXPORT_SYMBOL_GPL(round_jiffies_up);
459 : :
460 : : /**
461 : : * round_jiffies_up_relative - function to round jiffies up to a full second
462 : : * @j: the time in (relative) jiffies that should be rounded
463 : : *
464 : : * This is the same as round_jiffies_relative() except that it will never
465 : : * round down. This is useful for timeouts for which the exact time
466 : : * of firing does not matter too much, as long as they don't fire too
467 : : * early.
468 : : */
469 : 0 : unsigned long round_jiffies_up_relative(unsigned long j)
470 : : {
471 [ # # ]: 0 : return __round_jiffies_up_relative(j, raw_smp_processor_id());
472 : : }
473 : : EXPORT_SYMBOL_GPL(round_jiffies_up_relative);
474 : :
475 : :
476 : 46077 : static inline unsigned int timer_get_idx(struct timer_list *timer)
477 : : {
478 : 46077 : return (timer->flags & TIMER_ARRAYMASK) >> TIMER_ARRAYSHIFT;
479 : : }
480 : :
481 : 23442 : static inline void timer_set_idx(struct timer_list *timer, unsigned int idx)
482 : : {
483 : 23442 : timer->flags = (timer->flags & ~TIMER_ARRAYMASK) |
484 : 23442 : idx << TIMER_ARRAYSHIFT;
485 : : }
486 : :
487 : : /*
488 : : * Helper function to calculate the array index for a given expiry
489 : : * time.
490 : : */
491 : 23435 : static inline unsigned calc_index(unsigned expires, unsigned lvl)
492 : : {
493 : 23435 : expires = (expires + LVL_GRAN(lvl)) >> LVL_SHIFT(lvl);
494 : 23435 : return LVL_OFFS(lvl) + (expires & LVL_MASK);
495 : : }
496 : :
497 : 23442 : static int calc_wheel_index(unsigned long expires, unsigned long clk)
498 : : {
499 : 23442 : unsigned long delta = expires - clk;
500 : 23442 : unsigned int idx;
501 : :
502 [ + + ]: 23442 : if (delta < LVL_START(1)) {
503 : 22509 : idx = calc_index(expires, 0);
504 [ + + ]: 933 : } else if (delta < LVL_START(2)) {
505 : 267 : idx = calc_index(expires, 1);
506 [ + + ]: 666 : } else if (delta < LVL_START(3)) {
507 : 156 : idx = calc_index(expires, 2);
508 [ + + ]: 510 : } else if (delta < LVL_START(4)) {
509 : 359 : idx = calc_index(expires, 3);
510 [ + + ]: 151 : } else if (delta < LVL_START(5)) {
511 : 100 : idx = calc_index(expires, 4);
512 [ + + ]: 51 : } else if (delta < LVL_START(6)) {
513 : 33 : idx = calc_index(expires, 5);
514 [ - + ]: 18 : } else if (delta < LVL_START(7)) {
515 : 0 : idx = calc_index(expires, 6);
516 [ + + ]: 18 : } else if (LVL_DEPTH > 8 && delta < LVL_START(8)) {
517 : 11 : idx = calc_index(expires, 7);
518 [ + - ]: 7 : } else if ((long) delta < 0) {
519 : 7 : idx = clk & LVL_MASK;
520 : : } else {
521 : : /*
522 : : * Force expire obscene large timeouts to expire at the
523 : : * capacity limit of the wheel.
524 : : */
525 [ # # ]: 0 : if (expires >= WHEEL_TIMEOUT_CUTOFF)
526 : 0 : expires = WHEEL_TIMEOUT_MAX;
527 : :
528 : 0 : idx = calc_index(expires, LVL_DEPTH - 1);
529 : : }
530 : 23442 : return idx;
531 : : }
532 : :
533 : : /*
534 : : * Enqueue the timer into the hash bucket, mark it pending in
535 : : * the bitmap and store the index in the timer flags.
536 : : */
537 : 23442 : static void enqueue_timer(struct timer_base *base, struct timer_list *timer,
538 : : unsigned int idx)
539 : : {
540 [ + + ]: 23442 : hlist_add_head(&timer->entry, base->vectors + idx);
541 : 23442 : __set_bit(idx, base->pending_map);
542 : 23442 : timer_set_idx(timer, idx);
543 : :
544 : 23442 : trace_timer_start(timer, timer->expires, timer->flags);
545 : 23442 : }
546 : :
547 : : static void
548 : 23442 : __internal_add_timer(struct timer_base *base, struct timer_list *timer)
549 : : {
550 : 23442 : unsigned int idx;
551 : :
552 : 23442 : idx = calc_wheel_index(timer->expires, base->clk);
553 : 23442 : enqueue_timer(base, timer, idx);
554 : 23442 : }
555 : :
556 : : static void
557 : : trigger_dyntick_cpu(struct timer_base *base, struct timer_list *timer)
558 : : {
559 : : if (!is_timers_nohz_active())
560 : : return;
561 : :
562 : : /*
563 : : * TODO: This wants some optimizing similar to the code below, but we
564 : : * will do that when we switch from push to pull for deferrable timers.
565 : : */
566 : : if (timer->flags & TIMER_DEFERRABLE) {
567 : : if (tick_nohz_full_cpu(base->cpu))
568 : : wake_up_nohz_cpu(base->cpu);
569 : : return;
570 : : }
571 : :
572 : : /*
573 : : * We might have to IPI the remote CPU if the base is idle and the
574 : : * timer is not deferrable. If the other CPU is on the way to idle
575 : : * then it can't set base->is_idle as we hold the base lock:
576 : : */
577 : : if (!base->is_idle)
578 : : return;
579 : :
580 : : /* Check whether this is the new first expiring timer: */
581 : : if (time_after_eq(timer->expires, base->next_expiry))
582 : : return;
583 : :
584 : : /*
585 : : * Set the next expiry time and kick the CPU so it can reevaluate the
586 : : * wheel:
587 : : */
588 : : base->next_expiry = timer->expires;
589 : : wake_up_nohz_cpu(base->cpu);
590 : : }
591 : :
592 : : static void
593 : 23442 : internal_add_timer(struct timer_base *base, struct timer_list *timer)
594 : : {
595 : 23442 : __internal_add_timer(base, timer);
596 : 23442 : trigger_dyntick_cpu(base, timer);
597 : 23342 : }
598 : :
599 : : #ifdef CONFIG_DEBUG_OBJECTS_TIMERS
600 : :
601 : : static struct debug_obj_descr timer_debug_descr;
602 : :
603 : : static void *timer_debug_hint(void *addr)
604 : : {
605 : : return ((struct timer_list *) addr)->function;
606 : : }
607 : :
608 : : static bool timer_is_static_object(void *addr)
609 : : {
610 : : struct timer_list *timer = addr;
611 : :
612 : : return (timer->entry.pprev == NULL &&
613 : : timer->entry.next == TIMER_ENTRY_STATIC);
614 : : }
615 : :
616 : : /*
617 : : * fixup_init is called when:
618 : : * - an active object is initialized
619 : : */
620 : : static bool timer_fixup_init(void *addr, enum debug_obj_state state)
621 : : {
622 : : struct timer_list *timer = addr;
623 : :
624 : : switch (state) {
625 : : case ODEBUG_STATE_ACTIVE:
626 : : del_timer_sync(timer);
627 : : debug_object_init(timer, &timer_debug_descr);
628 : : return true;
629 : : default:
630 : : return false;
631 : : }
632 : : }
633 : :
634 : : /* Stub timer callback for improperly used timers. */
635 : : static void stub_timer(struct timer_list *unused)
636 : : {
637 : : WARN_ON(1);
638 : : }
639 : :
640 : : /*
641 : : * fixup_activate is called when:
642 : : * - an active object is activated
643 : : * - an unknown non-static object is activated
644 : : */
645 : : static bool timer_fixup_activate(void *addr, enum debug_obj_state state)
646 : : {
647 : : struct timer_list *timer = addr;
648 : :
649 : : switch (state) {
650 : : case ODEBUG_STATE_NOTAVAILABLE:
651 : : timer_setup(timer, stub_timer, 0);
652 : : return true;
653 : :
654 : : case ODEBUG_STATE_ACTIVE:
655 : : WARN_ON(1);
656 : : /* fall through */
657 : : default:
658 : : return false;
659 : : }
660 : : }
661 : :
662 : : /*
663 : : * fixup_free is called when:
664 : : * - an active object is freed
665 : : */
666 : : static bool timer_fixup_free(void *addr, enum debug_obj_state state)
667 : : {
668 : : struct timer_list *timer = addr;
669 : :
670 : : switch (state) {
671 : : case ODEBUG_STATE_ACTIVE:
672 : : del_timer_sync(timer);
673 : : debug_object_free(timer, &timer_debug_descr);
674 : : return true;
675 : : default:
676 : : return false;
677 : : }
678 : : }
679 : :
680 : : /*
681 : : * fixup_assert_init is called when:
682 : : * - an untracked/uninit-ed object is found
683 : : */
684 : : static bool timer_fixup_assert_init(void *addr, enum debug_obj_state state)
685 : : {
686 : : struct timer_list *timer = addr;
687 : :
688 : : switch (state) {
689 : : case ODEBUG_STATE_NOTAVAILABLE:
690 : : timer_setup(timer, stub_timer, 0);
691 : : return true;
692 : : default:
693 : : return false;
694 : : }
695 : : }
696 : :
697 : : static struct debug_obj_descr timer_debug_descr = {
698 : : .name = "timer_list",
699 : : .debug_hint = timer_debug_hint,
700 : : .is_static_object = timer_is_static_object,
701 : : .fixup_init = timer_fixup_init,
702 : : .fixup_activate = timer_fixup_activate,
703 : : .fixup_free = timer_fixup_free,
704 : : .fixup_assert_init = timer_fixup_assert_init,
705 : : };
706 : :
707 : : static inline void debug_timer_init(struct timer_list *timer)
708 : : {
709 : : debug_object_init(timer, &timer_debug_descr);
710 : : }
711 : :
712 : : static inline void debug_timer_activate(struct timer_list *timer)
713 : : {
714 : : debug_object_activate(timer, &timer_debug_descr);
715 : : }
716 : :
717 : : static inline void debug_timer_deactivate(struct timer_list *timer)
718 : : {
719 : : debug_object_deactivate(timer, &timer_debug_descr);
720 : : }
721 : :
722 : : static inline void debug_timer_free(struct timer_list *timer)
723 : : {
724 : : debug_object_free(timer, &timer_debug_descr);
725 : : }
726 : :
727 : : static inline void debug_timer_assert_init(struct timer_list *timer)
728 : : {
729 : : debug_object_assert_init(timer, &timer_debug_descr);
730 : : }
731 : :
732 : : static void do_init_timer(struct timer_list *timer,
733 : : void (*func)(struct timer_list *),
734 : : unsigned int flags,
735 : : const char *name, struct lock_class_key *key);
736 : :
737 : : void init_timer_on_stack_key(struct timer_list *timer,
738 : : void (*func)(struct timer_list *),
739 : : unsigned int flags,
740 : : const char *name, struct lock_class_key *key)
741 : : {
742 : : debug_object_init_on_stack(timer, &timer_debug_descr);
743 : : do_init_timer(timer, func, flags, name, key);
744 : : }
745 : : EXPORT_SYMBOL_GPL(init_timer_on_stack_key);
746 : :
747 : : void destroy_timer_on_stack(struct timer_list *timer)
748 : : {
749 : : debug_object_free(timer, &timer_debug_descr);
750 : : }
751 : : EXPORT_SYMBOL_GPL(destroy_timer_on_stack);
752 : :
753 : : #else
754 : 41476 : static inline void debug_timer_init(struct timer_list *timer) { }
755 : 23442 : static inline void debug_timer_activate(struct timer_list *timer) { }
756 : 23165 : static inline void debug_timer_deactivate(struct timer_list *timer) { }
757 : 26087 : static inline void debug_timer_assert_init(struct timer_list *timer) { }
758 : : #endif
759 : :
760 : 41476 : static inline void debug_init(struct timer_list *timer)
761 : : {
762 : 41476 : debug_timer_init(timer);
763 : 41476 : trace_timer_init(timer);
764 : : }
765 : :
766 : 23165 : static inline void debug_deactivate(struct timer_list *timer)
767 : : {
768 : 23165 : debug_timer_deactivate(timer);
769 : 23165 : trace_timer_cancel(timer);
770 : : }
771 : :
772 : 26087 : static inline void debug_assert_init(struct timer_list *timer)
773 : : {
774 : 26087 : debug_timer_assert_init(timer);
775 : : }
776 : :
777 : 41476 : static void do_init_timer(struct timer_list *timer,
778 : : void (*func)(struct timer_list *),
779 : : unsigned int flags,
780 : : const char *name, struct lock_class_key *key)
781 : : {
782 : 41476 : timer->entry.pprev = NULL;
783 : 41476 : timer->function = func;
784 : 82952 : timer->flags = flags | raw_smp_processor_id();
785 : 41476 : lockdep_init_map(&timer->lockdep_map, name, key, 0);
786 : : }
787 : :
788 : : /**
789 : : * init_timer_key - initialize a timer
790 : : * @timer: the timer to be initialized
791 : : * @func: timer callback function
792 : : * @flags: timer flags
793 : : * @name: name of the timer
794 : : * @key: lockdep class key of the fake lock used for tracking timer
795 : : * sync lock dependencies
796 : : *
797 : : * init_timer_key() must be done to a timer prior calling *any* of the
798 : : * other timer functions.
799 : : */
800 : 41476 : void init_timer_key(struct timer_list *timer,
801 : : void (*func)(struct timer_list *), unsigned int flags,
802 : : const char *name, struct lock_class_key *key)
803 : : {
804 : 41476 : debug_init(timer);
805 : 41476 : do_init_timer(timer, func, flags, name, key);
806 : 19480 : }
807 : : EXPORT_SYMBOL(init_timer_key);
808 : :
809 : 23165 : static inline void detach_timer(struct timer_list *timer, bool clear_pending)
810 : : {
811 : 23165 : struct hlist_node *entry = &timer->entry;
812 : :
813 : 23165 : debug_deactivate(timer);
814 : :
815 [ - - + + : 23165 : __hlist_del(entry);
+ + ]
816 [ + - ]: 22186 : if (clear_pending)
817 : 22186 : entry->pprev = NULL;
818 : 23165 : entry->next = LIST_POISON2;
819 : : }
820 : :
821 : 46077 : static int detach_if_pending(struct timer_list *timer, struct timer_base *base,
822 : : bool clear_pending)
823 : : {
824 : 46077 : unsigned idx = timer_get_idx(timer);
825 : :
826 [ + + ]: 46077 : if (!timer_pending(timer))
827 : : return 0;
828 : :
829 [ + + + + ]: 44372 : if (hlist_is_singular_node(&timer->entry, base->vectors + idx))
830 : 21388 : __clear_bit(idx, base->pending_map);
831 : :
832 : 22186 : detach_timer(timer, clear_pending);
833 : 22186 : return 1;
834 : : }
835 : :
836 : 46277 : static inline struct timer_base *get_timer_cpu_base(u32 tflags, u32 cpu)
837 : : {
838 : 46277 : struct timer_base *base = per_cpu_ptr(&timer_bases[BASE_STD], cpu);
839 : :
840 : : /*
841 : : * If the timer is deferrable and NO_HZ_COMMON is set then we need
842 : : * to use the deferrable base.
843 : : */
844 : 46277 : if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && (tflags & TIMER_DEFERRABLE))
845 : 413 : base = per_cpu_ptr(&timer_bases[BASE_DEF], cpu);
846 : 46277 : return base;
847 : : }
848 : :
849 : : static inline struct timer_base *get_timer_this_cpu_base(u32 tflags)
850 : : {
851 : : struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
852 : :
853 : : /*
854 : : * If the timer is deferrable and NO_HZ_COMMON is set then we need
855 : : * to use the deferrable base.
856 : : */
857 : : if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && (tflags & TIMER_DEFERRABLE))
858 : : base = this_cpu_ptr(&timer_bases[BASE_DEF]);
859 : : return base;
860 : : }
861 : :
862 : 46177 : static inline struct timer_base *get_timer_base(u32 tflags)
863 : : {
864 : 46177 : return get_timer_cpu_base(tflags, tflags & TIMER_CPUMASK);
865 : : }
866 : :
867 : : static inline struct timer_base *
868 : : get_target_base(struct timer_base *base, unsigned tflags)
869 : : {
870 : : #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
871 : : if (static_branch_likely(&timers_migration_enabled) &&
872 : : !(tflags & TIMER_PINNED))
873 : : return get_timer_cpu_base(tflags, get_nohz_timer_target());
874 : : #endif
875 : : return get_timer_this_cpu_base(tflags);
876 : : }
877 : :
878 : 23442 : static inline void forward_timer_base(struct timer_base *base)
879 : : {
880 : : #ifdef CONFIG_NO_HZ_COMMON
881 : 23442 : unsigned long jnow;
882 : :
883 : : /*
884 : : * We only forward the base when we are idle or have just come out of
885 : : * idle (must_forward_clk logic), and have a delta between base clock
886 : : * and jiffies. In the common case, run_timers will take care of it.
887 : : */
888 : 23442 : if (likely(!base->must_forward_clk))
889 : : return;
890 : :
891 [ # # # # : 0 : jnow = READ_ONCE(jiffies);
# # # # #
# ]
892 : 0 : base->must_forward_clk = base->is_idle;
893 [ # # # # : 0 : if ((long)(jnow - base->clk) < 2)
# # # # #
# ]
894 : : return;
895 : :
896 : : /*
897 : : * If the next expiry value is > jiffies, then we fast forward to
898 : : * jiffies otherwise we forward to the next expiry value.
899 : : */
900 [ # # # # : 0 : if (time_after(base->next_expiry, jnow))
# # # # #
# ]
901 : 0 : base->clk = jnow;
902 : : else
903 : 0 : base->clk = base->next_expiry;
904 : : #endif
905 : : }
906 : :
907 : :
908 : : /*
909 : : * We are using hashed locking: Holding per_cpu(timer_bases[x]).lock means
910 : : * that all timers which are tied to this base are locked, and the base itself
911 : : * is locked too.
912 : : *
913 : : * So __run_timers/migrate_timers can safely modify all timers which could
914 : : * be found in the base->vectors array.
915 : : *
916 : : * When a timer is migrating then the TIMER_MIGRATING flag is set and we need
917 : : * to wait until the migration is done.
918 : : */
919 : 46177 : static struct timer_base *lock_timer_base(struct timer_list *timer,
920 : : unsigned long *flags)
921 : : __acquires(timer->base->lock)
922 : : {
923 : 46177 : for (;;) {
924 : 46177 : struct timer_base *base;
925 : 46177 : u32 tf;
926 : :
927 : : /*
928 : : * We need to use READ_ONCE() here, otherwise the compiler
929 : : * might re-read @tf between the check for TIMER_MIGRATING
930 : : * and spin_lock().
931 : : */
932 [ + - ]: 46177 : tf = READ_ONCE(timer->flags);
933 : :
934 [ + - ]: 46177 : if (!(tf & TIMER_MIGRATING)) {
935 [ + + ]: 46177 : base = get_timer_base(tf);
936 : 46177 : raw_spin_lock_irqsave(&base->lock, *flags);
937 [ + - ]: 46177 : if (timer->flags == tf)
938 : 46177 : return base;
939 : 0 : raw_spin_unlock_irqrestore(&base->lock, *flags);
940 : : }
941 : 0 : cpu_relax();
942 : : }
943 : : }
944 : :
945 : : #define MOD_TIMER_PENDING_ONLY 0x01
946 : : #define MOD_TIMER_REDUCE 0x02
947 : :
948 : : static inline int
949 : 23342 : __mod_timer(struct timer_list *timer, unsigned long expires, unsigned int options)
950 : : {
951 : 23342 : struct timer_base *base, *new_base;
952 : 23342 : unsigned int idx = UINT_MAX;
953 : 23342 : unsigned long clk = 0, flags;
954 : 23342 : int ret = 0;
955 : :
956 [ - + ]: 23342 : BUG_ON(!timer->function);
957 : :
958 : : /*
959 : : * This is a common optimization triggered by the networking code - if
960 : : * the timer is re-modified to have the same timeout or ends up in the
961 : : * same array bucket then just return:
962 : : */
963 [ - + ]: 23342 : if (timer_pending(timer)) {
964 : : /*
965 : : * The downside of this optimization is that it can result in
966 : : * larger granularity than you would get from adding a new
967 : : * timer with this expiry.
968 : : */
969 : 0 : long diff = timer->expires - expires;
970 : :
971 [ # # ]: 0 : if (!diff)
972 : : return 1;
973 [ # # # # ]: 0 : if (options & MOD_TIMER_REDUCE && diff <= 0)
974 : : return 1;
975 : :
976 : : /*
977 : : * We lock timer base and calculate the bucket index right
978 : : * here. If the timer ends up in the same bucket, then we
979 : : * just update the expiry time and avoid the whole
980 : : * dequeue/enqueue dance.
981 : : */
982 : 0 : base = lock_timer_base(timer, &flags);
983 [ # # ]: 0 : forward_timer_base(base);
984 : :
985 [ # # # # ]: 0 : if (timer_pending(timer) && (options & MOD_TIMER_REDUCE) &&
986 [ # # ]: 0 : time_before_eq(timer->expires, expires)) {
987 : 0 : ret = 1;
988 : 0 : goto out_unlock;
989 : : }
990 : :
991 : 0 : clk = base->clk;
992 : 0 : idx = calc_wheel_index(expires, clk);
993 : :
994 : : /*
995 : : * Retrieve and compare the array index of the pending
996 : : * timer. If it matches set the expiry to the new value so a
997 : : * subsequent call will exit in the expires check above.
998 : : */
999 [ # # ]: 0 : if (idx == timer_get_idx(timer)) {
1000 [ # # ]: 0 : if (!(options & MOD_TIMER_REDUCE))
1001 : 0 : timer->expires = expires;
1002 [ # # ]: 0 : else if (time_after(timer->expires, expires))
1003 : 0 : timer->expires = expires;
1004 : 0 : ret = 1;
1005 : 0 : goto out_unlock;
1006 : : }
1007 : : } else {
1008 : 23342 : base = lock_timer_base(timer, &flags);
1009 [ - + ]: 23342 : forward_timer_base(base);
1010 : : }
1011 : :
1012 : 23342 : ret = detach_if_pending(timer, base, false);
1013 [ + - - + ]: 23342 : if (!ret && (options & MOD_TIMER_PENDING_ONLY))
1014 : 0 : goto out_unlock;
1015 : :
1016 : 23342 : new_base = get_target_base(base, timer->flags);
1017 : :
1018 [ - + ]: 23342 : if (base != new_base) {
1019 : : /*
1020 : : * We are trying to schedule the timer on the new base.
1021 : : * However we can't change timer's base while it is running,
1022 : : * otherwise del_timer_sync() can't detect that the timer's
1023 : : * handler yet has not finished. This also guarantees that the
1024 : : * timer is serialized wrt itself.
1025 : : */
1026 [ # # ]: 0 : if (likely(base->running_timer != timer)) {
1027 : : /* See the comment in lock_timer_base() */
1028 : 0 : timer->flags |= TIMER_MIGRATING;
1029 : :
1030 : 0 : raw_spin_unlock(&base->lock);
1031 : 0 : base = new_base;
1032 : 0 : raw_spin_lock(&base->lock);
1033 [ # # ]: 0 : WRITE_ONCE(timer->flags,
1034 : : (timer->flags & ~TIMER_BASEMASK) | base->cpu);
1035 [ # # ]: 0 : forward_timer_base(base);
1036 : : }
1037 : : }
1038 : :
1039 : 23342 : debug_timer_activate(timer);
1040 : :
1041 : 23342 : timer->expires = expires;
1042 : : /*
1043 : : * If 'idx' was calculated above and the base time did not advance
1044 : : * between calculating 'idx' and possibly switching the base, only
1045 : : * enqueue_timer() and trigger_dyntick_cpu() is required. Otherwise
1046 : : * we need to (re)calculate the wheel index via
1047 : : * internal_add_timer().
1048 : : */
1049 [ - + - - ]: 23342 : if (idx != UINT_MAX && clk == base->clk) {
1050 : 0 : enqueue_timer(base, timer, idx);
1051 : 0 : trigger_dyntick_cpu(base, timer);
1052 : : } else {
1053 : 23342 : internal_add_timer(base, timer);
1054 : : }
1055 : :
1056 : 23342 : out_unlock:
1057 : 23342 : raw_spin_unlock_irqrestore(&base->lock, flags);
1058 : :
1059 : 23342 : return ret;
1060 : : }
1061 : :
1062 : : /**
1063 : : * mod_timer_pending - modify a pending timer's timeout
1064 : : * @timer: the pending timer to be modified
1065 : : * @expires: new timeout in jiffies
1066 : : *
1067 : : * mod_timer_pending() is the same for pending timers as mod_timer(),
1068 : : * but will not re-activate and modify already deleted timers.
1069 : : *
1070 : : * It is useful for unserialized use of timers.
1071 : : */
1072 : 0 : int mod_timer_pending(struct timer_list *timer, unsigned long expires)
1073 : : {
1074 : 0 : return __mod_timer(timer, expires, MOD_TIMER_PENDING_ONLY);
1075 : : }
1076 : : EXPORT_SYMBOL(mod_timer_pending);
1077 : :
1078 : : /**
1079 : : * mod_timer - modify a timer's timeout
1080 : : * @timer: the timer to be modified
1081 : : * @expires: new timeout in jiffies
1082 : : *
1083 : : * mod_timer() is a more efficient way to update the expire field of an
1084 : : * active timer (if the timer is inactive it will be activated)
1085 : : *
1086 : : * mod_timer(timer, expires) is equivalent to:
1087 : : *
1088 : : * del_timer(timer); timer->expires = expires; add_timer(timer);
1089 : : *
1090 : : * Note that if there are multiple unserialized concurrent users of the
1091 : : * same timer, then mod_timer() is the only safe way to modify the timeout,
1092 : : * since add_timer() cannot modify an already running timer.
1093 : : *
1094 : : * The function returns whether it has modified a pending timer or not.
1095 : : * (ie. mod_timer() of an inactive timer returns 0, mod_timer() of an
1096 : : * active timer returns 1.)
1097 : : */
1098 : 1278 : int mod_timer(struct timer_list *timer, unsigned long expires)
1099 : : {
1100 : 165 : return __mod_timer(timer, expires, 0);
1101 : : }
1102 : : EXPORT_SYMBOL(mod_timer);
1103 : :
1104 : : /**
1105 : : * timer_reduce - Modify a timer's timeout if it would reduce the timeout
1106 : : * @timer: The timer to be modified
1107 : : * @expires: New timeout in jiffies
1108 : : *
1109 : : * timer_reduce() is very similar to mod_timer(), except that it will only
1110 : : * modify a running timer if that would reduce the expiration time (it will
1111 : : * start a timer that isn't running).
1112 : : */
1113 : 68 : int timer_reduce(struct timer_list *timer, unsigned long expires)
1114 : : {
1115 : 68 : return __mod_timer(timer, expires, MOD_TIMER_REDUCE);
1116 : : }
1117 : : EXPORT_SYMBOL(timer_reduce);
1118 : :
1119 : : /**
1120 : : * add_timer - start a timer
1121 : : * @timer: the timer to be added
1122 : : *
1123 : : * The kernel will do a ->function(@timer) callback from the
1124 : : * timer interrupt at the ->expires point in the future. The
1125 : : * current time is 'jiffies'.
1126 : : *
1127 : : * The timer's ->expires, ->function fields must be set prior calling this
1128 : : * function.
1129 : : *
1130 : : * Timers with an ->expires field in the past will be executed in the next
1131 : : * timer tick.
1132 : : */
1133 : 1113 : void add_timer(struct timer_list *timer)
1134 : : {
1135 [ - + ]: 1113 : BUG_ON(timer_pending(timer));
1136 : 1113 : mod_timer(timer, timer->expires);
1137 : 1113 : }
1138 : : EXPORT_SYMBOL(add_timer);
1139 : :
1140 : : /**
1141 : : * add_timer_on - start a timer on a particular CPU
1142 : : * @timer: the timer to be added
1143 : : * @cpu: the CPU to start it on
1144 : : *
1145 : : * This is not very scalable on SMP. Double adds are not possible.
1146 : : */
1147 : 100 : void add_timer_on(struct timer_list *timer, int cpu)
1148 : : {
1149 : 100 : struct timer_base *new_base, *base;
1150 : 100 : unsigned long flags;
1151 : :
1152 [ + - - + ]: 100 : BUG_ON(timer_pending(timer) || !timer->function);
1153 : :
1154 [ + + ]: 100 : new_base = get_timer_cpu_base(timer->flags, cpu);
1155 : :
1156 : : /*
1157 : : * If @timer was on a different CPU, it should be migrated with the
1158 : : * old base locked to prevent other operations proceeding with the
1159 : : * wrong base locked. See lock_timer_base().
1160 : : */
1161 : 100 : base = lock_timer_base(timer, &flags);
1162 [ - + ]: 100 : if (base != new_base) {
1163 : 0 : timer->flags |= TIMER_MIGRATING;
1164 : :
1165 : 0 : raw_spin_unlock(&base->lock);
1166 : 0 : base = new_base;
1167 : 0 : raw_spin_lock(&base->lock);
1168 : 0 : WRITE_ONCE(timer->flags,
1169 : : (timer->flags & ~TIMER_BASEMASK) | cpu);
1170 : : }
1171 [ - + ]: 100 : forward_timer_base(base);
1172 : :
1173 : 100 : debug_timer_activate(timer);
1174 : 100 : internal_add_timer(base, timer);
1175 : 100 : raw_spin_unlock_irqrestore(&base->lock, flags);
1176 : 100 : }
1177 : : EXPORT_SYMBOL_GPL(add_timer_on);
1178 : :
1179 : : /**
1180 : : * del_timer - deactivate a timer.
1181 : : * @timer: the timer to be deactivated
1182 : : *
1183 : : * del_timer() deactivates a timer - this works on both active and inactive
1184 : : * timers.
1185 : : *
1186 : : * The function returns whether it has deactivated a pending timer or not.
1187 : : * (ie. del_timer() of an inactive timer returns 0, del_timer() of an
1188 : : * active timer returns 1.)
1189 : : */
1190 : 3452 : int del_timer(struct timer_list *timer)
1191 : : {
1192 : 3452 : struct timer_base *base;
1193 : 3452 : unsigned long flags;
1194 : 3452 : int ret = 0;
1195 : :
1196 : 3452 : debug_assert_init(timer);
1197 : :
1198 [ + + ]: 3452 : if (timer_pending(timer)) {
1199 : 100 : base = lock_timer_base(timer, &flags);
1200 : 100 : ret = detach_if_pending(timer, base, true);
1201 : 100 : raw_spin_unlock_irqrestore(&base->lock, flags);
1202 : : }
1203 : :
1204 : 3452 : return ret;
1205 : : }
1206 : : EXPORT_SYMBOL(del_timer);
1207 : :
1208 : : /**
1209 : : * try_to_del_timer_sync - Try to deactivate a timer
1210 : : * @timer: timer to delete
1211 : : *
1212 : : * This function tries to deactivate a timer. Upon successful (ret >= 0)
1213 : : * exit the timer is not queued and the handler is not running on any CPU.
1214 : : */
1215 : 22635 : int try_to_del_timer_sync(struct timer_list *timer)
1216 : : {
1217 : 22635 : struct timer_base *base;
1218 : 22635 : unsigned long flags;
1219 : 22635 : int ret = -1;
1220 : :
1221 : 22635 : debug_assert_init(timer);
1222 : :
1223 : 22635 : base = lock_timer_base(timer, &flags);
1224 : :
1225 [ + - ]: 22635 : if (base->running_timer != timer)
1226 : 22635 : ret = detach_if_pending(timer, base, true);
1227 : :
1228 : 22635 : raw_spin_unlock_irqrestore(&base->lock, flags);
1229 : :
1230 : 22635 : return ret;
1231 : : }
1232 : : EXPORT_SYMBOL(try_to_del_timer_sync);
1233 : :
1234 : : #ifdef CONFIG_PREEMPT_RT
1235 : : static __init void timer_base_init_expiry_lock(struct timer_base *base)
1236 : : {
1237 : : spin_lock_init(&base->expiry_lock);
1238 : : }
1239 : :
1240 : : static inline void timer_base_lock_expiry(struct timer_base *base)
1241 : : {
1242 : : spin_lock(&base->expiry_lock);
1243 : : }
1244 : :
1245 : : static inline void timer_base_unlock_expiry(struct timer_base *base)
1246 : : {
1247 : : spin_unlock(&base->expiry_lock);
1248 : : }
1249 : :
1250 : : /*
1251 : : * The counterpart to del_timer_wait_running().
1252 : : *
1253 : : * If there is a waiter for base->expiry_lock, then it was waiting for the
1254 : : * timer callback to finish. Drop expiry_lock and reaquire it. That allows
1255 : : * the waiter to acquire the lock and make progress.
1256 : : */
1257 : : static void timer_sync_wait_running(struct timer_base *base)
1258 : : {
1259 : : if (atomic_read(&base->timer_waiters)) {
1260 : : spin_unlock(&base->expiry_lock);
1261 : : spin_lock(&base->expiry_lock);
1262 : : }
1263 : : }
1264 : :
1265 : : /*
1266 : : * This function is called on PREEMPT_RT kernels when the fast path
1267 : : * deletion of a timer failed because the timer callback function was
1268 : : * running.
1269 : : *
1270 : : * This prevents priority inversion, if the softirq thread on a remote CPU
1271 : : * got preempted, and it prevents a life lock when the task which tries to
1272 : : * delete a timer preempted the softirq thread running the timer callback
1273 : : * function.
1274 : : */
1275 : : static void del_timer_wait_running(struct timer_list *timer)
1276 : : {
1277 : : u32 tf;
1278 : :
1279 : : tf = READ_ONCE(timer->flags);
1280 : : if (!(tf & TIMER_MIGRATING)) {
1281 : : struct timer_base *base = get_timer_base(tf);
1282 : :
1283 : : /*
1284 : : * Mark the base as contended and grab the expiry lock,
1285 : : * which is held by the softirq across the timer
1286 : : * callback. Drop the lock immediately so the softirq can
1287 : : * expire the next timer. In theory the timer could already
1288 : : * be running again, but that's more than unlikely and just
1289 : : * causes another wait loop.
1290 : : */
1291 : : atomic_inc(&base->timer_waiters);
1292 : : spin_lock_bh(&base->expiry_lock);
1293 : : atomic_dec(&base->timer_waiters);
1294 : : spin_unlock_bh(&base->expiry_lock);
1295 : : }
1296 : : }
1297 : : #else
1298 : 22 : static inline void timer_base_init_expiry_lock(struct timer_base *base) { }
1299 : 46166 : static inline void timer_base_lock_expiry(struct timer_base *base) { }
1300 : 46166 : static inline void timer_base_unlock_expiry(struct timer_base *base) { }
1301 : 232 : static inline void timer_sync_wait_running(struct timer_base *base) { }
1302 : 0 : static inline void del_timer_wait_running(struct timer_list *timer) { }
1303 : : #endif
1304 : :
1305 : : #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
1306 : : /**
1307 : : * del_timer_sync - deactivate a timer and wait for the handler to finish.
1308 : : * @timer: the timer to be deactivated
1309 : : *
1310 : : * This function only differs from del_timer() on SMP: besides deactivating
1311 : : * the timer it also makes sure the handler has finished executing on other
1312 : : * CPUs.
1313 : : *
1314 : : * Synchronization rules: Callers must prevent restarting of the timer,
1315 : : * otherwise this function is meaningless. It must not be called from
1316 : : * interrupt contexts unless the timer is an irqsafe one. The caller must
1317 : : * not hold locks which would prevent completion of the timer's
1318 : : * handler. The timer's handler must not call add_timer_on(). Upon exit the
1319 : : * timer is not queued and the handler is not running on any CPU.
1320 : : *
1321 : : * Note: For !irqsafe timers, you must not hold locks that are held in
1322 : : * interrupt context while calling this function. Even if the lock has
1323 : : * nothing to do with the timer in question. Here's why::
1324 : : *
1325 : : * CPU0 CPU1
1326 : : * ---- ----
1327 : : * <SOFTIRQ>
1328 : : * call_timer_fn();
1329 : : * base->running_timer = mytimer;
1330 : : * spin_lock_irq(somelock);
1331 : : * <IRQ>
1332 : : * spin_lock(somelock);
1333 : : * del_timer_sync(mytimer);
1334 : : * while (base->running_timer == mytimer);
1335 : : *
1336 : : * Now del_timer_sync() will never return and never release somelock.
1337 : : * The interrupt on the other CPU is waiting to grab somelock but
1338 : : * it has interrupted the softirq that CPU0 is waiting to finish.
1339 : : *
1340 : : * The function returns whether it has deactivated a pending timer or not.
1341 : : */
1342 : 22635 : int del_timer_sync(struct timer_list *timer)
1343 : : {
1344 : 22635 : int ret;
1345 : :
1346 : : #ifdef CONFIG_LOCKDEP
1347 : : unsigned long flags;
1348 : :
1349 : : /*
1350 : : * If lockdep gives a backtrace here, please reference
1351 : : * the synchronization rules above.
1352 : : */
1353 : : local_irq_save(flags);
1354 : : lock_map_acquire(&timer->lockdep_map);
1355 : : lock_map_release(&timer->lockdep_map);
1356 : : local_irq_restore(flags);
1357 : : #endif
1358 : : /*
1359 : : * don't use it in hardirq context, because it
1360 : : * could lead to deadlock.
1361 : : */
1362 [ - + - - : 45270 : WARN_ON(in_irq() && !(timer->flags & TIMER_IRQSAFE));
- + ]
1363 : :
1364 : 22635 : do {
1365 : 22635 : ret = try_to_del_timer_sync(timer);
1366 : :
1367 [ - + ]: 22635 : if (unlikely(ret < 0)) {
1368 : 0 : del_timer_wait_running(timer);
1369 : 0 : cpu_relax();
1370 : : }
1371 [ - + ]: 22635 : } while (ret < 0);
1372 : :
1373 : 22635 : return ret;
1374 : : }
1375 : : EXPORT_SYMBOL(del_timer_sync);
1376 : : #endif
1377 : :
1378 : 979 : static void call_timer_fn(struct timer_list *timer,
1379 : : void (*fn)(struct timer_list *),
1380 : : unsigned long baseclk)
1381 : : {
1382 : 979 : int count = preempt_count();
1383 : :
1384 : : #ifdef CONFIG_LOCKDEP
1385 : : /*
1386 : : * It is permissible to free the timer from inside the
1387 : : * function that is called from it, this we need to take into
1388 : : * account for lockdep too. To avoid bogus "held lock freed"
1389 : : * warnings as well as problems when looking into
1390 : : * timer->lockdep_map, make a copy and use that here.
1391 : : */
1392 : : struct lockdep_map lockdep_map;
1393 : :
1394 : : lockdep_copy_map(&lockdep_map, &timer->lockdep_map);
1395 : : #endif
1396 : : /*
1397 : : * Couple the lock chain with the lock chain at
1398 : : * del_timer_sync() by acquiring the lock_map around the fn()
1399 : : * call here and in del_timer_sync().
1400 : : */
1401 : 979 : lock_map_acquire(&lockdep_map);
1402 : :
1403 : 979 : trace_timer_expire_entry(timer, baseclk);
1404 : 979 : fn(timer);
1405 : 979 : trace_timer_expire_exit(timer);
1406 : :
1407 : 979 : lock_map_release(&lockdep_map);
1408 : :
1409 [ - + ]: 979 : if (count != preempt_count()) {
1410 [ # # ]: 0 : WARN_ONCE(1, "timer: %pS preempt leak: %08x -> %08x\n",
1411 : : fn, count, preempt_count());
1412 : : /*
1413 : : * Restore the preempt count. That gives us a decent
1414 : : * chance to survive and extract information. If the
1415 : : * callback kept a lock held, bad luck, but not worse
1416 : : * than the BUG() we had.
1417 : : */
1418 : : preempt_count_set(count);
1419 : : }
1420 : 979 : }
1421 : :
1422 : 938 : static void expire_timers(struct timer_base *base, struct hlist_head *head)
1423 : : {
1424 : : /*
1425 : : * This value is required only for tracing. base->clk was
1426 : : * incremented directly before expire_timers was called. But expiry
1427 : : * is related to the old base->clk value.
1428 : : */
1429 : 938 : unsigned long baseclk = base->clk - 1;
1430 : :
1431 [ + + ]: 1917 : while (!hlist_empty(head)) {
1432 : 979 : struct timer_list *timer;
1433 : 979 : void (*fn)(struct timer_list *);
1434 : :
1435 : 979 : timer = hlist_entry(head->first, struct timer_list, entry);
1436 : :
1437 : 979 : base->running_timer = timer;
1438 : 979 : detach_timer(timer, true);
1439 : :
1440 : 979 : fn = timer->function;
1441 : :
1442 [ + + ]: 979 : if (timer->flags & TIMER_IRQSAFE) {
1443 : 747 : raw_spin_unlock(&base->lock);
1444 : 747 : call_timer_fn(timer, fn, baseclk);
1445 : 747 : base->running_timer = NULL;
1446 : 747 : raw_spin_lock(&base->lock);
1447 : : } else {
1448 : 232 : raw_spin_unlock_irq(&base->lock);
1449 : 232 : call_timer_fn(timer, fn, baseclk);
1450 : 232 : base->running_timer = NULL;
1451 : 232 : timer_sync_wait_running(base);
1452 : 232 : raw_spin_lock_irq(&base->lock);
1453 : : }
1454 : : }
1455 : 938 : }
1456 : :
1457 : 46173 : static int __collect_expired_timers(struct timer_base *base,
1458 : : struct hlist_head *heads)
1459 : : {
1460 : 46173 : unsigned long clk = base->clk;
1461 : 46173 : struct hlist_head *vec;
1462 : 46173 : int i, levels = 0;
1463 : 46173 : unsigned int idx;
1464 : :
1465 [ + - ]: 52888 : for (i = 0; i < LVL_DEPTH; i++) {
1466 : 52888 : idx = (clk & LVL_MASK) + i * LVL_SIZE;
1467 : :
1468 [ + + ]: 105776 : if (__test_and_clear_bit(idx, base->pending_map)) {
1469 : 938 : vec = base->vectors + idx;
1470 [ + - ]: 938 : hlist_move_list(vec, heads++);
1471 : 938 : levels++;
1472 : : }
1473 : : /* Is it time to look at the next level? */
1474 [ + + ]: 52888 : if (clk & LVL_CLK_MASK)
1475 : : break;
1476 : : /* Shift clock for the next level granularity */
1477 : 6715 : clk >>= LVL_CLK_SHIFT;
1478 : : }
1479 : 46173 : return levels;
1480 : : }
1481 : :
1482 : : #ifdef CONFIG_NO_HZ_COMMON
1483 : : /*
1484 : : * Find the next pending bucket of a level. Search from level start (@offset)
1485 : : * + @clk upwards and if nothing there, search from start of the level
1486 : : * (@offset) up to @offset + clk.
1487 : : */
1488 : 4725 : static int next_pending_bucket(struct timer_base *base, unsigned offset,
1489 : : unsigned clk)
1490 : : {
1491 : 4725 : unsigned pos, start = offset + clk;
1492 : 4725 : unsigned end = offset + LVL_SIZE;
1493 : :
1494 : 4725 : pos = find_next_bit(base->pending_map, end, start);
1495 [ + + ]: 4725 : if (pos < end)
1496 : 1042 : return pos - start;
1497 : :
1498 : 3683 : pos = find_next_bit(base->pending_map, start, offset);
1499 [ + + ]: 3683 : return pos < start ? pos + LVL_SIZE - start : -1;
1500 : : }
1501 : :
1502 : : /*
1503 : : * Search the first expiring timer in the various clock levels. Caller must
1504 : : * hold base->lock.
1505 : : */
1506 : 525 : static unsigned long __next_timer_interrupt(struct timer_base *base)
1507 : : {
1508 : 525 : unsigned long clk, next, adj;
1509 : 525 : unsigned lvl, offset = 0;
1510 : :
1511 : 525 : next = base->clk + NEXT_TIMER_MAX_DELTA;
1512 : 525 : clk = base->clk;
1513 [ + + ]: 5250 : for (lvl = 0; lvl < LVL_DEPTH; lvl++, offset += LVL_SIZE) {
1514 : 4725 : int pos = next_pending_bucket(base, offset, clk & LVL_MASK);
1515 : :
1516 [ + + ]: 4725 : if (pos >= 0) {
1517 : 2757 : unsigned long tmp = clk + (unsigned long) pos;
1518 : :
1519 : 2757 : tmp <<= LVL_SHIFT(lvl);
1520 [ + + ]: 2757 : if (time_before(tmp, next))
1521 : 571 : next = tmp;
1522 : : }
1523 : : /*
1524 : : * Clock for the next level. If the current level clock lower
1525 : : * bits are zero, we look at the next level as is. If not we
1526 : : * need to advance it by one because that's going to be the
1527 : : * next expiring bucket in that level. base->clk is the next
1528 : : * expiring jiffie. So in case of:
1529 : : *
1530 : : * LVL5 LVL4 LVL3 LVL2 LVL1 LVL0
1531 : : * 0 0 0 0 0 0
1532 : : *
1533 : : * we have to look at all levels @index 0. With
1534 : : *
1535 : : * LVL5 LVL4 LVL3 LVL2 LVL1 LVL0
1536 : : * 0 0 0 0 0 2
1537 : : *
1538 : : * LVL0 has the next expiring bucket @index 2. The upper
1539 : : * levels have the next expiring bucket @index 1.
1540 : : *
1541 : : * In case that the propagation wraps the next level the same
1542 : : * rules apply:
1543 : : *
1544 : : * LVL5 LVL4 LVL3 LVL2 LVL1 LVL0
1545 : : * 0 0 0 0 F 2
1546 : : *
1547 : : * So after looking at LVL0 we get:
1548 : : *
1549 : : * LVL5 LVL4 LVL3 LVL2 LVL1
1550 : : * 0 0 0 1 0
1551 : : *
1552 : : * So no propagation from LVL1 to LVL2 because that happened
1553 : : * with the add already, but then we need to propagate further
1554 : : * from LVL2 to LVL3.
1555 : : *
1556 : : * So the simple check whether the lower bits of the current
1557 : : * level are 0 or not is sufficient for all cases.
1558 : : */
1559 : 4725 : adj = clk & LVL_CLK_MASK ? 1 : 0;
1560 : 4725 : clk >>= LVL_CLK_SHIFT;
1561 : 4725 : clk += adj;
1562 : : }
1563 : 525 : return next;
1564 : : }
1565 : :
1566 : : /*
1567 : : * Check, if the next hrtimer event is before the next timer wheel
1568 : : * event:
1569 : : */
1570 : 185 : static u64 cmp_next_hrtimer_event(u64 basem, u64 expires)
1571 : : {
1572 : 185 : u64 nextevt = hrtimer_get_next_event();
1573 : :
1574 : : /*
1575 : : * If high resolution timers are enabled
1576 : : * hrtimer_get_next_event() returns KTIME_MAX.
1577 : : */
1578 [ - + ]: 185 : if (expires <= nextevt)
1579 : : return expires;
1580 : :
1581 : : /*
1582 : : * If the next timer is already expired, return the tick base
1583 : : * time so the tick is fired immediately.
1584 : : */
1585 [ # # ]: 0 : if (nextevt <= basem)
1586 : : return basem;
1587 : :
1588 : : /*
1589 : : * Round up to the next jiffie. High resolution timers are
1590 : : * off, so the hrtimers are expired in the tick and we need to
1591 : : * make sure that this tick really expires the timer to avoid
1592 : : * a ping pong of the nohz stop code.
1593 : : *
1594 : : * Use DIV_ROUND_UP_ULL to prevent gcc calling __divdi3
1595 : : */
1596 : 0 : return DIV_ROUND_UP_ULL(nextevt, TICK_NSEC) * TICK_NSEC;
1597 : : }
1598 : :
1599 : : /**
1600 : : * get_next_timer_interrupt - return the time (clock mono) of the next timer
1601 : : * @basej: base time jiffies
1602 : : * @basem: base time clock monotonic
1603 : : *
1604 : : * Returns the tick aligned clock monotonic time of the next pending
1605 : : * timer or KTIME_MAX if no timer is pending.
1606 : : */
1607 : 185 : u64 get_next_timer_interrupt(unsigned long basej, u64 basem)
1608 : : {
1609 : 185 : struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
1610 : 185 : u64 expires = KTIME_MAX;
1611 : 185 : unsigned long nextevt;
1612 : 185 : bool is_max_delta;
1613 : :
1614 : : /*
1615 : : * Pretend that there is no timer pending if the cpu is offline.
1616 : : * Possible pending timers will be migrated later to an active cpu.
1617 : : */
1618 [ + - ]: 185 : if (cpu_is_offline(smp_processor_id()))
1619 : : return expires;
1620 : :
1621 : 185 : raw_spin_lock(&base->lock);
1622 : 185 : nextevt = __next_timer_interrupt(base);
1623 : 185 : is_max_delta = (nextevt == base->clk + NEXT_TIMER_MAX_DELTA);
1624 : 185 : base->next_expiry = nextevt;
1625 : : /*
1626 : : * We have a fresh next event. Check whether we can forward the
1627 : : * base. We can only do that when @basej is past base->clk
1628 : : * otherwise we might rewind base->clk.
1629 : : */
1630 [ + + ]: 185 : if (time_after(basej, base->clk)) {
1631 [ + - ]: 18 : if (time_after(nextevt, basej))
1632 : 18 : base->clk = basej;
1633 [ # # ]: 0 : else if (time_after(nextevt, base->clk))
1634 : 0 : base->clk = nextevt;
1635 : : }
1636 : :
1637 [ - + ]: 185 : if (time_before_eq(nextevt, basej)) {
1638 : 0 : expires = basem;
1639 : 0 : base->is_idle = false;
1640 : : } else {
1641 [ + - ]: 185 : if (!is_max_delta)
1642 : 185 : expires = basem + (u64)(nextevt - basej) * TICK_NSEC;
1643 : : /*
1644 : : * If we expect to sleep more than a tick, mark the base idle.
1645 : : * Also the tick is stopped so any added timer must forward
1646 : : * the base clk itself to keep granularity small. This idle
1647 : : * logic is only maintained for the BASE_STD base, deferrable
1648 : : * timers may still see large granularity skew (by design).
1649 : : */
1650 [ + + ]: 185 : if ((expires - basem) > TICK_NSEC) {
1651 : 183 : base->must_forward_clk = true;
1652 : 183 : base->is_idle = true;
1653 : : }
1654 : : }
1655 : 185 : raw_spin_unlock(&base->lock);
1656 : :
1657 : 185 : return cmp_next_hrtimer_event(basem, expires);
1658 : : }
1659 : :
1660 : : /**
1661 : : * timer_clear_idle - Clear the idle state of the timer base
1662 : : *
1663 : : * Called with interrupts disabled
1664 : : */
1665 : 3920 : void timer_clear_idle(void)
1666 : : {
1667 : 3920 : struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
1668 : :
1669 : : /*
1670 : : * We do this unlocked. The worst outcome is a remote enqueue sending
1671 : : * a pointless IPI, but taking the lock would just make the window for
1672 : : * sending the IPI a few instructions smaller for the cost of taking
1673 : : * the lock in the exit from idle path.
1674 : : */
1675 : 3920 : base->is_idle = false;
1676 : 3920 : }
1677 : :
1678 : 46342 : static int collect_expired_timers(struct timer_base *base,
1679 : : struct hlist_head *heads)
1680 : : {
1681 [ + + ]: 46342 : unsigned long now = READ_ONCE(jiffies);
1682 : :
1683 : : /*
1684 : : * NOHZ optimization. After a long idle sleep we need to forward the
1685 : : * base to current jiffies. Avoid a loop by searching the bitfield for
1686 : : * the next expiring timer.
1687 : : */
1688 [ + + ]: 46342 : if ((long)(now - base->clk) > 2) {
1689 : 340 : unsigned long next = __next_timer_interrupt(base);
1690 : :
1691 : : /*
1692 : : * If the next timer is ahead of time forward to current
1693 : : * jiffies, otherwise forward to the next expiry time:
1694 : : */
1695 [ + + ]: 340 : if (time_after(next, now)) {
1696 : : /*
1697 : : * The call site will increment base->clk and then
1698 : : * terminate the expiry loop immediately.
1699 : : */
1700 : 169 : base->clk = now;
1701 : 169 : return 0;
1702 : : }
1703 : 171 : base->clk = next;
1704 : : }
1705 : 46173 : return __collect_expired_timers(base, heads);
1706 : : }
1707 : : #else
1708 : : static inline int collect_expired_timers(struct timer_base *base,
1709 : : struct hlist_head *heads)
1710 : : {
1711 : : return __collect_expired_timers(base, heads);
1712 : : }
1713 : : #endif
1714 : :
1715 : : /*
1716 : : * Called from the timer interrupt handler to charge one tick to the current
1717 : : * process. user_tick is 1 if the tick is user time, 0 for system.
1718 : : */
1719 : 23095 : void update_process_times(int user_tick)
1720 : : {
1721 : 23095 : struct task_struct *p = current;
1722 : :
1723 : : /* Note: this timer irq context must be accounted for as well. */
1724 : 23095 : account_process_tick(p, user_tick);
1725 : 23095 : run_local_timers();
1726 : 23095 : rcu_sched_clock_irq(user_tick);
1727 : : #ifdef CONFIG_IRQ_WORK
1728 [ + - ]: 23095 : if (in_irq())
1729 : 23095 : irq_work_tick();
1730 : : #endif
1731 : 23095 : scheduler_tick();
1732 : 23095 : if (IS_ENABLED(CONFIG_POSIX_TIMERS))
1733 : 23095 : run_posix_cpu_timers();
1734 : 23095 : }
1735 : :
1736 : : /**
1737 : : * __run_timers - run all expired timers (if any) on this CPU.
1738 : : * @base: the timer vector to be processed.
1739 : : */
1740 : 46166 : static inline void __run_timers(struct timer_base *base)
1741 : : {
1742 : 46166 : struct hlist_head heads[LVL_DEPTH];
1743 : 46166 : int levels;
1744 : :
1745 [ - + ]: 46166 : if (!time_after_eq(jiffies, base->clk))
1746 : 0 : return;
1747 : :
1748 : 46166 : timer_base_lock_expiry(base);
1749 : 46166 : raw_spin_lock_irq(&base->lock);
1750 : :
1751 : : /*
1752 : : * timer_base::must_forward_clk must be cleared before running
1753 : : * timers so that any timer functions that call mod_timer() will
1754 : : * not try to forward the base. Idle tracking / clock forwarding
1755 : : * logic is only used with BASE_STD timers.
1756 : : *
1757 : : * The must_forward_clk flag is cleared unconditionally also for
1758 : : * the deferrable base. The deferrable base is not affected by idle
1759 : : * tracking and never forwarded, so clearing the flag is a NOOP.
1760 : : *
1761 : : * The fact that the deferrable base is never forwarded can cause
1762 : : * large variations in granularity for deferrable timers, but they
1763 : : * can be deferred for long periods due to idle anyway.
1764 : : */
1765 : 46166 : base->must_forward_clk = false;
1766 : :
1767 [ + + ]: 92508 : while (time_after_eq(jiffies, base->clk)) {
1768 : :
1769 : 46342 : levels = collect_expired_timers(base, heads);
1770 : 46342 : base->clk++;
1771 : :
1772 [ + + ]: 47280 : while (levels--)
1773 : 938 : expire_timers(base, heads + levels);
1774 : : }
1775 : 46166 : raw_spin_unlock_irq(&base->lock);
1776 : 46166 : timer_base_unlock_expiry(base);
1777 : : }
1778 : :
1779 : : /*
1780 : : * This function runs timers and the timer-tq in bottom half context.
1781 : : */
1782 : 23083 : static __latent_entropy void run_timer_softirq(struct softirq_action *h)
1783 : : {
1784 : 23083 : struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
1785 : :
1786 : 23083 : __run_timers(base);
1787 : 23083 : if (IS_ENABLED(CONFIG_NO_HZ_COMMON))
1788 : 23083 : __run_timers(this_cpu_ptr(&timer_bases[BASE_DEF]));
1789 : 23083 : }
1790 : :
1791 : : /*
1792 : : * Called by the local, per-CPU timer interrupt on SMP.
1793 : : */
1794 : 23095 : void run_local_timers(void)
1795 : : {
1796 : 23095 : struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
1797 : :
1798 : 23095 : hrtimer_run_queues();
1799 : : /* Raise the softirq only if required. */
1800 [ + + ]: 23095 : if (time_before(jiffies, base->clk)) {
1801 : 11 : if (!IS_ENABLED(CONFIG_NO_HZ_COMMON))
1802 : : return;
1803 : : /* CPU is awake, so check the deferrable base. */
1804 : 11 : base++;
1805 [ - + ]: 11 : if (time_before(jiffies, base->clk))
1806 : : return;
1807 : : }
1808 : 23084 : raise_softirq(TIMER_SOFTIRQ);
1809 : : }
1810 : :
1811 : : /*
1812 : : * Since schedule_timeout()'s timer is defined on the stack, it must store
1813 : : * the target task on the stack as well.
1814 : : */
1815 : : struct process_timer {
1816 : : struct timer_list timer;
1817 : : struct task_struct *task;
1818 : : };
1819 : :
1820 : 87 : static void process_timeout(struct timer_list *t)
1821 : : {
1822 : 87 : struct process_timer *timeout = from_timer(timeout, t, timer);
1823 : :
1824 : 87 : wake_up_process(timeout->task);
1825 : 87 : }
1826 : :
1827 : : /**
1828 : : * schedule_timeout - sleep until timeout
1829 : : * @timeout: timeout value in jiffies
1830 : : *
1831 : : * Make the current task sleep until @timeout jiffies have
1832 : : * elapsed. The routine will return immediately unless
1833 : : * the current task state has been set (see set_current_state()).
1834 : : *
1835 : : * You can set the task state as follows -
1836 : : *
1837 : : * %TASK_UNINTERRUPTIBLE - at least @timeout jiffies are guaranteed to
1838 : : * pass before the routine returns unless the current task is explicitly
1839 : : * woken up, (e.g. by wake_up_process())".
1840 : : *
1841 : : * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
1842 : : * delivered to the current task or the current task is explicitly woken
1843 : : * up.
1844 : : *
1845 : : * The current task state is guaranteed to be TASK_RUNNING when this
1846 : : * routine returns.
1847 : : *
1848 : : * Specifying a @timeout value of %MAX_SCHEDULE_TIMEOUT will schedule
1849 : : * the CPU away without a bound on the timeout. In this case the return
1850 : : * value will be %MAX_SCHEDULE_TIMEOUT.
1851 : : *
1852 : : * Returns 0 when the timer has expired otherwise the remaining time in
1853 : : * jiffies will be returned. In all cases the return value is guaranteed
1854 : : * to be non-negative.
1855 : : */
1856 : 24361 : signed long __sched schedule_timeout(signed long timeout)
1857 : : {
1858 : 24361 : struct process_timer timer;
1859 : 24361 : unsigned long expire;
1860 : :
1861 [ + + ]: 24361 : switch (timeout)
1862 : : {
1863 : 2365 : case MAX_SCHEDULE_TIMEOUT:
1864 : : /*
1865 : : * These two special cases are useful to be comfortable
1866 : : * in the caller. Nothing more. We could take
1867 : : * MAX_SCHEDULE_TIMEOUT from one of the negative value
1868 : : * but I' d like to return a valid offset (>=0) to allow
1869 : : * the caller to do everything it want with the retval.
1870 : : */
1871 : 2365 : schedule();
1872 : 2354 : goto out;
1873 : 21996 : default:
1874 : : /*
1875 : : * Another bit of PARANOID. Note that the retval will be
1876 : : * 0 since no piece of kernel is supposed to do a check
1877 : : * for a negative retval of schedule_timeout() (since it
1878 : : * should never happens anyway). You just have the printk()
1879 : : * that will tell you if something is gone wrong and where.
1880 : : */
1881 [ - + ]: 21996 : if (timeout < 0) {
1882 : 0 : printk(KERN_ERR "schedule_timeout: wrong timeout "
1883 : : "value %lx\n", timeout);
1884 : 0 : dump_stack();
1885 : 0 : current->state = TASK_RUNNING;
1886 : 0 : goto out;
1887 : : }
1888 : : }
1889 : :
1890 : 21996 : expire = timeout + jiffies;
1891 : :
1892 : 21996 : timer.task = current;
1893 : 21996 : timer_setup_on_stack(&timer.timer, process_timeout, 0);
1894 : 21996 : __mod_timer(&timer.timer, expire, 0);
1895 : 21996 : schedule();
1896 : 21985 : del_singleshot_timer_sync(&timer.timer);
1897 : :
1898 : : /* Remove the timer from the object tracker */
1899 : 21985 : destroy_timer_on_stack(&timer.timer);
1900 : :
1901 : 21985 : timeout = expire - jiffies;
1902 : :
1903 : 24339 : out:
1904 : 24339 : return timeout < 0 ? 0 : timeout;
1905 : : }
1906 : : EXPORT_SYMBOL(schedule_timeout);
1907 : :
1908 : : /*
1909 : : * We can use __set_current_state() here because schedule_timeout() calls
1910 : : * schedule() unconditionally.
1911 : : */
1912 : 0 : signed long __sched schedule_timeout_interruptible(signed long timeout)
1913 : : {
1914 : 0 : __set_current_state(TASK_INTERRUPTIBLE);
1915 : 0 : return schedule_timeout(timeout);
1916 : : }
1917 : : EXPORT_SYMBOL(schedule_timeout_interruptible);
1918 : :
1919 : 0 : signed long __sched schedule_timeout_killable(signed long timeout)
1920 : : {
1921 : 0 : __set_current_state(TASK_KILLABLE);
1922 : 0 : return schedule_timeout(timeout);
1923 : : }
1924 : : EXPORT_SYMBOL(schedule_timeout_killable);
1925 : :
1926 : 22 : signed long __sched schedule_timeout_uninterruptible(signed long timeout)
1927 : : {
1928 : 0 : __set_current_state(TASK_UNINTERRUPTIBLE);
1929 : 22 : return schedule_timeout(timeout);
1930 : : }
1931 : : EXPORT_SYMBOL(schedule_timeout_uninterruptible);
1932 : :
1933 : : /*
1934 : : * Like schedule_timeout_uninterruptible(), except this task will not contribute
1935 : : * to load average.
1936 : : */
1937 : 0 : signed long __sched schedule_timeout_idle(signed long timeout)
1938 : : {
1939 : 0 : __set_current_state(TASK_IDLE);
1940 : 0 : return schedule_timeout(timeout);
1941 : : }
1942 : : EXPORT_SYMBOL(schedule_timeout_idle);
1943 : :
1944 : : #ifdef CONFIG_HOTPLUG_CPU
1945 : 0 : static void migrate_timer_list(struct timer_base *new_base, struct hlist_head *head)
1946 : : {
1947 : 0 : struct timer_list *timer;
1948 : 0 : int cpu = new_base->cpu;
1949 : :
1950 [ # # ]: 0 : while (!hlist_empty(head)) {
1951 : 0 : timer = hlist_entry(head->first, struct timer_list, entry);
1952 : 0 : detach_timer(timer, false);
1953 : 0 : timer->flags = (timer->flags & ~TIMER_BASEMASK) | cpu;
1954 : 0 : internal_add_timer(new_base, timer);
1955 : : }
1956 : 0 : }
1957 : :
1958 : 0 : int timers_prepare_cpu(unsigned int cpu)
1959 : : {
1960 : 0 : struct timer_base *base;
1961 : 0 : int b;
1962 : :
1963 [ # # ]: 0 : for (b = 0; b < NR_BASES; b++) {
1964 : 0 : base = per_cpu_ptr(&timer_bases[b], cpu);
1965 : 0 : base->clk = jiffies;
1966 : 0 : base->next_expiry = base->clk + NEXT_TIMER_MAX_DELTA;
1967 : 0 : base->is_idle = false;
1968 : 0 : base->must_forward_clk = true;
1969 : : }
1970 : 0 : return 0;
1971 : : }
1972 : :
1973 : 0 : int timers_dead_cpu(unsigned int cpu)
1974 : : {
1975 : 0 : struct timer_base *old_base;
1976 : 0 : struct timer_base *new_base;
1977 : 0 : int b, i;
1978 : :
1979 [ # # ]: 0 : BUG_ON(cpu_online(cpu));
1980 : :
1981 [ # # ]: 0 : for (b = 0; b < NR_BASES; b++) {
1982 : 0 : old_base = per_cpu_ptr(&timer_bases[b], cpu);
1983 : 0 : new_base = get_cpu_ptr(&timer_bases[b]);
1984 : : /*
1985 : : * The caller is globally serialized and nobody else
1986 : : * takes two locks at once, deadlock is not possible.
1987 : : */
1988 : 0 : raw_spin_lock_irq(&new_base->lock);
1989 : 0 : raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
1990 : :
1991 : : /*
1992 : : * The current CPUs base clock might be stale. Update it
1993 : : * before moving the timers over.
1994 : : */
1995 [ # # ]: 0 : forward_timer_base(new_base);
1996 : :
1997 [ # # ]: 0 : BUG_ON(old_base->running_timer);
1998 : :
1999 [ # # ]: 0 : for (i = 0; i < WHEEL_SIZE; i++)
2000 : 0 : migrate_timer_list(new_base, old_base->vectors + i);
2001 : :
2002 : 0 : raw_spin_unlock(&old_base->lock);
2003 : 0 : raw_spin_unlock_irq(&new_base->lock);
2004 : 0 : put_cpu_ptr(&timer_bases);
2005 : : }
2006 : 0 : return 0;
2007 : : }
2008 : :
2009 : : #endif /* CONFIG_HOTPLUG_CPU */
2010 : :
2011 : 11 : static void __init init_timer_cpu(int cpu)
2012 : : {
2013 : 11 : struct timer_base *base;
2014 : 11 : int i;
2015 : :
2016 [ + + ]: 33 : for (i = 0; i < NR_BASES; i++) {
2017 : 22 : base = per_cpu_ptr(&timer_bases[i], cpu);
2018 : 22 : base->cpu = cpu;
2019 : 22 : raw_spin_lock_init(&base->lock);
2020 : 22 : base->clk = jiffies;
2021 : 22 : timer_base_init_expiry_lock(base);
2022 : : }
2023 : 11 : }
2024 : :
2025 : 11 : static void __init init_timer_cpus(void)
2026 : : {
2027 : 11 : int cpu;
2028 : :
2029 [ + + ]: 22 : for_each_possible_cpu(cpu)
2030 : 11 : init_timer_cpu(cpu);
2031 : 11 : }
2032 : :
2033 : 11 : void __init init_timers(void)
2034 : : {
2035 : 11 : init_timer_cpus();
2036 : 11 : open_softirq(TIMER_SOFTIRQ, run_timer_softirq);
2037 : 11 : }
2038 : :
2039 : : /**
2040 : : * msleep - sleep safely even with waitqueue interruptions
2041 : : * @msecs: Time in milliseconds to sleep for
2042 : : */
2043 : 22 : void msleep(unsigned int msecs)
2044 : : {
2045 [ - + ]: 22 : unsigned long timeout = msecs_to_jiffies(msecs) + 1;
2046 : :
2047 [ + + ]: 44 : while (timeout)
2048 : 22 : timeout = schedule_timeout_uninterruptible(timeout);
2049 : 22 : }
2050 : :
2051 : : EXPORT_SYMBOL(msleep);
2052 : :
2053 : : /**
2054 : : * msleep_interruptible - sleep waiting for signals
2055 : : * @msecs: Time in milliseconds to sleep for
2056 : : */
2057 : 0 : unsigned long msleep_interruptible(unsigned int msecs)
2058 : : {
2059 [ # # ]: 0 : unsigned long timeout = msecs_to_jiffies(msecs) + 1;
2060 : :
2061 [ # # # # ]: 0 : while (timeout && !signal_pending(current))
2062 : 0 : timeout = schedule_timeout_interruptible(timeout);
2063 : 0 : return jiffies_to_msecs(timeout);
2064 : : }
2065 : :
2066 : : EXPORT_SYMBOL(msleep_interruptible);
2067 : :
2068 : : /**
2069 : : * usleep_range - Sleep for an approximate time
2070 : : * @min: Minimum time in usecs to sleep
2071 : : * @max: Maximum time in usecs to sleep
2072 : : *
2073 : : * In non-atomic context where the exact wakeup time is flexible, use
2074 : : * usleep_range() instead of udelay(). The sleep improves responsiveness
2075 : : * by avoiding the CPU-hogging busy-wait of udelay(), and the range reduces
2076 : : * power usage by allowing hrtimers to take advantage of an already-
2077 : : * scheduled interrupt instead of scheduling a new one just for this sleep.
2078 : : */
2079 : 0 : void __sched usleep_range(unsigned long min, unsigned long max)
2080 : : {
2081 : 0 : ktime_t exp = ktime_add_us(ktime_get(), min);
2082 : 0 : u64 delta = (u64)(max - min) * NSEC_PER_USEC;
2083 : :
2084 : 0 : for (;;) {
2085 : 0 : __set_current_state(TASK_UNINTERRUPTIBLE);
2086 : : /* Do not return before the requested sleep time has elapsed */
2087 [ # # ]: 0 : if (!schedule_hrtimeout_range(&exp, delta, HRTIMER_MODE_ABS))
2088 : : break;
2089 : : }
2090 : 0 : }
2091 : : EXPORT_SYMBOL(usleep_range);
|
