Branch data Line data Source code
1 : : // SPDX-License-Identifier: GPL-2.0
2 : : /*
3 : : * Kernel internal timers
4 : : *
5 : : * Copyright (C) 1991, 1992 Linus Torvalds
6 : : *
7 : : * 1997-01-28 Modified by Finn Arne Gangstad to make timers scale better.
8 : : *
9 : : * 1997-09-10 Updated NTP code according to technical memorandum Jan '96
10 : : * "A Kernel Model for Precision Timekeeping" by Dave Mills
11 : : * 1998-12-24 Fixed a xtime SMP race (we need the xtime_lock rw spinlock to
12 : : * serialize accesses to xtime/lost_ticks).
13 : : * Copyright (C) 1998 Andrea Arcangeli
14 : : * 1999-03-10 Improved NTP compatibility by Ulrich Windl
15 : : * 2002-05-31 Move sys_sysinfo here and make its locking sane, Robert Love
16 : : * 2000-10-05 Implemented scalable SMP per-CPU timer handling.
17 : : * Copyright (C) 2000, 2001, 2002 Ingo Molnar
18 : : * Designed by David S. Miller, Alexey Kuznetsov and Ingo Molnar
19 : : */
20 : :
21 : : #include <linux/kernel_stat.h>
22 : : #include <linux/export.h>
23 : : #include <linux/interrupt.h>
24 : : #include <linux/percpu.h>
25 : : #include <linux/init.h>
26 : : #include <linux/mm.h>
27 : : #include <linux/swap.h>
28 : : #include <linux/pid_namespace.h>
29 : : #include <linux/notifier.h>
30 : : #include <linux/thread_info.h>
31 : : #include <linux/time.h>
32 : : #include <linux/jiffies.h>
33 : : #include <linux/posix-timers.h>
34 : : #include <linux/cpu.h>
35 : : #include <linux/syscalls.h>
36 : : #include <linux/delay.h>
37 : : #include <linux/tick.h>
38 : : #include <linux/kallsyms.h>
39 : : #include <linux/irq_work.h>
40 : : #include <linux/sched/signal.h>
41 : : #include <linux/sched/sysctl.h>
42 : : #include <linux/sched/nohz.h>
43 : : #include <linux/sched/debug.h>
44 : : #include <linux/slab.h>
45 : : #include <linux/compat.h>
46 : : #include <linux/random.h>
47 : :
48 : : #include <linux/uaccess.h>
49 : : #include <asm/unistd.h>
50 : : #include <asm/div64.h>
51 : : #include <asm/timex.h>
52 : : #include <asm/io.h>
53 : :
54 : : #include "tick-internal.h"
55 : :
56 : : #define CREATE_TRACE_POINTS
57 : : #include <trace/events/timer.h>
58 : :
59 : : __visible u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES;
60 : :
61 : : EXPORT_SYMBOL(jiffies_64);
62 : :
63 : : /*
64 : : * The timer wheel has LVL_DEPTH array levels. Each level provides an array of
65 : : * LVL_SIZE buckets. Each level is driven by its own clock and therefor each
66 : : * level has a different granularity.
67 : : *
68 : : * The level granularity is: LVL_CLK_DIV ^ lvl
69 : : * The level clock frequency is: HZ / (LVL_CLK_DIV ^ level)
70 : : *
71 : : * The array level of a newly armed timer depends on the relative expiry
72 : : * time. The farther the expiry time is away the higher the array level and
73 : : * therefor the granularity becomes.
74 : : *
75 : : * Contrary to the original timer wheel implementation, which aims for 'exact'
76 : : * expiry of the timers, this implementation removes the need for recascading
77 : : * the timers into the lower array levels. The previous 'classic' timer wheel
78 : : * implementation of the kernel already violated the 'exact' expiry by adding
79 : : * slack to the expiry time to provide batched expiration. The granularity
80 : : * levels provide implicit batching.
81 : : *
82 : : * This is an optimization of the original timer wheel implementation for the
83 : : * majority of the timer wheel use cases: timeouts. The vast majority of
84 : : * timeout timers (networking, disk I/O ...) are canceled before expiry. If
85 : : * the timeout expires it indicates that normal operation is disturbed, so it
86 : : * does not matter much whether the timeout comes with a slight delay.
87 : : *
88 : : * The only exception to this are networking timers with a small expiry
89 : : * time. They rely on the granularity. Those fit into the first wheel level,
90 : : * which has HZ granularity.
91 : : *
92 : : * We don't have cascading anymore. timers with a expiry time above the
93 : : * capacity of the last wheel level are force expired at the maximum timeout
94 : : * value of the last wheel level. From data sampling we know that the maximum
95 : : * value observed is 5 days (network connection tracking), so this should not
96 : : * be an issue.
97 : : *
98 : : * The currently chosen array constants values are a good compromise between
99 : : * array size and granularity.
100 : : *
101 : : * This results in the following granularity and range levels:
102 : : *
103 : : * HZ 1000 steps
104 : : * Level Offset Granularity Range
105 : : * 0 0 1 ms 0 ms - 63 ms
106 : : * 1 64 8 ms 64 ms - 511 ms
107 : : * 2 128 64 ms 512 ms - 4095 ms (512ms - ~4s)
108 : : * 3 192 512 ms 4096 ms - 32767 ms (~4s - ~32s)
109 : : * 4 256 4096 ms (~4s) 32768 ms - 262143 ms (~32s - ~4m)
110 : : * 5 320 32768 ms (~32s) 262144 ms - 2097151 ms (~4m - ~34m)
111 : : * 6 384 262144 ms (~4m) 2097152 ms - 16777215 ms (~34m - ~4h)
112 : : * 7 448 2097152 ms (~34m) 16777216 ms - 134217727 ms (~4h - ~1d)
113 : : * 8 512 16777216 ms (~4h) 134217728 ms - 1073741822 ms (~1d - ~12d)
114 : : *
115 : : * HZ 300
116 : : * Level Offset Granularity Range
117 : : * 0 0 3 ms 0 ms - 210 ms
118 : : * 1 64 26 ms 213 ms - 1703 ms (213ms - ~1s)
119 : : * 2 128 213 ms 1706 ms - 13650 ms (~1s - ~13s)
120 : : * 3 192 1706 ms (~1s) 13653 ms - 109223 ms (~13s - ~1m)
121 : : * 4 256 13653 ms (~13s) 109226 ms - 873810 ms (~1m - ~14m)
122 : : * 5 320 109226 ms (~1m) 873813 ms - 6990503 ms (~14m - ~1h)
123 : : * 6 384 873813 ms (~14m) 6990506 ms - 55924050 ms (~1h - ~15h)
124 : : * 7 448 6990506 ms (~1h) 55924053 ms - 447392423 ms (~15h - ~5d)
125 : : * 8 512 55924053 ms (~15h) 447392426 ms - 3579139406 ms (~5d - ~41d)
126 : : *
127 : : * HZ 250
128 : : * Level Offset Granularity Range
129 : : * 0 0 4 ms 0 ms - 255 ms
130 : : * 1 64 32 ms 256 ms - 2047 ms (256ms - ~2s)
131 : : * 2 128 256 ms 2048 ms - 16383 ms (~2s - ~16s)
132 : : * 3 192 2048 ms (~2s) 16384 ms - 131071 ms (~16s - ~2m)
133 : : * 4 256 16384 ms (~16s) 131072 ms - 1048575 ms (~2m - ~17m)
134 : : * 5 320 131072 ms (~2m) 1048576 ms - 8388607 ms (~17m - ~2h)
135 : : * 6 384 1048576 ms (~17m) 8388608 ms - 67108863 ms (~2h - ~18h)
136 : : * 7 448 8388608 ms (~2h) 67108864 ms - 536870911 ms (~18h - ~6d)
137 : : * 8 512 67108864 ms (~18h) 536870912 ms - 4294967288 ms (~6d - ~49d)
138 : : *
139 : : * HZ 100
140 : : * Level Offset Granularity Range
141 : : * 0 0 10 ms 0 ms - 630 ms
142 : : * 1 64 80 ms 640 ms - 5110 ms (640ms - ~5s)
143 : : * 2 128 640 ms 5120 ms - 40950 ms (~5s - ~40s)
144 : : * 3 192 5120 ms (~5s) 40960 ms - 327670 ms (~40s - ~5m)
145 : : * 4 256 40960 ms (~40s) 327680 ms - 2621430 ms (~5m - ~43m)
146 : : * 5 320 327680 ms (~5m) 2621440 ms - 20971510 ms (~43m - ~5h)
147 : : * 6 384 2621440 ms (~43m) 20971520 ms - 167772150 ms (~5h - ~1d)
148 : : * 7 448 20971520 ms (~5h) 167772160 ms - 1342177270 ms (~1d - ~15d)
149 : : */
150 : :
151 : : /* Clock divisor for the next level */
152 : : #define LVL_CLK_SHIFT 3
153 : : #define LVL_CLK_DIV (1UL << LVL_CLK_SHIFT)
154 : : #define LVL_CLK_MASK (LVL_CLK_DIV - 1)
155 : : #define LVL_SHIFT(n) ((n) * LVL_CLK_SHIFT)
156 : : #define LVL_GRAN(n) (1UL << LVL_SHIFT(n))
157 : :
158 : : /*
159 : : * The time start value for each level to select the bucket at enqueue
160 : : * time.
161 : : */
162 : : #define LVL_START(n) ((LVL_SIZE - 1) << (((n) - 1) * LVL_CLK_SHIFT))
163 : :
164 : : /* Size of each clock level */
165 : : #define LVL_BITS 6
166 : : #define LVL_SIZE (1UL << LVL_BITS)
167 : : #define LVL_MASK (LVL_SIZE - 1)
168 : : #define LVL_OFFS(n) ((n) * LVL_SIZE)
169 : :
170 : : /* Level depth */
171 : : #if HZ > 100
172 : : # define LVL_DEPTH 9
173 : : # else
174 : : # define LVL_DEPTH 8
175 : : #endif
176 : :
177 : : /* The cutoff (max. capacity of the wheel) */
178 : : #define WHEEL_TIMEOUT_CUTOFF (LVL_START(LVL_DEPTH))
179 : : #define WHEEL_TIMEOUT_MAX (WHEEL_TIMEOUT_CUTOFF - LVL_GRAN(LVL_DEPTH - 1))
180 : :
181 : : /*
182 : : * The resulting wheel size. If NOHZ is configured we allocate two
183 : : * wheels so we have a separate storage for the deferrable timers.
184 : : */
185 : : #define WHEEL_SIZE (LVL_SIZE * LVL_DEPTH)
186 : :
187 : : #ifdef CONFIG_NO_HZ_COMMON
188 : : # define NR_BASES 2
189 : : # define BASE_STD 0
190 : : # define BASE_DEF 1
191 : : #else
192 : : # define NR_BASES 1
193 : : # define BASE_STD 0
194 : : # define BASE_DEF 0
195 : : #endif
196 : :
197 : : struct timer_base {
198 : : raw_spinlock_t lock;
199 : : struct timer_list *running_timer;
200 : : #ifdef CONFIG_PREEMPT_RT
201 : : spinlock_t expiry_lock;
202 : : atomic_t timer_waiters;
203 : : #endif
204 : : unsigned long clk;
205 : : unsigned long next_expiry;
206 : : unsigned int cpu;
207 : : bool is_idle;
208 : : bool must_forward_clk;
209 : : DECLARE_BITMAP(pending_map, WHEEL_SIZE);
210 : : struct hlist_head vectors[WHEEL_SIZE];
211 : : } ____cacheline_aligned;
212 : :
213 : : static DEFINE_PER_CPU(struct timer_base, timer_bases[NR_BASES]);
214 : :
215 : : #ifdef CONFIG_NO_HZ_COMMON
216 : :
217 : : static DEFINE_STATIC_KEY_FALSE(timers_nohz_active);
218 : : static DEFINE_MUTEX(timer_keys_mutex);
219 : :
220 : : static void timer_update_keys(struct work_struct *work);
221 : : static DECLARE_WORK(timer_update_work, timer_update_keys);
222 : :
223 : : #ifdef CONFIG_SMP
224 : : unsigned int sysctl_timer_migration = 1;
225 : :
226 : : DEFINE_STATIC_KEY_FALSE(timers_migration_enabled);
227 : :
228 : 404 : static void timers_update_migration(void)
229 : : {
230 [ + - + - ]: 404 : if (sysctl_timer_migration && tick_nohz_active)
231 : 404 : static_branch_enable(&timers_migration_enabled);
232 : : else
233 : 0 : static_branch_disable(&timers_migration_enabled);
234 : 404 : }
235 : : #else
236 : : static inline void timers_update_migration(void) { }
237 : : #endif /* !CONFIG_SMP */
238 : :
239 : 404 : static void timer_update_keys(struct work_struct *work)
240 : : {
241 : 404 : mutex_lock(&timer_keys_mutex);
242 : 404 : timers_update_migration();
243 : 404 : static_branch_enable(&timers_nohz_active);
244 : 404 : mutex_unlock(&timer_keys_mutex);
245 : 404 : }
246 : :
247 : 404 : void timers_update_nohz(void)
248 : : {
249 : : schedule_work(&timer_update_work);
250 : 404 : }
251 : :
252 : 0 : int timer_migration_handler(struct ctl_table *table, int write,
253 : : void __user *buffer, size_t *lenp,
254 : : loff_t *ppos)
255 : : {
256 : : int ret;
257 : :
258 : 0 : mutex_lock(&timer_keys_mutex);
259 : 0 : ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
260 [ # # ]: 0 : if (!ret && write)
261 : 0 : timers_update_migration();
262 : 0 : mutex_unlock(&timer_keys_mutex);
263 : 0 : return ret;
264 : : }
265 : :
266 : : static inline bool is_timers_nohz_active(void)
267 : : {
268 : 6036964 : return static_branch_unlikely(&timers_nohz_active);
269 : : }
270 : : #else
271 : : static inline bool is_timers_nohz_active(void) { return false; }
272 : : #endif /* NO_HZ_COMMON */
273 : :
274 : : static unsigned long round_jiffies_common(unsigned long j, int cpu,
275 : : bool force_up)
276 : : {
277 : : int rem;
278 : : unsigned long original = j;
279 : :
280 : : /*
281 : : * We don't want all cpus firing their timers at once hitting the
282 : : * same lock or cachelines, so we skew each extra cpu with an extra
283 : : * 3 jiffies. This 3 jiffies came originally from the mm/ code which
284 : : * already did this.
285 : : * The skew is done by adding 3*cpunr, then round, then subtract this
286 : : * extra offset again.
287 : : */
288 : 5991136 : j += cpu * 3;
289 : :
290 : 5991136 : rem = j % HZ;
291 : :
292 : : /*
293 : : * If the target jiffie is just after a whole second (which can happen
294 : : * due to delays of the timer irq, long irq off times etc etc) then
295 : : * we should round down to the whole second, not up. Use 1/4th second
296 : : * as cutoff for this rounding as an extreme upper bound for this.
297 : : * But never round down if @force_up is set.
298 : : */
299 [ + + + + : 184194 : if (rem < HZ/4 && !force_up) /* round down */
# # ]
300 : 157112 : j = j - rem;
301 : : else /* round up */
302 : 5834024 : j = j - rem + HZ;
303 : :
304 : : /* now that we have rounded, subtract the extra skew again */
305 : 5991136 : j -= cpu * 3;
306 : :
307 : : /*
308 : : * Make sure j is still in the future. Otherwise return the
309 : : * unmodified value.
310 : : */
311 [ # # - + : 5991136 : return time_is_after_jiffies(j) ? j : original;
# # # # -
+ - + #
# ]
312 : : }
313 : :
314 : : /**
315 : : * __round_jiffies - function to round jiffies to a full second
316 : : * @j: the time in (absolute) jiffies that should be rounded
317 : : * @cpu: the processor number on which the timeout will happen
318 : : *
319 : : * __round_jiffies() rounds an absolute time in the future (in jiffies)
320 : : * up or down to (approximately) full seconds. This is useful for timers
321 : : * for which the exact time they fire does not matter too much, as long as
322 : : * they fire approximately every X seconds.
323 : : *
324 : : * By rounding these timers to whole seconds, all such timers will fire
325 : : * at the same time, rather than at various times spread out. The goal
326 : : * of this is to have the CPU wake up less, which saves power.
327 : : *
328 : : * The exact rounding is skewed for each processor to avoid all
329 : : * processors firing at the exact same time, which could lead
330 : : * to lock contention or spurious cache line bouncing.
331 : : *
332 : : * The return value is the rounded version of the @j parameter.
333 : : */
334 : 0 : unsigned long __round_jiffies(unsigned long j, int cpu)
335 : : {
336 : 0 : return round_jiffies_common(j, cpu, false);
337 : : }
338 : : EXPORT_SYMBOL_GPL(__round_jiffies);
339 : :
340 : : /**
341 : : * __round_jiffies_relative - function to round jiffies to a full second
342 : : * @j: the time in (relative) jiffies that should be rounded
343 : : * @cpu: the processor number on which the timeout will happen
344 : : *
345 : : * __round_jiffies_relative() rounds a time delta in the future (in jiffies)
346 : : * up or down to (approximately) full seconds. This is useful for timers
347 : : * for which the exact time they fire does not matter too much, as long as
348 : : * they fire approximately every X seconds.
349 : : *
350 : : * By rounding these timers to whole seconds, all such timers will fire
351 : : * at the same time, rather than at various times spread out. The goal
352 : : * of this is to have the CPU wake up less, which saves power.
353 : : *
354 : : * The exact rounding is skewed for each processor to avoid all
355 : : * processors firing at the exact same time, which could lead
356 : : * to lock contention or spurious cache line bouncing.
357 : : *
358 : : * The return value is the rounded version of the @j parameter.
359 : : */
360 : 179268 : unsigned long __round_jiffies_relative(unsigned long j, int cpu)
361 : : {
362 : 179268 : unsigned long j0 = jiffies;
363 : :
364 : : /* Use j0 because jiffies might change while we run */
365 : 358536 : return round_jiffies_common(j + j0, cpu, false) - j0;
366 : : }
367 : : EXPORT_SYMBOL_GPL(__round_jiffies_relative);
368 : :
369 : : /**
370 : : * round_jiffies - function to round jiffies to a full second
371 : : * @j: the time in (absolute) jiffies that should be rounded
372 : : *
373 : : * round_jiffies() rounds an absolute time in the future (in jiffies)
374 : : * up or down to (approximately) full seconds. This is useful for timers
375 : : * for which the exact time they fire does not matter too much, as long as
376 : : * they fire approximately every X seconds.
377 : : *
378 : : * By rounding these timers to whole seconds, all such timers will fire
379 : : * at the same time, rather than at various times spread out. The goal
380 : : * of this is to have the CPU wake up less, which saves power.
381 : : *
382 : : * The return value is the rounded version of the @j parameter.
383 : : */
384 : 4926 : unsigned long round_jiffies(unsigned long j)
385 : : {
386 : 9852 : return round_jiffies_common(j, raw_smp_processor_id(), false);
387 : : }
388 : : EXPORT_SYMBOL_GPL(round_jiffies);
389 : :
390 : : /**
391 : : * round_jiffies_relative - function to round jiffies to a full second
392 : : * @j: the time in (relative) jiffies that should be rounded
393 : : *
394 : : * round_jiffies_relative() rounds a time delta in the future (in jiffies)
395 : : * up or down to (approximately) full seconds. This is useful for timers
396 : : * for which the exact time they fire does not matter too much, as long as
397 : : * they fire approximately every X seconds.
398 : : *
399 : : * By rounding these timers to whole seconds, all such timers will fire
400 : : * at the same time, rather than at various times spread out. The goal
401 : : * of this is to have the CPU wake up less, which saves power.
402 : : *
403 : : * The return value is the rounded version of the @j parameter.
404 : : */
405 : 179300 : unsigned long round_jiffies_relative(unsigned long j)
406 : : {
407 : 179300 : return __round_jiffies_relative(j, raw_smp_processor_id());
408 : : }
409 : : EXPORT_SYMBOL_GPL(round_jiffies_relative);
410 : :
411 : : /**
412 : : * __round_jiffies_up - function to round jiffies up to a full second
413 : : * @j: the time in (absolute) jiffies that should be rounded
414 : : * @cpu: the processor number on which the timeout will happen
415 : : *
416 : : * This is the same as __round_jiffies() except that it will never
417 : : * round down. This is useful for timeouts for which the exact time
418 : : * of firing does not matter too much, as long as they don't fire too
419 : : * early.
420 : : */
421 : 0 : unsigned long __round_jiffies_up(unsigned long j, int cpu)
422 : : {
423 : 0 : return round_jiffies_common(j, cpu, true);
424 : : }
425 : : EXPORT_SYMBOL_GPL(__round_jiffies_up);
426 : :
427 : : /**
428 : : * __round_jiffies_up_relative - function to round jiffies up to a full second
429 : : * @j: the time in (relative) jiffies that should be rounded
430 : : * @cpu: the processor number on which the timeout will happen
431 : : *
432 : : * This is the same as __round_jiffies_relative() except that it will never
433 : : * round down. This is useful for timeouts for which the exact time
434 : : * of firing does not matter too much, as long as they don't fire too
435 : : * early.
436 : : */
437 : 0 : unsigned long __round_jiffies_up_relative(unsigned long j, int cpu)
438 : : {
439 : 0 : unsigned long j0 = jiffies;
440 : :
441 : : /* Use j0 because jiffies might change while we run */
442 : 0 : return round_jiffies_common(j + j0, cpu, true) - j0;
443 : : }
444 : : EXPORT_SYMBOL_GPL(__round_jiffies_up_relative);
445 : :
446 : : /**
447 : : * round_jiffies_up - function to round jiffies up to a full second
448 : : * @j: the time in (absolute) jiffies that should be rounded
449 : : *
450 : : * This is the same as round_jiffies() except that it will never
451 : : * round down. This is useful for timeouts for which the exact time
452 : : * of firing does not matter too much, as long as they don't fire too
453 : : * early.
454 : : */
455 : 5806942 : unsigned long round_jiffies_up(unsigned long j)
456 : : {
457 : 11613884 : return round_jiffies_common(j, raw_smp_processor_id(), true);
458 : : }
459 : : EXPORT_SYMBOL_GPL(round_jiffies_up);
460 : :
461 : : /**
462 : : * round_jiffies_up_relative - function to round jiffies up to a full second
463 : : * @j: the time in (relative) jiffies that should be rounded
464 : : *
465 : : * This is the same as round_jiffies_relative() except that it will never
466 : : * round down. This is useful for timeouts for which the exact time
467 : : * of firing does not matter too much, as long as they don't fire too
468 : : * early.
469 : : */
470 : 0 : unsigned long round_jiffies_up_relative(unsigned long j)
471 : : {
472 : 0 : return __round_jiffies_up_relative(j, raw_smp_processor_id());
473 : : }
474 : : EXPORT_SYMBOL_GPL(round_jiffies_up_relative);
475 : :
476 : :
477 : : static inline unsigned int timer_get_idx(struct timer_list *timer)
478 : : {
479 : 11931368 : return (timer->flags & TIMER_ARRAYMASK) >> TIMER_ARRAYSHIFT;
480 : : }
481 : :
482 : : static inline void timer_set_idx(struct timer_list *timer, unsigned int idx)
483 : : {
484 : 12074192 : timer->flags = (timer->flags & ~TIMER_ARRAYMASK) |
485 : 6037096 : idx << TIMER_ARRAYSHIFT;
486 : : }
487 : :
488 : : /*
489 : : * Helper function to calculate the array index for a given expiry
490 : : * time.
491 : : */
492 : : static inline unsigned calc_index(unsigned expires, unsigned lvl)
493 : : {
494 : 6643950 : expires = (expires + LVL_GRAN(lvl)) >> LVL_SHIFT(lvl);
495 : 6643950 : return LVL_OFFS(lvl) + (expires & LVL_MASK);
496 : : }
497 : :
498 : 6644536 : static int calc_wheel_index(unsigned long expires, unsigned long clk)
499 : : {
500 : 6644536 : unsigned long delta = expires - clk;
501 : : unsigned int idx;
502 : :
503 [ + + ]: 6644536 : if (delta < LVL_START(1)) {
504 : : idx = calc_index(expires, 0);
505 [ + + ]: 4028664 : } else if (delta < LVL_START(2)) {
506 : : idx = calc_index(expires, 1);
507 [ + + ]: 3710680 : } else if (delta < LVL_START(3)) {
508 : : idx = calc_index(expires, 2);
509 [ + + ]: 24982 : } else if (delta < LVL_START(4)) {
510 : : idx = calc_index(expires, 3);
511 [ - + ]: 11478 : } else if (delta < LVL_START(5)) {
512 : : idx = calc_index(expires, 4);
513 [ + + ]: 11478 : } else if (delta < LVL_START(6)) {
514 : : idx = calc_index(expires, 5);
515 [ + + ]: 996 : } else if (delta < LVL_START(7)) {
516 : : idx = calc_index(expires, 6);
517 : : } else if (LVL_DEPTH > 8 && delta < LVL_START(8)) {
518 : : idx = calc_index(expires, 7);
519 [ + - ]: 586 : } else if ((long) delta < 0) {
520 : 586 : idx = clk & LVL_MASK;
521 : : } else {
522 : : /*
523 : : * Force expire obscene large timeouts to expire at the
524 : : * capacity limit of the wheel.
525 : : */
526 [ # # ]: 0 : if (delta >= WHEEL_TIMEOUT_CUTOFF)
527 : 0 : expires = clk + WHEEL_TIMEOUT_MAX;
528 : :
529 : : idx = calc_index(expires, LVL_DEPTH - 1);
530 : : }
531 : 6644536 : return idx;
532 : : }
533 : :
534 : : /*
535 : : * Enqueue the timer into the hash bucket, mark it pending in
536 : : * the bitmap and store the index in the timer flags.
537 : : */
538 : 6037096 : static void enqueue_timer(struct timer_base *base, struct timer_list *timer,
539 : : unsigned int idx)
540 : : {
541 : 6037096 : hlist_add_head(&timer->entry, base->vectors + idx);
542 : 6037096 : __set_bit(idx, base->pending_map);
543 : : timer_set_idx(timer, idx);
544 : :
545 : 6037096 : trace_timer_start(timer, timer->expires, timer->flags);
546 : 6037128 : }
547 : :
548 : : static void
549 : 6023754 : __internal_add_timer(struct timer_base *base, struct timer_list *timer)
550 : : {
551 : : unsigned int idx;
552 : :
553 : 6023754 : idx = calc_wheel_index(timer->expires, base->clk);
554 : 6023762 : enqueue_timer(base, timer, idx);
555 : 6023586 : }
556 : :
557 : : static void
558 : 6037252 : trigger_dyntick_cpu(struct timer_base *base, struct timer_list *timer)
559 : : {
560 [ + + ]: 6036964 : if (!is_timers_nohz_active())
561 : : return;
562 : :
563 : : /*
564 : : * TODO: This wants some optimizing similar to the code below, but we
565 : : * will do that when we switch from push to pull for deferrable timers.
566 : : */
567 [ + + ]: 6027660 : if (timer->flags & TIMER_DEFERRABLE) {
568 : : if (tick_nohz_full_cpu(base->cpu))
569 : : wake_up_nohz_cpu(base->cpu);
570 : : return;
571 : : }
572 : :
573 : : /*
574 : : * We might have to IPI the remote CPU if the base is idle and the
575 : : * timer is not deferrable. If the other CPU is on the way to idle
576 : : * then it can't set base->is_idle as we hold the base lock:
577 : : */
578 [ + + ]: 5825110 : if (!base->is_idle)
579 : : return;
580 : :
581 : : /* Check whether this is the new first expiring timer: */
582 [ + + ]: 20512 : if (time_after_eq(timer->expires, base->next_expiry))
583 : : return;
584 : :
585 : : /*
586 : : * Set the next expiry time and kick the CPU so it can reevaluate the
587 : : * wheel:
588 : : */
589 [ - + ]: 372 : if (time_before(timer->expires, base->clk)) {
590 : : /*
591 : : * Prevent from forward_timer_base() moving the base->clk
592 : : * backward
593 : : */
594 : 0 : base->next_expiry = base->clk;
595 : : } else {
596 : 372 : base->next_expiry = timer->expires;
597 : : }
598 : 372 : wake_up_nohz_cpu(base->cpu);
599 : : }
600 : :
601 : : static void
602 : : internal_add_timer(struct timer_base *base, struct timer_list *timer)
603 : : {
604 : 6023652 : __internal_add_timer(base, timer);
605 : 6023736 : trigger_dyntick_cpu(base, timer);
606 : : }
607 : :
608 : : #ifdef CONFIG_DEBUG_OBJECTS_TIMERS
609 : :
610 : : static struct debug_obj_descr timer_debug_descr;
611 : :
612 : : static void *timer_debug_hint(void *addr)
613 : : {
614 : : return ((struct timer_list *) addr)->function;
615 : : }
616 : :
617 : : static bool timer_is_static_object(void *addr)
618 : : {
619 : : struct timer_list *timer = addr;
620 : :
621 : : return (timer->entry.pprev == NULL &&
622 : : timer->entry.next == TIMER_ENTRY_STATIC);
623 : : }
624 : :
625 : : /*
626 : : * fixup_init is called when:
627 : : * - an active object is initialized
628 : : */
629 : : static bool timer_fixup_init(void *addr, enum debug_obj_state state)
630 : : {
631 : : struct timer_list *timer = addr;
632 : :
633 : : switch (state) {
634 : : case ODEBUG_STATE_ACTIVE:
635 : : del_timer_sync(timer);
636 : : debug_object_init(timer, &timer_debug_descr);
637 : : return true;
638 : : default:
639 : : return false;
640 : : }
641 : : }
642 : :
643 : : /* Stub timer callback for improperly used timers. */
644 : : static void stub_timer(struct timer_list *unused)
645 : : {
646 : : WARN_ON(1);
647 : : }
648 : :
649 : : /*
650 : : * fixup_activate is called when:
651 : : * - an active object is activated
652 : : * - an unknown non-static object is activated
653 : : */
654 : : static bool timer_fixup_activate(void *addr, enum debug_obj_state state)
655 : : {
656 : : struct timer_list *timer = addr;
657 : :
658 : : switch (state) {
659 : : case ODEBUG_STATE_NOTAVAILABLE:
660 : : timer_setup(timer, stub_timer, 0);
661 : : return true;
662 : :
663 : : case ODEBUG_STATE_ACTIVE:
664 : : WARN_ON(1);
665 : : /* fall through */
666 : : default:
667 : : return false;
668 : : }
669 : : }
670 : :
671 : : /*
672 : : * fixup_free is called when:
673 : : * - an active object is freed
674 : : */
675 : : static bool timer_fixup_free(void *addr, enum debug_obj_state state)
676 : : {
677 : : struct timer_list *timer = addr;
678 : :
679 : : switch (state) {
680 : : case ODEBUG_STATE_ACTIVE:
681 : : del_timer_sync(timer);
682 : : debug_object_free(timer, &timer_debug_descr);
683 : : return true;
684 : : default:
685 : : return false;
686 : : }
687 : : }
688 : :
689 : : /*
690 : : * fixup_assert_init is called when:
691 : : * - an untracked/uninit-ed object is found
692 : : */
693 : : static bool timer_fixup_assert_init(void *addr, enum debug_obj_state state)
694 : : {
695 : : struct timer_list *timer = addr;
696 : :
697 : : switch (state) {
698 : : case ODEBUG_STATE_NOTAVAILABLE:
699 : : timer_setup(timer, stub_timer, 0);
700 : : return true;
701 : : default:
702 : : return false;
703 : : }
704 : : }
705 : :
706 : : static struct debug_obj_descr timer_debug_descr = {
707 : : .name = "timer_list",
708 : : .debug_hint = timer_debug_hint,
709 : : .is_static_object = timer_is_static_object,
710 : : .fixup_init = timer_fixup_init,
711 : : .fixup_activate = timer_fixup_activate,
712 : : .fixup_free = timer_fixup_free,
713 : : .fixup_assert_init = timer_fixup_assert_init,
714 : : };
715 : :
716 : : static inline void debug_timer_init(struct timer_list *timer)
717 : : {
718 : : debug_object_init(timer, &timer_debug_descr);
719 : : }
720 : :
721 : : static inline void debug_timer_activate(struct timer_list *timer)
722 : : {
723 : : debug_object_activate(timer, &timer_debug_descr);
724 : : }
725 : :
726 : : static inline void debug_timer_deactivate(struct timer_list *timer)
727 : : {
728 : : debug_object_deactivate(timer, &timer_debug_descr);
729 : : }
730 : :
731 : : static inline void debug_timer_free(struct timer_list *timer)
732 : : {
733 : : debug_object_free(timer, &timer_debug_descr);
734 : : }
735 : :
736 : : static inline void debug_timer_assert_init(struct timer_list *timer)
737 : : {
738 : : debug_object_assert_init(timer, &timer_debug_descr);
739 : : }
740 : :
741 : : static void do_init_timer(struct timer_list *timer,
742 : : void (*func)(struct timer_list *),
743 : : unsigned int flags,
744 : : const char *name, struct lock_class_key *key);
745 : :
746 : : void init_timer_on_stack_key(struct timer_list *timer,
747 : : void (*func)(struct timer_list *),
748 : : unsigned int flags,
749 : : const char *name, struct lock_class_key *key)
750 : : {
751 : : debug_object_init_on_stack(timer, &timer_debug_descr);
752 : : do_init_timer(timer, func, flags, name, key);
753 : : }
754 : : EXPORT_SYMBOL_GPL(init_timer_on_stack_key);
755 : :
756 : : void destroy_timer_on_stack(struct timer_list *timer)
757 : : {
758 : : debug_object_free(timer, &timer_debug_descr);
759 : : }
760 : : EXPORT_SYMBOL_GPL(destroy_timer_on_stack);
761 : :
762 : : #else
763 : : static inline void debug_timer_init(struct timer_list *timer) { }
764 : : static inline void debug_timer_activate(struct timer_list *timer) { }
765 : : static inline void debug_timer_deactivate(struct timer_list *timer) { }
766 : : static inline void debug_timer_assert_init(struct timer_list *timer) { }
767 : : #endif
768 : :
769 : : static inline void debug_init(struct timer_list *timer)
770 : : {
771 : : debug_timer_init(timer);
772 : 2927824 : trace_timer_init(timer);
773 : : }
774 : :
775 : : static inline void debug_deactivate(struct timer_list *timer)
776 : : {
777 : : debug_timer_deactivate(timer);
778 : 6022800 : trace_timer_cancel(timer);
779 : : }
780 : :
781 : : static inline void debug_assert_init(struct timer_list *timer)
782 : : {
783 : : debug_timer_assert_init(timer);
784 : : }
785 : :
786 : : static void do_init_timer(struct timer_list *timer,
787 : : void (*func)(struct timer_list *),
788 : : unsigned int flags,
789 : : const char *name, struct lock_class_key *key)
790 : : {
791 : 2927814 : timer->entry.pprev = NULL;
792 : 2927814 : timer->function = func;
793 : 2927814 : timer->flags = flags | raw_smp_processor_id();
794 : : lockdep_init_map(&timer->lockdep_map, name, key, 0);
795 : : }
796 : :
797 : : /**
798 : : * init_timer_key - initialize a timer
799 : : * @timer: the timer to be initialized
800 : : * @func: timer callback function
801 : : * @flags: timer flags
802 : : * @name: name of the timer
803 : : * @key: lockdep class key of the fake lock used for tracking timer
804 : : * sync lock dependencies
805 : : *
806 : : * init_timer_key() must be done to a timer prior calling *any* of the
807 : : * other timer functions.
808 : : */
809 : 605252 : void init_timer_key(struct timer_list *timer,
810 : : void (*func)(struct timer_list *), unsigned int flags,
811 : : const char *name, struct lock_class_key *key)
812 : : {
813 : : debug_init(timer);
814 : : do_init_timer(timer, func, flags, name, key);
815 : 605242 : }
816 : : EXPORT_SYMBOL(init_timer_key);
817 : :
818 : 6022800 : static inline void detach_timer(struct timer_list *timer, bool clear_pending)
819 : : {
820 : : struct hlist_node *entry = &timer->entry;
821 : :
822 : : debug_deactivate(timer);
823 : :
824 : : __hlist_del(entry);
825 [ + + ]: 6023126 : if (clear_pending)
826 : 5999634 : entry->pprev = NULL;
827 : 6023126 : entry->next = LIST_POISON2;
828 : 6023126 : }
829 : :
830 : 11310558 : static int detach_if_pending(struct timer_list *timer, struct timer_base *base,
831 : : bool clear_pending)
832 : : {
833 : : unsigned idx = timer_get_idx(timer);
834 : :
835 [ + + ]: 11310558 : if (!timer_pending(timer))
836 : : return 0;
837 : :
838 [ + + ]: 3502316 : if (hlist_is_singular_node(&timer->entry, base->vectors + idx))
839 : 3489358 : __clear_bit(idx, base->pending_map);
840 : :
841 : 3502316 : detach_timer(timer, clear_pending);
842 : 3502306 : return 1;
843 : : }
844 : :
845 : : static inline struct timer_base *get_timer_cpu_base(u32 tflags, u32 cpu)
846 : : {
847 : 18604740 : struct timer_base *base = per_cpu_ptr(&timer_bases[BASE_STD], cpu);
848 : :
849 : : /*
850 : : * If the timer is deferrable and NO_HZ_COMMON is set then we need
851 : : * to use the deferrable base.
852 : : */
853 [ + + + + : 18604740 : if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && (tflags & TIMER_DEFERRABLE))
+ + ]
854 : 405868 : base = per_cpu_ptr(&timer_bases[BASE_DEF], cpu);
855 : : return base;
856 : : }
857 : :
858 : : static inline struct timer_base *get_timer_this_cpu_base(u32 tflags)
859 : : {
860 : 18552 : struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
861 : :
862 : : /*
863 : : * If the timer is deferrable and NO_HZ_COMMON is set then we need
864 : : * to use the deferrable base.
865 : : */
866 [ + + ]: 9276 : if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && (tflags & TIMER_DEFERRABLE))
867 : 808 : base = this_cpu_ptr(&timer_bases[BASE_DEF]);
868 : : return base;
869 : : }
870 : :
871 : : static inline struct timer_base *get_timer_base(u32 tflags)
872 : : {
873 : 12576838 : return get_timer_cpu_base(tflags, tflags & TIMER_CPUMASK);
874 : : }
875 : :
876 : : static inline struct timer_base *
877 : 5894766 : get_target_base(struct timer_base *base, unsigned tflags)
878 : : {
879 : : #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
880 [ + + + + ]: 11780302 : if (static_branch_likely(&timers_migration_enabled) &&
881 : 5885502 : !(tflags & TIMER_PINNED))
882 : 11771018 : return get_timer_cpu_base(tflags, get_nohz_timer_target());
883 : : #endif
884 : 9276 : return get_timer_this_cpu_base(tflags);
885 : : }
886 : :
887 : 7177520 : static inline void forward_timer_base(struct timer_base *base)
888 : : {
889 : : #ifdef CONFIG_NO_HZ_COMMON
890 : : unsigned long jnow;
891 : :
892 : : /*
893 : : * We only forward the base when we are idle or have just come out of
894 : : * idle (must_forward_clk logic), and have a delta between base clock
895 : : * and jiffies. In the common case, run_timers will take care of it.
896 : : */
897 [ + + ]: 7177520 : if (likely(!base->must_forward_clk))
898 : : return;
899 : :
900 : 466738 : jnow = READ_ONCE(jiffies);
901 : 466738 : base->must_forward_clk = base->is_idle;
902 [ + + ]: 466738 : if ((long)(jnow - base->clk) < 2)
903 : : return;
904 : :
905 : : /*
906 : : * If the next expiry value is > jiffies, then we fast forward to
907 : : * jiffies otherwise we forward to the next expiry value.
908 : : */
909 [ + + ]: 15954 : if (time_after(base->next_expiry, jnow)) {
910 : 15764 : base->clk = jnow;
911 : : } else {
912 [ + - - + : 190 : if (WARN_ON_ONCE(time_before(base->next_expiry, base->clk)))
# # + - ]
913 : : return;
914 : 190 : base->clk = base->next_expiry;
915 : : }
916 : : #endif
917 : : }
918 : :
919 : :
920 : : /*
921 : : * We are using hashed locking: Holding per_cpu(timer_bases[x]).lock means
922 : : * that all timers which are tied to this base are locked, and the base itself
923 : : * is locked too.
924 : : *
925 : : * So __run_timers/migrate_timers can safely modify all timers which could
926 : : * be found in the base->vectors array.
927 : : *
928 : : * When a timer is migrating then the TIMER_MIGRATING flag is set and we need
929 : : * to wait until the migration is done.
930 : : */
931 : 12577088 : static struct timer_base *lock_timer_base(struct timer_list *timer,
932 : : unsigned long *flags)
933 : : __acquires(timer->base->lock)
934 : : {
935 : : for (;;) {
936 : : struct timer_base *base;
937 : : u32 tf;
938 : :
939 : : /*
940 : : * We need to use READ_ONCE() here, otherwise the compiler
941 : : * might re-read @tf between the check for TIMER_MIGRATING
942 : : * and spin_lock().
943 : : */
944 : : tf = READ_ONCE(timer->flags);
945 : :
946 [ + + ]: 12577094 : if (!(tf & TIMER_MIGRATING)) {
947 : : base = get_timer_base(tf);
948 : 12576838 : raw_spin_lock_irqsave(&base->lock, *flags);
949 [ + + ]: 12577166 : if (timer->flags == tf)
950 : 12577160 : return base;
951 : 6 : raw_spin_unlock_irqrestore(&base->lock, *flags);
952 : : }
953 : 302 : cpu_relax();
954 : 6 : }
955 : : }
956 : :
957 : : #define MOD_TIMER_PENDING_ONLY 0x01
958 : : #define MOD_TIMER_REDUCE 0x02
959 : :
960 : : static inline int
961 : 7883780 : __mod_timer(struct timer_list *timer, unsigned long expires, unsigned int options)
962 : : {
963 : : struct timer_base *base, *new_base;
964 : : unsigned int idx = UINT_MAX;
965 : : unsigned long clk = 0, flags;
966 : : int ret = 0;
967 : :
968 [ - + ]: 7883780 : BUG_ON(!timer->function);
969 : :
970 : : /*
971 : : * This is a common optimization triggered by the networking code - if
972 : : * the timer is re-modified to have the same timeout or ends up in the
973 : : * same array bucket then just return:
974 : : */
975 [ + + ]: 7883780 : if (timer_pending(timer)) {
976 : : /*
977 : : * The downside of this optimization is that it can result in
978 : : * larger granularity than you would get from adding a new
979 : : * timer with this expiry.
980 : : */
981 : 2011440 : long diff = timer->expires - expires;
982 : :
983 [ + + ]: 2011440 : if (!diff)
984 : : return 1;
985 [ - + # # ]: 620810 : if (options & MOD_TIMER_REDUCE && diff <= 0)
986 : : return 1;
987 : :
988 : : /*
989 : : * We lock timer base and calculate the bucket index right
990 : : * here. If the timer ends up in the same bucket, then we
991 : : * just update the expiry time and avoid the whole
992 : : * dequeue/enqueue dance.
993 : : */
994 : 620810 : base = lock_timer_base(timer, &flags);
995 : 620810 : forward_timer_base(base);
996 : :
997 [ + - - + ]: 620810 : if (timer_pending(timer) && (options & MOD_TIMER_REDUCE) &&
998 [ # # ]: 0 : time_before_eq(timer->expires, expires)) {
999 : : ret = 1;
1000 : : goto out_unlock;
1001 : : }
1002 : :
1003 : 620810 : clk = base->clk;
1004 : 620810 : idx = calc_wheel_index(expires, clk);
1005 : :
1006 : : /*
1007 : : * Retrieve and compare the array index of the pending
1008 : : * timer. If it matches set the expiry to the new value so a
1009 : : * subsequent call will exit in the expires check above.
1010 : : */
1011 [ + + ]: 620810 : if (idx == timer_get_idx(timer)) {
1012 [ + - ]: 598398 : if (!(options & MOD_TIMER_REDUCE))
1013 : 598398 : timer->expires = expires;
1014 [ # # ]: 0 : else if (time_after(timer->expires, expires))
1015 : 0 : timer->expires = expires;
1016 : : ret = 1;
1017 : : goto out_unlock;
1018 : : }
1019 : : } else {
1020 : 5872340 : base = lock_timer_base(timer, &flags);
1021 : 5872360 : forward_timer_base(base);
1022 : : }
1023 : :
1024 : 5894790 : ret = detach_if_pending(timer, base, false);
1025 [ + + + + ]: 5894810 : if (!ret && (options & MOD_TIMER_PENDING_ONLY))
1026 : : goto out_unlock;
1027 : :
1028 : 5894798 : new_base = get_target_base(base, timer->flags);
1029 : :
1030 [ + + ]: 5894802 : if (base != new_base) {
1031 : : /*
1032 : : * We are trying to schedule the timer on the new base.
1033 : : * However we can't change timer's base while it is running,
1034 : : * otherwise del_timer_sync() can't detect that the timer's
1035 : : * handler yet has not finished. This also guarantees that the
1036 : : * timer is serialized wrt itself.
1037 : : */
1038 [ + + ]: 598246 : if (likely(base->running_timer != timer)) {
1039 : : /* See the comment in lock_timer_base() */
1040 : 542088 : timer->flags |= TIMER_MIGRATING;
1041 : :
1042 : : raw_spin_unlock(&base->lock);
1043 : : base = new_base;
1044 : 542088 : raw_spin_lock(&base->lock);
1045 : 542088 : WRITE_ONCE(timer->flags,
1046 : : (timer->flags & ~TIMER_BASEMASK) | base->cpu);
1047 : 542088 : forward_timer_base(base);
1048 : : }
1049 : : }
1050 : :
1051 : : debug_timer_activate(timer);
1052 : :
1053 : 5894766 : timer->expires = expires;
1054 : : /*
1055 : : * If 'idx' was calculated above and the base time did not advance
1056 : : * between calculating 'idx' and possibly switching the base, only
1057 : : * enqueue_timer() and trigger_dyntick_cpu() is required. Otherwise
1058 : : * we need to (re)calculate the wheel index via
1059 : : * internal_add_timer().
1060 : : */
1061 [ + + + + ]: 5894766 : if (idx != UINT_MAX && clk == base->clk) {
1062 : 13490 : enqueue_timer(base, timer, idx);
1063 : 13490 : trigger_dyntick_cpu(base, timer);
1064 : : } else {
1065 : : internal_add_timer(base, timer);
1066 : : }
1067 : :
1068 : : out_unlock:
1069 : 6493146 : raw_spin_unlock_irqrestore(&base->lock, flags);
1070 : :
1071 : 6493156 : return ret;
1072 : : }
1073 : :
1074 : : /**
1075 : : * mod_timer_pending - modify a pending timer's timeout
1076 : : * @timer: the pending timer to be modified
1077 : : * @expires: new timeout in jiffies
1078 : : *
1079 : : * mod_timer_pending() is the same for pending timers as mod_timer(),
1080 : : * but will not re-activate and modify already deleted timers.
1081 : : *
1082 : : * It is useful for unserialized use of timers.
1083 : : */
1084 : 0 : int mod_timer_pending(struct timer_list *timer, unsigned long expires)
1085 : : {
1086 : 0 : return __mod_timer(timer, expires, MOD_TIMER_PENDING_ONLY);
1087 : : }
1088 : : EXPORT_SYMBOL(mod_timer_pending);
1089 : :
1090 : : /**
1091 : : * mod_timer - modify a timer's timeout
1092 : : * @timer: the timer to be modified
1093 : : * @expires: new timeout in jiffies
1094 : : *
1095 : : * mod_timer() is a more efficient way to update the expire field of an
1096 : : * active timer (if the timer is inactive it will be activated)
1097 : : *
1098 : : * mod_timer(timer, expires) is equivalent to:
1099 : : *
1100 : : * del_timer(timer); timer->expires = expires; add_timer(timer);
1101 : : *
1102 : : * Note that if there are multiple unserialized concurrent users of the
1103 : : * same timer, then mod_timer() is the only safe way to modify the timeout,
1104 : : * since add_timer() cannot modify an already running timer.
1105 : : *
1106 : : * The function returns whether it has modified a pending timer or not.
1107 : : * (ie. mod_timer() of an inactive timer returns 0, mod_timer() of an
1108 : : * active timer returns 1.)
1109 : : */
1110 : 5416100 : int mod_timer(struct timer_list *timer, unsigned long expires)
1111 : : {
1112 : 5560714 : return __mod_timer(timer, expires, 0);
1113 : : }
1114 : : EXPORT_SYMBOL(mod_timer);
1115 : :
1116 : : /**
1117 : : * timer_reduce - Modify a timer's timeout if it would reduce the timeout
1118 : : * @timer: The timer to be modified
1119 : : * @expires: New timeout in jiffies
1120 : : *
1121 : : * timer_reduce() is very similar to mod_timer(), except that it will only
1122 : : * modify a running timer if that would reduce the expiration time (it will
1123 : : * start a timer that isn't running).
1124 : : */
1125 : 528 : int timer_reduce(struct timer_list *timer, unsigned long expires)
1126 : : {
1127 : 528 : return __mod_timer(timer, expires, MOD_TIMER_REDUCE);
1128 : : }
1129 : : EXPORT_SYMBOL(timer_reduce);
1130 : :
1131 : : /**
1132 : : * add_timer - start a timer
1133 : : * @timer: the timer to be added
1134 : : *
1135 : : * The kernel will do a ->function(@timer) callback from the
1136 : : * timer interrupt at the ->expires point in the future. The
1137 : : * current time is 'jiffies'.
1138 : : *
1139 : : * The timer's ->expires, ->function fields must be set prior calling this
1140 : : * function.
1141 : : *
1142 : : * Timers with an ->expires field in the past will be executed in the next
1143 : : * timer tick.
1144 : : */
1145 : 144614 : void add_timer(struct timer_list *timer)
1146 : : {
1147 [ - + ]: 144614 : BUG_ON(timer_pending(timer));
1148 : 144614 : mod_timer(timer, timer->expires);
1149 : 144610 : }
1150 : : EXPORT_SYMBOL(add_timer);
1151 : :
1152 : : /**
1153 : : * add_timer_on - start a timer on a particular CPU
1154 : : * @timer: the timer to be added
1155 : : * @cpu: the CPU to start it on
1156 : : *
1157 : : * This is not very scalable on SMP. Double adds are not possible.
1158 : : */
1159 : 142428 : void add_timer_on(struct timer_list *timer, int cpu)
1160 : : {
1161 : : struct timer_base *new_base, *base;
1162 : : unsigned long flags;
1163 : :
1164 [ + + + + ]: 142428 : BUG_ON(timer_pending(timer) || !timer->function);
1165 : :
1166 : 142408 : new_base = get_timer_cpu_base(timer->flags, cpu);
1167 : :
1168 : : /*
1169 : : * If @timer was on a different CPU, it should be migrated with the
1170 : : * old base locked to prevent other operations proceeding with the
1171 : : * wrong base locked. See lock_timer_base().
1172 : : */
1173 : 142408 : base = lock_timer_base(timer, &flags);
1174 [ + + ]: 142470 : if (base != new_base) {
1175 : 1212 : timer->flags |= TIMER_MIGRATING;
1176 : :
1177 : : raw_spin_unlock(&base->lock);
1178 : : base = new_base;
1179 : 1212 : raw_spin_lock(&base->lock);
1180 : 1212 : WRITE_ONCE(timer->flags,
1181 : : (timer->flags & ~TIMER_BASEMASK) | cpu);
1182 : : }
1183 : 142470 : forward_timer_base(base);
1184 : :
1185 : : debug_timer_activate(timer);
1186 : : internal_add_timer(base, timer);
1187 : 142434 : raw_spin_unlock_irqrestore(&base->lock, flags);
1188 : 142414 : }
1189 : : EXPORT_SYMBOL_GPL(add_timer_on);
1190 : :
1191 : : /**
1192 : : * del_timer - deactivate a timer.
1193 : : * @timer: the timer to be deactivated
1194 : : *
1195 : : * del_timer() deactivates a timer - this works on both active and inactive
1196 : : * timers.
1197 : : *
1198 : : * The function returns whether it has deactivated a pending timer or not.
1199 : : * (ie. del_timer() of an inactive timer returns 0, del_timer() of an
1200 : : * active timer returns 1.)
1201 : : */
1202 : 5879800 : int del_timer(struct timer_list *timer)
1203 : : {
1204 : : struct timer_base *base;
1205 : : unsigned long flags;
1206 : : int ret = 0;
1207 : :
1208 : : debug_assert_init(timer);
1209 : :
1210 [ + + ]: 5879800 : if (timer_pending(timer)) {
1211 : 3036436 : base = lock_timer_base(timer, &flags);
1212 : 3036436 : ret = detach_if_pending(timer, base, true);
1213 : 3036436 : raw_spin_unlock_irqrestore(&base->lock, flags);
1214 : : }
1215 : :
1216 : 5879800 : return ret;
1217 : : }
1218 : : EXPORT_SYMBOL(del_timer);
1219 : :
1220 : : /**
1221 : : * try_to_del_timer_sync - Try to deactivate a timer
1222 : : * @timer: timer to delete
1223 : : *
1224 : : * This function tries to deactivate a timer. Upon successful (ret >= 0)
1225 : : * exit the timer is not queued and the handler is not running on any CPU.
1226 : : */
1227 : 2905148 : int try_to_del_timer_sync(struct timer_list *timer)
1228 : : {
1229 : : struct timer_base *base;
1230 : : unsigned long flags;
1231 : : int ret = -1;
1232 : :
1233 : : debug_assert_init(timer);
1234 : :
1235 : 2905148 : base = lock_timer_base(timer, &flags);
1236 : :
1237 [ + + ]: 2905156 : if (base->running_timer != timer)
1238 : 2379406 : ret = detach_if_pending(timer, base, true);
1239 : :
1240 : 2905144 : raw_spin_unlock_irqrestore(&base->lock, flags);
1241 : :
1242 : 2905152 : return ret;
1243 : : }
1244 : : EXPORT_SYMBOL(try_to_del_timer_sync);
1245 : :
1246 : : #ifdef CONFIG_PREEMPT_RT
1247 : : static __init void timer_base_init_expiry_lock(struct timer_base *base)
1248 : : {
1249 : : spin_lock_init(&base->expiry_lock);
1250 : : }
1251 : :
1252 : : static inline void timer_base_lock_expiry(struct timer_base *base)
1253 : : {
1254 : : spin_lock(&base->expiry_lock);
1255 : : }
1256 : :
1257 : : static inline void timer_base_unlock_expiry(struct timer_base *base)
1258 : : {
1259 : : spin_unlock(&base->expiry_lock);
1260 : : }
1261 : :
1262 : : /*
1263 : : * The counterpart to del_timer_wait_running().
1264 : : *
1265 : : * If there is a waiter for base->expiry_lock, then it was waiting for the
1266 : : * timer callback to finish. Drop expiry_lock and reaquire it. That allows
1267 : : * the waiter to acquire the lock and make progress.
1268 : : */
1269 : : static void timer_sync_wait_running(struct timer_base *base)
1270 : : {
1271 : : if (atomic_read(&base->timer_waiters)) {
1272 : : spin_unlock(&base->expiry_lock);
1273 : : spin_lock(&base->expiry_lock);
1274 : : }
1275 : : }
1276 : :
1277 : : /*
1278 : : * This function is called on PREEMPT_RT kernels when the fast path
1279 : : * deletion of a timer failed because the timer callback function was
1280 : : * running.
1281 : : *
1282 : : * This prevents priority inversion, if the softirq thread on a remote CPU
1283 : : * got preempted, and it prevents a life lock when the task which tries to
1284 : : * delete a timer preempted the softirq thread running the timer callback
1285 : : * function.
1286 : : */
1287 : : static void del_timer_wait_running(struct timer_list *timer)
1288 : : {
1289 : : u32 tf;
1290 : :
1291 : : tf = READ_ONCE(timer->flags);
1292 : : if (!(tf & TIMER_MIGRATING)) {
1293 : : struct timer_base *base = get_timer_base(tf);
1294 : :
1295 : : /*
1296 : : * Mark the base as contended and grab the expiry lock,
1297 : : * which is held by the softirq across the timer
1298 : : * callback. Drop the lock immediately so the softirq can
1299 : : * expire the next timer. In theory the timer could already
1300 : : * be running again, but that's more than unlikely and just
1301 : : * causes another wait loop.
1302 : : */
1303 : : atomic_inc(&base->timer_waiters);
1304 : : spin_lock_bh(&base->expiry_lock);
1305 : : atomic_dec(&base->timer_waiters);
1306 : : spin_unlock_bh(&base->expiry_lock);
1307 : : }
1308 : : }
1309 : : #else
1310 : : static inline void timer_base_init_expiry_lock(struct timer_base *base) { }
1311 : : static inline void timer_base_lock_expiry(struct timer_base *base) { }
1312 : : static inline void timer_base_unlock_expiry(struct timer_base *base) { }
1313 : : static inline void timer_sync_wait_running(struct timer_base *base) { }
1314 : : static inline void del_timer_wait_running(struct timer_list *timer) { }
1315 : : #endif
1316 : :
1317 : : #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
1318 : : /**
1319 : : * del_timer_sync - deactivate a timer and wait for the handler to finish.
1320 : : * @timer: the timer to be deactivated
1321 : : *
1322 : : * This function only differs from del_timer() on SMP: besides deactivating
1323 : : * the timer it also makes sure the handler has finished executing on other
1324 : : * CPUs.
1325 : : *
1326 : : * Synchronization rules: Callers must prevent restarting of the timer,
1327 : : * otherwise this function is meaningless. It must not be called from
1328 : : * interrupt contexts unless the timer is an irqsafe one. The caller must
1329 : : * not hold locks which would prevent completion of the timer's
1330 : : * handler. The timer's handler must not call add_timer_on(). Upon exit the
1331 : : * timer is not queued and the handler is not running on any CPU.
1332 : : *
1333 : : * Note: For !irqsafe timers, you must not hold locks that are held in
1334 : : * interrupt context while calling this function. Even if the lock has
1335 : : * nothing to do with the timer in question. Here's why::
1336 : : *
1337 : : * CPU0 CPU1
1338 : : * ---- ----
1339 : : * <SOFTIRQ>
1340 : : * call_timer_fn();
1341 : : * base->running_timer = mytimer;
1342 : : * spin_lock_irq(somelock);
1343 : : * <IRQ>
1344 : : * spin_lock(somelock);
1345 : : * del_timer_sync(mytimer);
1346 : : * while (base->running_timer == mytimer);
1347 : : *
1348 : : * Now del_timer_sync() will never return and never release somelock.
1349 : : * The interrupt on the other CPU is waiting to grab somelock but
1350 : : * it has interrupted the softirq that CPU0 is waiting to finish.
1351 : : *
1352 : : * The function returns whether it has deactivated a pending timer or not.
1353 : : */
1354 : 2379398 : int del_timer_sync(struct timer_list *timer)
1355 : : {
1356 : : int ret;
1357 : :
1358 : : #ifdef CONFIG_LOCKDEP
1359 : : unsigned long flags;
1360 : :
1361 : : /*
1362 : : * If lockdep gives a backtrace here, please reference
1363 : : * the synchronization rules above.
1364 : : */
1365 : : local_irq_save(flags);
1366 : : lock_map_acquire(&timer->lockdep_map);
1367 : : lock_map_release(&timer->lockdep_map);
1368 : : local_irq_restore(flags);
1369 : : #endif
1370 : : /*
1371 : : * don't use it in hardirq context, because it
1372 : : * could lead to deadlock.
1373 : : */
1374 [ - + # # : 2379398 : WARN_ON(in_irq() && !(timer->flags & TIMER_IRQSAFE));
- + ]
1375 : :
1376 : : do {
1377 : 2905148 : ret = try_to_del_timer_sync(timer);
1378 : :
1379 [ + + ]: 2905150 : if (unlikely(ret < 0)) {
1380 : : del_timer_wait_running(timer);
1381 : 525752 : cpu_relax();
1382 : : }
1383 [ + + ]: 2905154 : } while (ret < 0);
1384 : :
1385 : 2379402 : return ret;
1386 : : }
1387 : : EXPORT_SYMBOL(del_timer_sync);
1388 : : #endif
1389 : :
1390 : 2520764 : static void call_timer_fn(struct timer_list *timer,
1391 : : void (*fn)(struct timer_list *),
1392 : : unsigned long baseclk)
1393 : : {
1394 : : int count = preempt_count();
1395 : :
1396 : : #ifdef CONFIG_LOCKDEP
1397 : : /*
1398 : : * It is permissible to free the timer from inside the
1399 : : * function that is called from it, this we need to take into
1400 : : * account for lockdep too. To avoid bogus "held lock freed"
1401 : : * warnings as well as problems when looking into
1402 : : * timer->lockdep_map, make a copy and use that here.
1403 : : */
1404 : : struct lockdep_map lockdep_map;
1405 : :
1406 : : lockdep_copy_map(&lockdep_map, &timer->lockdep_map);
1407 : : #endif
1408 : : /*
1409 : : * Couple the lock chain with the lock chain at
1410 : : * del_timer_sync() by acquiring the lock_map around the fn()
1411 : : * call here and in del_timer_sync().
1412 : : */
1413 : : lock_map_acquire(&lockdep_map);
1414 : :
1415 : 2521168 : trace_timer_expire_entry(timer, baseclk);
1416 : 2520770 : fn(timer);
1417 : 2521464 : trace_timer_expire_exit(timer);
1418 : :
1419 : : lock_map_release(&lockdep_map);
1420 : :
1421 [ - + ]: 2521298 : if (count != preempt_count()) {
1422 [ # # ]: 0 : WARN_ONCE(1, "timer: %pS preempt leak: %08x -> %08x\n",
1423 : : fn, count, preempt_count());
1424 : : /*
1425 : : * Restore the preempt count. That gives us a decent
1426 : : * chance to survive and extract information. If the
1427 : : * callback kept a lock held, bad luck, but not worse
1428 : : * than the BUG() we had.
1429 : : */
1430 : : preempt_count_set(count);
1431 : : }
1432 : 2521298 : }
1433 : :
1434 : 2454686 : static void expire_timers(struct timer_base *base, struct hlist_head *head)
1435 : : {
1436 : : /*
1437 : : * This value is required only for tracing. base->clk was
1438 : : * incremented directly before expire_timers was called. But expiry
1439 : : * is related to the old base->clk value.
1440 : : */
1441 : 2454686 : unsigned long baseclk = base->clk - 1;
1442 : :
1443 [ + + ]: 7431144 : while (!hlist_empty(head)) {
1444 : : struct timer_list *timer;
1445 : : void (*fn)(struct timer_list *);
1446 : :
1447 : 2520184 : timer = hlist_entry(head->first, struct timer_list, entry);
1448 : :
1449 : 2520184 : base->running_timer = timer;
1450 : 2520184 : detach_timer(timer, true);
1451 : :
1452 : 2521198 : fn = timer->function;
1453 : :
1454 [ + + ]: 2521198 : if (timer->flags & TIMER_IRQSAFE) {
1455 : : raw_spin_unlock(&base->lock);
1456 : 266572 : call_timer_fn(timer, fn, baseclk);
1457 : 266494 : base->running_timer = NULL;
1458 : 266494 : raw_spin_lock(&base->lock);
1459 : : } else {
1460 : 2254572 : raw_spin_unlock_irq(&base->lock);
1461 : 2254602 : call_timer_fn(timer, fn, baseclk);
1462 : 2255238 : base->running_timer = NULL;
1463 : : timer_sync_wait_running(base);
1464 : 2255238 : raw_spin_lock_irq(&base->lock);
1465 : : }
1466 : : }
1467 : 2456274 : }
1468 : :
1469 : 30942228 : static int __collect_expired_timers(struct timer_base *base,
1470 : : struct hlist_head *heads)
1471 : : {
1472 : 30942228 : unsigned long clk = base->clk;
1473 : : struct hlist_head *vec;
1474 : : int i, levels = 0;
1475 : : unsigned int idx;
1476 : :
1477 [ + + ]: 35413614 : for (i = 0; i < LVL_DEPTH; i++) {
1478 : 35187178 : idx = (clk & LVL_MASK) + i * LVL_SIZE;
1479 : :
1480 [ + + ]: 70374356 : if (__test_and_clear_bit(idx, base->pending_map)) {
1481 : : vec = base->vectors + idx;
1482 : 2455622 : hlist_move_list(vec, heads++);
1483 : 2455622 : levels++;
1484 : : }
1485 : : /* Is it time to look at the next level? */
1486 [ + + ]: 35187178 : if (clk & LVL_CLK_MASK)
1487 : : break;
1488 : : /* Shift clock for the next level granularity */
1489 : 4471386 : clk >>= LVL_CLK_SHIFT;
1490 : : }
1491 : 30942228 : return levels;
1492 : : }
1493 : :
1494 : : #ifdef CONFIG_NO_HZ_COMMON
1495 : : /*
1496 : : * Find the next pending bucket of a level. Search from level start (@offset)
1497 : : * + @clk upwards and if nothing there, search from start of the level
1498 : : * (@offset) up to @offset + clk.
1499 : : */
1500 : 231490746 : static int next_pending_bucket(struct timer_base *base, unsigned offset,
1501 : : unsigned clk)
1502 : : {
1503 : 231490746 : unsigned pos, start = offset + clk;
1504 : 231490746 : unsigned end = offset + LVL_SIZE;
1505 : :
1506 : 231490746 : pos = find_next_bit(base->pending_map, end, start);
1507 [ + + ]: 231492530 : if (pos < end)
1508 : 61827968 : return pos - start;
1509 : :
1510 : 169664562 : pos = find_next_bit(base->pending_map, start, offset);
1511 [ + + ]: 169666568 : return pos < start ? pos + LVL_SIZE - start : -1;
1512 : : }
1513 : :
1514 : : /*
1515 : : * Search the first expiring timer in the various clock levels. Caller must
1516 : : * hold base->lock.
1517 : : */
1518 : 28937862 : static unsigned long __next_timer_interrupt(struct timer_base *base)
1519 : : {
1520 : : unsigned long clk, next, adj;
1521 : : unsigned lvl, offset = 0;
1522 : :
1523 : 28937862 : next = base->clk + NEXT_TIMER_MAX_DELTA;
1524 : : clk = base->clk;
1525 [ + + ]: 260429394 : for (lvl = 0; lvl < LVL_DEPTH; lvl++, offset += LVL_SIZE) {
1526 : 231492368 : int pos = next_pending_bucket(base, offset, clk & LVL_MASK);
1527 : :
1528 [ + + ]: 231491532 : if (pos >= 0) {
1529 : 71454412 : unsigned long tmp = clk + (unsigned long) pos;
1530 : :
1531 : 71454412 : tmp <<= LVL_SHIFT(lvl);
1532 [ + + ]: 71454412 : if (time_before(tmp, next))
1533 : : next = tmp;
1534 : : }
1535 : : /*
1536 : : * Clock for the next level. If the current level clock lower
1537 : : * bits are zero, we look at the next level as is. If not we
1538 : : * need to advance it by one because that's going to be the
1539 : : * next expiring bucket in that level. base->clk is the next
1540 : : * expiring jiffie. So in case of:
1541 : : *
1542 : : * LVL5 LVL4 LVL3 LVL2 LVL1 LVL0
1543 : : * 0 0 0 0 0 0
1544 : : *
1545 : : * we have to look at all levels @index 0. With
1546 : : *
1547 : : * LVL5 LVL4 LVL3 LVL2 LVL1 LVL0
1548 : : * 0 0 0 0 0 2
1549 : : *
1550 : : * LVL0 has the next expiring bucket @index 2. The upper
1551 : : * levels have the next expiring bucket @index 1.
1552 : : *
1553 : : * In case that the propagation wraps the next level the same
1554 : : * rules apply:
1555 : : *
1556 : : * LVL5 LVL4 LVL3 LVL2 LVL1 LVL0
1557 : : * 0 0 0 0 F 2
1558 : : *
1559 : : * So after looking at LVL0 we get:
1560 : : *
1561 : : * LVL5 LVL4 LVL3 LVL2 LVL1
1562 : : * 0 0 0 1 0
1563 : : *
1564 : : * So no propagation from LVL1 to LVL2 because that happened
1565 : : * with the add already, but then we need to propagate further
1566 : : * from LVL2 to LVL3.
1567 : : *
1568 : : * So the simple check whether the lower bits of the current
1569 : : * level are 0 or not is sufficient for all cases.
1570 : : */
1571 : 231491532 : adj = clk & LVL_CLK_MASK ? 1 : 0;
1572 : 231491532 : clk >>= LVL_CLK_SHIFT;
1573 : 231491532 : clk += adj;
1574 : : }
1575 : 28937026 : return next;
1576 : : }
1577 : :
1578 : : /*
1579 : : * Check, if the next hrtimer event is before the next timer wheel
1580 : : * event:
1581 : : */
1582 : 28402580 : static u64 cmp_next_hrtimer_event(u64 basem, u64 expires)
1583 : : {
1584 : 28402580 : u64 nextevt = hrtimer_get_next_event();
1585 : :
1586 : : /*
1587 : : * If high resolution timers are enabled
1588 : : * hrtimer_get_next_event() returns KTIME_MAX.
1589 : : */
1590 [ - + ]: 28402624 : if (expires <= nextevt)
1591 : : return expires;
1592 : :
1593 : : /*
1594 : : * If the next timer is already expired, return the tick base
1595 : : * time so the tick is fired immediately.
1596 : : */
1597 [ # # ]: 0 : if (nextevt <= basem)
1598 : : return basem;
1599 : :
1600 : : /*
1601 : : * Round up to the next jiffie. High resolution timers are
1602 : : * off, so the hrtimers are expired in the tick and we need to
1603 : : * make sure that this tick really expires the timer to avoid
1604 : : * a ping pong of the nohz stop code.
1605 : : *
1606 : : * Use DIV_ROUND_UP_ULL to prevent gcc calling __divdi3
1607 : : */
1608 : 0 : return DIV_ROUND_UP_ULL(nextevt, TICK_NSEC) * TICK_NSEC;
1609 : : }
1610 : :
1611 : : /**
1612 : : * get_next_timer_interrupt - return the time (clock mono) of the next timer
1613 : : * @basej: base time jiffies
1614 : : * @basem: base time clock monotonic
1615 : : *
1616 : : * Returns the tick aligned clock monotonic time of the next pending
1617 : : * timer or KTIME_MAX if no timer is pending.
1618 : : */
1619 : 28402696 : u64 get_next_timer_interrupt(unsigned long basej, u64 basem)
1620 : : {
1621 : 56805392 : struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
1622 : : u64 expires = KTIME_MAX;
1623 : : unsigned long nextevt;
1624 : : bool is_max_delta;
1625 : :
1626 : : /*
1627 : : * Pretend that there is no timer pending if the cpu is offline.
1628 : : * Possible pending timers will be migrated later to an active cpu.
1629 : : */
1630 [ + + ]: 56805392 : if (cpu_is_offline(smp_processor_id()))
1631 : : return expires;
1632 : :
1633 : 28402652 : raw_spin_lock(&base->lock);
1634 : 28402750 : nextevt = __next_timer_interrupt(base);
1635 : 28402612 : is_max_delta = (nextevt == base->clk + NEXT_TIMER_MAX_DELTA);
1636 : 28402612 : base->next_expiry = nextevt;
1637 : : /*
1638 : : * We have a fresh next event. Check whether we can forward the
1639 : : * base. We can only do that when @basej is past base->clk
1640 : : * otherwise we might rewind base->clk.
1641 : : */
1642 [ + + ]: 28402612 : if (time_after(basej, base->clk)) {
1643 [ + + ]: 1385392 : if (time_after(nextevt, basej))
1644 : 1380226 : base->clk = basej;
1645 [ + + ]: 5166 : else if (time_after(nextevt, base->clk))
1646 : 2094 : base->clk = nextevt;
1647 : : }
1648 : :
1649 [ + + ]: 28402612 : if (time_before_eq(nextevt, basej)) {
1650 : : expires = basem;
1651 : 127802 : base->is_idle = false;
1652 : : } else {
1653 [ + + ]: 28274810 : if (!is_max_delta)
1654 : 26829112 : expires = basem + (u64)(nextevt - basej) * TICK_NSEC;
1655 : : /*
1656 : : * If we expect to sleep more than a tick, mark the base idle.
1657 : : * Also the tick is stopped so any added timer must forward
1658 : : * the base clk itself to keep granularity small. This idle
1659 : : * logic is only maintained for the BASE_STD base, deferrable
1660 : : * timers may still see large granularity skew (by design).
1661 : : */
1662 [ + + ]: 28274810 : if ((expires - basem) > TICK_NSEC) {
1663 : 24373968 : base->must_forward_clk = true;
1664 : 24373968 : base->is_idle = true;
1665 : : }
1666 : : }
1667 : : raw_spin_unlock(&base->lock);
1668 : :
1669 : 28402548 : return cmp_next_hrtimer_event(basem, expires);
1670 : : }
1671 : :
1672 : : /**
1673 : : * timer_clear_idle - Clear the idle state of the timer base
1674 : : *
1675 : : * Called with interrupts disabled
1676 : : */
1677 : 39660578 : void timer_clear_idle(void)
1678 : : {
1679 : 79321156 : struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
1680 : :
1681 : : /*
1682 : : * We do this unlocked. The worst outcome is a remote enqueue sending
1683 : : * a pointless IPI, but taking the lock would just make the window for
1684 : : * sending the IPI a few instructions smaller for the cost of taking
1685 : : * the lock in the exit from idle path.
1686 : : */
1687 : 39660578 : base->is_idle = false;
1688 : 39660578 : }
1689 : :
1690 : 31299742 : static int collect_expired_timers(struct timer_base *base,
1691 : : struct hlist_head *heads)
1692 : : {
1693 : 31299742 : unsigned long now = READ_ONCE(jiffies);
1694 : :
1695 : : /*
1696 : : * NOHZ optimization. After a long idle sleep we need to forward the
1697 : : * base to current jiffies. Avoid a loop by searching the bitfield for
1698 : : * the next expiring timer.
1699 : : */
1700 [ + + ]: 31299742 : if ((long)(now - base->clk) > 2) {
1701 : 534088 : unsigned long next = __next_timer_interrupt(base);
1702 : :
1703 : : /*
1704 : : * If the next timer is ahead of time forward to current
1705 : : * jiffies, otherwise forward to the next expiry time:
1706 : : */
1707 [ + + ]: 534156 : if (time_after(next, now)) {
1708 : : /*
1709 : : * The call site will increment base->clk and then
1710 : : * terminate the expiry loop immediately.
1711 : : */
1712 : 425182 : base->clk = now;
1713 : 425182 : return 0;
1714 : : }
1715 : 108974 : base->clk = next;
1716 : : }
1717 : 30874628 : return __collect_expired_timers(base, heads);
1718 : : }
1719 : : #else
1720 : : static inline int collect_expired_timers(struct timer_base *base,
1721 : : struct hlist_head *heads)
1722 : : {
1723 : : return __collect_expired_timers(base, heads);
1724 : : }
1725 : : #endif
1726 : :
1727 : : /*
1728 : : * Called from the timer interrupt handler to charge one tick to the current
1729 : : * process. user_tick is 1 if the tick is user time, 0 for system.
1730 : : */
1731 : 16990630 : void update_process_times(int user_tick)
1732 : : {
1733 : 16990630 : struct task_struct *p = current;
1734 : :
1735 : : /* Note: this timer irq context must be accounted for as well. */
1736 : 16990630 : account_process_tick(p, user_tick);
1737 : 17018200 : run_local_timers();
1738 : 16999358 : rcu_sched_clock_irq(user_tick);
1739 : : #ifdef CONFIG_IRQ_WORK
1740 [ + - ]: 16786556 : if (in_irq())
1741 : 16999466 : irq_work_tick();
1742 : : #endif
1743 : 16784002 : scheduler_tick();
1744 : : if (IS_ENABLED(CONFIG_POSIX_TIMERS))
1745 : 16981200 : run_posix_cpu_timers();
1746 : :
1747 : : /* The current CPU might make use of net randoms without receiving IRQs
1748 : : * to renew them often enough. Let's update the net_rand_state from a
1749 : : * non-constant value that's not affine to the number of calls to make
1750 : : * sure it's updated when there's some activity (we don't care in idle).
1751 : : */
1752 : 50424168 : this_cpu_add(net_rand_state.s1, rol32(jiffies, 24) + user_tick);
1753 : 17005366 : }
1754 : :
1755 : : /**
1756 : : * __run_timers - run all expired timers (if any) on this CPU.
1757 : : * @base: the timer vector to be processed.
1758 : : */
1759 : 24906920 : static inline void __run_timers(struct timer_base *base)
1760 : : {
1761 : : struct hlist_head heads[LVL_DEPTH];
1762 : : int levels;
1763 : :
1764 [ + + ]: 24906920 : if (!time_after_eq(jiffies, base->clk))
1765 : 37676 : return;
1766 : :
1767 : : timer_base_lock_expiry(base);
1768 : 24869244 : raw_spin_lock_irq(&base->lock);
1769 : :
1770 : : /*
1771 : : * timer_base::must_forward_clk must be cleared before running
1772 : : * timers so that any timer functions that call mod_timer() will
1773 : : * not try to forward the base. Idle tracking / clock forwarding
1774 : : * logic is only used with BASE_STD timers.
1775 : : *
1776 : : * The must_forward_clk flag is cleared unconditionally also for
1777 : : * the deferrable base. The deferrable base is not affected by idle
1778 : : * tracking and never forwarded, so clearing the flag is a NOOP.
1779 : : *
1780 : : * The fact that the deferrable base is never forwarded can cause
1781 : : * large variations in granularity for deferrable timers, but they
1782 : : * can be deferred for long periods due to idle anyway.
1783 : : */
1784 : 24878348 : base->must_forward_clk = false;
1785 : :
1786 [ + + ]: 81122796 : while (time_after_eq(jiffies, base->clk)) {
1787 : :
1788 : 31394846 : levels = collect_expired_timers(base, heads);
1789 : 31276386 : base->clk++;
1790 : :
1791 [ + + ]: 65097854 : while (levels--)
1792 : 2455368 : expire_timers(base, heads + levels);
1793 : : }
1794 : 24849602 : raw_spin_unlock_irq(&base->lock);
1795 : : timer_base_unlock_expiry(base);
1796 : : }
1797 : :
1798 : : /*
1799 : : * This function runs timers and the timer-tq in bottom half context.
1800 : : */
1801 : 12458714 : static __latent_entropy void run_timer_softirq(struct softirq_action *h)
1802 : : {
1803 : 24917428 : struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
1804 : :
1805 : 12458714 : __run_timers(base);
1806 : : if (IS_ENABLED(CONFIG_NO_HZ_COMMON))
1807 : 24831152 : __run_timers(this_cpu_ptr(&timer_bases[BASE_DEF]));
1808 : 12439308 : }
1809 : :
1810 : : /*
1811 : : * Called by the local, per-CPU timer interrupt on SMP.
1812 : : */
1813 : 16977112 : void run_local_timers(void)
1814 : : {
1815 : 33954224 : struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);
1816 : :
1817 : 16977112 : hrtimer_run_queues();
1818 : : /* Raise the softirq only if required. */
1819 [ + + ]: 16753046 : if (time_before(jiffies, base->clk)) {
1820 : : if (!IS_ENABLED(CONFIG_NO_HZ_COMMON))
1821 : : return;
1822 : : /* CPU is awake, so check the deferrable base. */
1823 : : base++;
1824 [ + + ]: 2434412 : if (time_before(jiffies, base->clk))
1825 : : return;
1826 : : }
1827 : 14323196 : raise_softirq(TIMER_SOFTIRQ);
1828 : : }
1829 : :
1830 : : /*
1831 : : * Since schedule_timeout()'s timer is defined on the stack, it must store
1832 : : * the target task on the stack as well.
1833 : : */
1834 : : struct process_timer {
1835 : : struct timer_list timer;
1836 : : struct task_struct *task;
1837 : : };
1838 : :
1839 : 1895112 : static void process_timeout(struct timer_list *t)
1840 : : {
1841 : : struct process_timer *timeout = from_timer(timeout, t, timer);
1842 : :
1843 : 1895112 : wake_up_process(timeout->task);
1844 : 1895112 : }
1845 : :
1846 : : /**
1847 : : * schedule_timeout - sleep until timeout
1848 : : * @timeout: timeout value in jiffies
1849 : : *
1850 : : * Make the current task sleep until @timeout jiffies have
1851 : : * elapsed. The routine will return immediately unless
1852 : : * the current task state has been set (see set_current_state()).
1853 : : *
1854 : : * You can set the task state as follows -
1855 : : *
1856 : : * %TASK_UNINTERRUPTIBLE - at least @timeout jiffies are guaranteed to
1857 : : * pass before the routine returns unless the current task is explicitly
1858 : : * woken up, (e.g. by wake_up_process())".
1859 : : *
1860 : : * %TASK_INTERRUPTIBLE - the routine may return early if a signal is
1861 : : * delivered to the current task or the current task is explicitly woken
1862 : : * up.
1863 : : *
1864 : : * The current task state is guaranteed to be TASK_RUNNING when this
1865 : : * routine returns.
1866 : : *
1867 : : * Specifying a @timeout value of %MAX_SCHEDULE_TIMEOUT will schedule
1868 : : * the CPU away without a bound on the timeout. In this case the return
1869 : : * value will be %MAX_SCHEDULE_TIMEOUT.
1870 : : *
1871 : : * Returns 0 when the timer has expired otherwise the remaining time in
1872 : : * jiffies will be returned. In all cases the return value is guaranteed
1873 : : * to be non-negative.
1874 : : */
1875 : 2652120 : signed long __sched schedule_timeout(signed long timeout)
1876 : : {
1877 : : struct process_timer timer;
1878 : : unsigned long expire;
1879 : :
1880 [ + + ]: 2652120 : switch (timeout)
1881 : : {
1882 : : case MAX_SCHEDULE_TIMEOUT:
1883 : : /*
1884 : : * These two special cases are useful to be comfortable
1885 : : * in the caller. Nothing more. We could take
1886 : : * MAX_SCHEDULE_TIMEOUT from one of the negative value
1887 : : * but I' d like to return a valid offset (>=0) to allow
1888 : : * the caller to do everything it want with the retval.
1889 : : */
1890 : 329548 : schedule();
1891 : 327938 : goto out;
1892 : : default:
1893 : : /*
1894 : : * Another bit of PARANOID. Note that the retval will be
1895 : : * 0 since no piece of kernel is supposed to do a check
1896 : : * for a negative retval of schedule_timeout() (since it
1897 : : * should never happens anyway). You just have the printk()
1898 : : * that will tell you if something is gone wrong and where.
1899 : : */
1900 [ - + ]: 2322572 : if (timeout < 0) {
1901 : 0 : printk(KERN_ERR "schedule_timeout: wrong timeout "
1902 : : "value %lx\n", timeout);
1903 : 0 : dump_stack();
1904 : 0 : current->state = TASK_RUNNING;
1905 : 0 : goto out;
1906 : : }
1907 : : }
1908 : :
1909 : 2322572 : expire = timeout + jiffies;
1910 : :
1911 : 2322572 : timer.task = current;
1912 : : timer_setup_on_stack(&timer.timer, process_timeout, 0);
1913 : 2322572 : __mod_timer(&timer.timer, expire, 0);
1914 : 2322572 : schedule();
1915 : 2321766 : del_singleshot_timer_sync(&timer.timer);
1916 : :
1917 : : /* Remove the timer from the object tracker */
1918 : : destroy_timer_on_stack(&timer.timer);
1919 : :
1920 : 2321768 : timeout = expire - jiffies;
1921 : :
1922 : : out:
1923 : 2649706 : return timeout < 0 ? 0 : timeout;
1924 : : }
1925 : : EXPORT_SYMBOL(schedule_timeout);
1926 : :
1927 : : /*
1928 : : * We can use __set_current_state() here because schedule_timeout() calls
1929 : : * schedule() unconditionally.
1930 : : */
1931 : 922 : signed long __sched schedule_timeout_interruptible(signed long timeout)
1932 : : {
1933 : 922 : __set_current_state(TASK_INTERRUPTIBLE);
1934 : 922 : return schedule_timeout(timeout);
1935 : : }
1936 : : EXPORT_SYMBOL(schedule_timeout_interruptible);
1937 : :
1938 : 0 : signed long __sched schedule_timeout_killable(signed long timeout)
1939 : : {
1940 : 0 : __set_current_state(TASK_KILLABLE);
1941 : 0 : return schedule_timeout(timeout);
1942 : : }
1943 : : EXPORT_SYMBOL(schedule_timeout_killable);
1944 : :
1945 : 0 : signed long __sched schedule_timeout_uninterruptible(signed long timeout)
1946 : : {
1947 : 11584 : __set_current_state(TASK_UNINTERRUPTIBLE);
1948 : 11584 : return schedule_timeout(timeout);
1949 : : }
1950 : : EXPORT_SYMBOL(schedule_timeout_uninterruptible);
1951 : :
1952 : : /*
1953 : : * Like schedule_timeout_uninterruptible(), except this task will not contribute
1954 : : * to load average.
1955 : : */
1956 : 0 : signed long __sched schedule_timeout_idle(signed long timeout)
1957 : : {
1958 : 0 : __set_current_state(TASK_IDLE);
1959 : 0 : return schedule_timeout(timeout);
1960 : : }
1961 : : EXPORT_SYMBOL(schedule_timeout_idle);
1962 : :
1963 : : #ifdef CONFIG_HOTPLUG_CPU
1964 : : static void migrate_timer_list(struct timer_base *new_base, struct hlist_head *head)
1965 : : {
1966 : : struct timer_list *timer;
1967 : : int cpu = new_base->cpu;
1968 : :
1969 : : while (!hlist_empty(head)) {
1970 : : timer = hlist_entry(head->first, struct timer_list, entry);
1971 : : detach_timer(timer, false);
1972 : : timer->flags = (timer->flags & ~TIMER_BASEMASK) | cpu;
1973 : : internal_add_timer(new_base, timer);
1974 : : }
1975 : : }
1976 : :
1977 : : int timers_prepare_cpu(unsigned int cpu)
1978 : : {
1979 : : struct timer_base *base;
1980 : : int b;
1981 : :
1982 : : for (b = 0; b < NR_BASES; b++) {
1983 : : base = per_cpu_ptr(&timer_bases[b], cpu);
1984 : : base->clk = jiffies;
1985 : : base->next_expiry = base->clk + NEXT_TIMER_MAX_DELTA;
1986 : : base->is_idle = false;
1987 : : base->must_forward_clk = true;
1988 : : }
1989 : : return 0;
1990 : : }
1991 : :
1992 : : int timers_dead_cpu(unsigned int cpu)
1993 : : {
1994 : : struct timer_base *old_base;
1995 : : struct timer_base *new_base;
1996 : : int b, i;
1997 : :
1998 : : BUG_ON(cpu_online(cpu));
1999 : :
2000 : : for (b = 0; b < NR_BASES; b++) {
2001 : : old_base = per_cpu_ptr(&timer_bases[b], cpu);
2002 : : new_base = get_cpu_ptr(&timer_bases[b]);
2003 : : /*
2004 : : * The caller is globally serialized and nobody else
2005 : : * takes two locks at once, deadlock is not possible.
2006 : : */
2007 : : raw_spin_lock_irq(&new_base->lock);
2008 : : raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
2009 : :
2010 : : /*
2011 : : * The current CPUs base clock might be stale. Update it
2012 : : * before moving the timers over.
2013 : : */
2014 : : forward_timer_base(new_base);
2015 : :
2016 : : BUG_ON(old_base->running_timer);
2017 : :
2018 : : for (i = 0; i < WHEEL_SIZE; i++)
2019 : : migrate_timer_list(new_base, old_base->vectors + i);
2020 : :
2021 : : raw_spin_unlock(&old_base->lock);
2022 : : raw_spin_unlock_irq(&new_base->lock);
2023 : : put_cpu_ptr(&timer_bases);
2024 : : }
2025 : : return 0;
2026 : : }
2027 : :
2028 : : #endif /* CONFIG_HOTPLUG_CPU */
2029 : :
2030 : 1616 : static void __init init_timer_cpu(int cpu)
2031 : : {
2032 : : struct timer_base *base;
2033 : : int i;
2034 : :
2035 [ + + ]: 4848 : for (i = 0; i < NR_BASES; i++) {
2036 : 3232 : base = per_cpu_ptr(&timer_bases[i], cpu);
2037 : 3232 : base->cpu = cpu;
2038 : 3232 : raw_spin_lock_init(&base->lock);
2039 : 3232 : base->clk = jiffies;
2040 : : timer_base_init_expiry_lock(base);
2041 : : }
2042 : 1616 : }
2043 : :
2044 : 404 : static void __init init_timer_cpus(void)
2045 : : {
2046 : : int cpu;
2047 : :
2048 [ + + ]: 2424 : for_each_possible_cpu(cpu)
2049 : 1616 : init_timer_cpu(cpu);
2050 : 404 : }
2051 : :
2052 : 404 : void __init init_timers(void)
2053 : : {
2054 : 404 : init_timer_cpus();
2055 : 404 : open_softirq(TIMER_SOFTIRQ, run_timer_softirq);
2056 : 404 : }
2057 : :
2058 : : /**
2059 : : * msleep - sleep safely even with waitqueue interruptions
2060 : : * @msecs: Time in milliseconds to sleep for
2061 : : */
2062 : 11584 : void msleep(unsigned int msecs)
2063 : : {
2064 : 11584 : unsigned long timeout = msecs_to_jiffies(msecs) + 1;
2065 : :
2066 [ + + ]: 34752 : while (timeout)
2067 : 23168 : timeout = schedule_timeout_uninterruptible(timeout);
2068 : 11584 : }
2069 : :
2070 : : EXPORT_SYMBOL(msleep);
2071 : :
2072 : : /**
2073 : : * msleep_interruptible - sleep waiting for signals
2074 : : * @msecs: Time in milliseconds to sleep for
2075 : : */
2076 : 0 : unsigned long msleep_interruptible(unsigned int msecs)
2077 : : {
2078 : 0 : unsigned long timeout = msecs_to_jiffies(msecs) + 1;
2079 : :
2080 [ # # # # ]: 0 : while (timeout && !signal_pending(current))
2081 : 0 : timeout = schedule_timeout_interruptible(timeout);
2082 : 0 : return jiffies_to_msecs(timeout);
2083 : : }
2084 : :
2085 : : EXPORT_SYMBOL(msleep_interruptible);
2086 : :
2087 : : /**
2088 : : * usleep_range - Sleep for an approximate time
2089 : : * @min: Minimum time in usecs to sleep
2090 : : * @max: Maximum time in usecs to sleep
2091 : : *
2092 : : * In non-atomic context where the exact wakeup time is flexible, use
2093 : : * usleep_range() instead of udelay(). The sleep improves responsiveness
2094 : : * by avoiding the CPU-hogging busy-wait of udelay(), and the range reduces
2095 : : * power usage by allowing hrtimers to take advantage of an already-
2096 : : * scheduled interrupt instead of scheduling a new one just for this sleep.
2097 : : */
2098 : 3232 : void __sched usleep_range(unsigned long min, unsigned long max)
2099 : : {
2100 : 6464 : ktime_t exp = ktime_add_us(ktime_get(), min);
2101 : 3232 : u64 delta = (u64)(max - min) * NSEC_PER_USEC;
2102 : :
2103 : : for (;;) {
2104 : 3232 : __set_current_state(TASK_UNINTERRUPTIBLE);
2105 : : /* Do not return before the requested sleep time has elapsed */
2106 [ - + ]: 3232 : if (!schedule_hrtimeout_range(&exp, delta, HRTIMER_MODE_ABS))
2107 : : break;
2108 : : }
2109 : 3232 : }
2110 : : EXPORT_SYMBOL(usleep_range);
|
