LCOV - code coverage report
Current view: top level - kernel/time - tick-broadcast.c (source / functions) Hit Total Coverage
Test: Real Lines: 36 247 14.6 %
Date: 2020-10-17 15:46:16 Functions: 1 32 3.1 %
Legend: Neither, QEMU, Real, Both Branches: 0 0 -

           Branch data     Line data    Source code
       1                 :            : // SPDX-License-Identifier: GPL-2.0
       2                 :            : /*
       3                 :            :  * This file contains functions which emulate a local clock-event
       4                 :            :  * device via a broadcast event source.
       5                 :            :  *
       6                 :            :  * Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de>
       7                 :            :  * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
       8                 :            :  * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
       9                 :            :  */
      10                 :            : #include <linux/cpu.h>
      11                 :            : #include <linux/err.h>
      12                 :            : #include <linux/hrtimer.h>
      13                 :            : #include <linux/interrupt.h>
      14                 :            : #include <linux/percpu.h>
      15                 :            : #include <linux/profile.h>
      16                 :            : #include <linux/sched.h>
      17                 :            : #include <linux/smp.h>
      18                 :            : #include <linux/module.h>
      19                 :            : 
      20                 :            : #include "tick-internal.h"
      21                 :            : 
      22                 :            : /*
      23                 :            :  * Broadcast support for broken x86 hardware, where the local apic
      24                 :            :  * timer stops in C3 state.
      25                 :            :  */
      26                 :            : 
      27                 :            : static struct tick_device tick_broadcast_device;
      28                 :            : static cpumask_var_t tick_broadcast_mask __cpumask_var_read_mostly;
      29                 :            : static cpumask_var_t tick_broadcast_on __cpumask_var_read_mostly;
      30                 :            : static cpumask_var_t tmpmask __cpumask_var_read_mostly;
      31                 :            : static int tick_broadcast_forced;
      32                 :            : 
      33                 :            : static __cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(tick_broadcast_lock);
      34                 :            : 
      35                 :            : #ifdef CONFIG_TICK_ONESHOT
      36                 :            : static void tick_broadcast_setup_oneshot(struct clock_event_device *bc);
      37                 :            : static void tick_broadcast_clear_oneshot(int cpu);
      38                 :            : static void tick_resume_broadcast_oneshot(struct clock_event_device *bc);
      39                 :            : # ifdef CONFIG_HOTPLUG_CPU
      40                 :            : static void tick_broadcast_oneshot_offline(unsigned int cpu);
      41                 :            : # endif
      42                 :            : #else
      43                 :            : static inline void tick_broadcast_setup_oneshot(struct clock_event_device *bc) { BUG(); }
      44                 :            : static inline void tick_broadcast_clear_oneshot(int cpu) { }
      45                 :            : static inline void tick_resume_broadcast_oneshot(struct clock_event_device *bc) { }
      46                 :            : # ifdef CONFIG_HOTPLUG_CPU
      47                 :            : static inline void tick_broadcast_oneshot_offline(unsigned int cpu) { }
      48                 :            : # endif
      49                 :            : #endif
      50                 :            : 
      51                 :            : /*
      52                 :            :  * Debugging: see timer_list.c
      53                 :            :  */
      54                 :          0 : struct tick_device *tick_get_broadcast_device(void)
      55                 :            : {
      56                 :          0 :         return &tick_broadcast_device;
      57                 :            : }
      58                 :            : 
      59                 :          0 : struct cpumask *tick_get_broadcast_mask(void)
      60                 :            : {
      61                 :          0 :         return tick_broadcast_mask;
      62                 :            : }
      63                 :            : 
      64                 :            : /*
      65                 :            :  * Start the device in periodic mode
      66                 :            :  */
      67                 :            : static void tick_broadcast_start_periodic(struct clock_event_device *bc)
      68                 :            : {
      69                 :          0 :         if (bc)
      70                 :          0 :                 tick_setup_periodic(bc, 1);
      71                 :            : }
      72                 :            : 
      73                 :            : /*
      74                 :            :  * Check, if the device can be utilized as broadcast device:
      75                 :            :  */
      76                 :            : static bool tick_check_broadcast_device(struct clock_event_device *curdev,
      77                 :            :                                         struct clock_event_device *newdev)
      78                 :            : {
      79                 :          3 :         if ((newdev->features & CLOCK_EVT_FEAT_DUMMY) ||
      80                 :          3 :             (newdev->features & CLOCK_EVT_FEAT_PERCPU) ||
      81                 :            :             (newdev->features & CLOCK_EVT_FEAT_C3STOP))
      82                 :            :                 return false;
      83                 :            : 
      84                 :          0 :         if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT &&
      85                 :          0 :             !(newdev->features & CLOCK_EVT_FEAT_ONESHOT))
      86                 :            :                 return false;
      87                 :            : 
      88                 :          0 :         return !curdev || newdev->rating > curdev->rating;
      89                 :            : }
      90                 :            : 
      91                 :            : /*
      92                 :            :  * Conditionally install/replace broadcast device
      93                 :            :  */
      94                 :          3 : void tick_install_broadcast_device(struct clock_event_device *dev)
      95                 :            : {
      96                 :          3 :         struct clock_event_device *cur = tick_broadcast_device.evtdev;
      97                 :            : 
      98                 :          3 :         if (!tick_check_broadcast_device(cur, dev))
      99                 :            :                 return;
     100                 :            : 
     101                 :          0 :         if (!try_module_get(dev->owner))
     102                 :            :                 return;
     103                 :            : 
     104                 :          0 :         clockevents_exchange_device(cur, dev);
     105                 :          0 :         if (cur)
     106                 :          0 :                 cur->event_handler = clockevents_handle_noop;
     107                 :          0 :         tick_broadcast_device.evtdev = dev;
     108                 :          0 :         if (!cpumask_empty(tick_broadcast_mask))
     109                 :            :                 tick_broadcast_start_periodic(dev);
     110                 :            :         /*
     111                 :            :          * Inform all cpus about this. We might be in a situation
     112                 :            :          * where we did not switch to oneshot mode because the per cpu
     113                 :            :          * devices are affected by CLOCK_EVT_FEAT_C3STOP and the lack
     114                 :            :          * of a oneshot capable broadcast device. Without that
     115                 :            :          * notification the systems stays stuck in periodic mode
     116                 :            :          * forever.
     117                 :            :          */
     118                 :          0 :         if (dev->features & CLOCK_EVT_FEAT_ONESHOT)
     119                 :          0 :                 tick_clock_notify();
     120                 :            : }
     121                 :            : 
     122                 :            : /*
     123                 :            :  * Check, if the device is the broadcast device
     124                 :            :  */
     125                 :          3 : int tick_is_broadcast_device(struct clock_event_device *dev)
     126                 :            : {
     127                 :          3 :         return (dev && tick_broadcast_device.evtdev == dev);
     128                 :            : }
     129                 :            : 
     130                 :          0 : int tick_broadcast_update_freq(struct clock_event_device *dev, u32 freq)
     131                 :            : {
     132                 :            :         int ret = -ENODEV;
     133                 :            : 
     134                 :          0 :         if (tick_is_broadcast_device(dev)) {
     135                 :          0 :                 raw_spin_lock(&tick_broadcast_lock);
     136                 :          0 :                 ret = __clockevents_update_freq(dev, freq);
     137                 :            :                 raw_spin_unlock(&tick_broadcast_lock);
     138                 :            :         }
     139                 :          0 :         return ret;
     140                 :            : }
     141                 :            : 
     142                 :            : 
     143                 :          0 : static void err_broadcast(const struct cpumask *mask)
     144                 :            : {
     145                 :          0 :         pr_crit_once("Failed to broadcast timer tick. Some CPUs may be unresponsive.\n");
     146                 :          0 : }
     147                 :            : 
     148                 :          0 : static void tick_device_setup_broadcast_func(struct clock_event_device *dev)
     149                 :            : {
     150                 :          0 :         if (!dev->broadcast)
     151                 :          0 :                 dev->broadcast = tick_broadcast;
     152                 :          0 :         if (!dev->broadcast) {
     153                 :          0 :                 pr_warn_once("%s depends on broadcast, but no broadcast function available\n",
     154                 :            :                              dev->name);
     155                 :          0 :                 dev->broadcast = err_broadcast;
     156                 :            :         }
     157                 :          0 : }
     158                 :            : 
     159                 :            : /*
     160                 :            :  * Check, if the device is disfunctional and a place holder, which
     161                 :            :  * needs to be handled by the broadcast device.
     162                 :            :  */
     163                 :          3 : int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
     164                 :            : {
     165                 :          3 :         struct clock_event_device *bc = tick_broadcast_device.evtdev;
     166                 :            :         unsigned long flags;
     167                 :            :         int ret = 0;
     168                 :            : 
     169                 :          3 :         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
     170                 :            : 
     171                 :            :         /*
     172                 :            :          * Devices might be registered with both periodic and oneshot
     173                 :            :          * mode disabled. This signals, that the device needs to be
     174                 :            :          * operated from the broadcast device and is a placeholder for
     175                 :            :          * the cpu local device.
     176                 :            :          */
     177                 :          3 :         if (!tick_device_is_functional(dev)) {
     178                 :          0 :                 dev->event_handler = tick_handle_periodic;
     179                 :          0 :                 tick_device_setup_broadcast_func(dev);
     180                 :            :                 cpumask_set_cpu(cpu, tick_broadcast_mask);
     181                 :          0 :                 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
     182                 :            :                         tick_broadcast_start_periodic(bc);
     183                 :            :                 else
     184                 :          0 :                         tick_broadcast_setup_oneshot(bc);
     185                 :            :                 ret = 1;
     186                 :            :         } else {
     187                 :            :                 /*
     188                 :            :                  * Clear the broadcast bit for this cpu if the
     189                 :            :                  * device is not power state affected.
     190                 :            :                  */
     191                 :          3 :                 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
     192                 :            :                         cpumask_clear_cpu(cpu, tick_broadcast_mask);
     193                 :            :                 else
     194                 :          0 :                         tick_device_setup_broadcast_func(dev);
     195                 :            : 
     196                 :            :                 /*
     197                 :            :                  * Clear the broadcast bit if the CPU is not in
     198                 :            :                  * periodic broadcast on state.
     199                 :            :                  */
     200                 :          3 :                 if (!cpumask_test_cpu(cpu, tick_broadcast_on))
     201                 :            :                         cpumask_clear_cpu(cpu, tick_broadcast_mask);
     202                 :            : 
     203                 :          3 :                 switch (tick_broadcast_device.mode) {
     204                 :            :                 case TICKDEV_MODE_ONESHOT:
     205                 :            :                         /*
     206                 :            :                          * If the system is in oneshot mode we can
     207                 :            :                          * unconditionally clear the oneshot mask bit,
     208                 :            :                          * because the CPU is running and therefore
     209                 :            :                          * not in an idle state which causes the power
     210                 :            :                          * state affected device to stop. Let the
     211                 :            :                          * caller initialize the device.
     212                 :            :                          */
     213                 :          0 :                         tick_broadcast_clear_oneshot(cpu);
     214                 :            :                         ret = 0;
     215                 :          0 :                         break;
     216                 :            : 
     217                 :            :                 case TICKDEV_MODE_PERIODIC:
     218                 :            :                         /*
     219                 :            :                          * If the system is in periodic mode, check
     220                 :            :                          * whether the broadcast device can be
     221                 :            :                          * switched off now.
     222                 :            :                          */
     223                 :          3 :                         if (cpumask_empty(tick_broadcast_mask) && bc)
     224                 :          0 :                                 clockevents_shutdown(bc);
     225                 :            :                         /*
     226                 :            :                          * If we kept the cpu in the broadcast mask,
     227                 :            :                          * tell the caller to leave the per cpu device
     228                 :            :                          * in shutdown state. The periodic interrupt
     229                 :            :                          * is delivered by the broadcast device, if
     230                 :            :                          * the broadcast device exists and is not
     231                 :            :                          * hrtimer based.
     232                 :            :                          */
     233                 :          3 :                         if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER))
     234                 :            :                                 ret = cpumask_test_cpu(cpu, tick_broadcast_mask);
     235                 :            :                         break;
     236                 :            :                 default:
     237                 :            :                         break;
     238                 :            :                 }
     239                 :            :         }
     240                 :          3 :         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
     241                 :          3 :         return ret;
     242                 :            : }
     243                 :            : 
     244                 :            : #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
     245                 :          0 : int tick_receive_broadcast(void)
     246                 :            : {
     247                 :          0 :         struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
     248                 :          0 :         struct clock_event_device *evt = td->evtdev;
     249                 :            : 
     250                 :          0 :         if (!evt)
     251                 :            :                 return -ENODEV;
     252                 :            : 
     253                 :          0 :         if (!evt->event_handler)
     254                 :            :                 return -EINVAL;
     255                 :            : 
     256                 :          0 :         evt->event_handler(evt);
     257                 :          0 :         return 0;
     258                 :            : }
     259                 :            : #endif
     260                 :            : 
     261                 :            : /*
     262                 :            :  * Broadcast the event to the cpus, which are set in the mask (mangled).
     263                 :            :  */
     264                 :          0 : static bool tick_do_broadcast(struct cpumask *mask)
     265                 :            : {
     266                 :          0 :         int cpu = smp_processor_id();
     267                 :            :         struct tick_device *td;
     268                 :            :         bool local = false;
     269                 :            : 
     270                 :            :         /*
     271                 :            :          * Check, if the current cpu is in the mask
     272                 :            :          */
     273                 :          0 :         if (cpumask_test_cpu(cpu, mask)) {
     274                 :          0 :                 struct clock_event_device *bc = tick_broadcast_device.evtdev;
     275                 :            : 
     276                 :            :                 cpumask_clear_cpu(cpu, mask);
     277                 :            :                 /*
     278                 :            :                  * We only run the local handler, if the broadcast
     279                 :            :                  * device is not hrtimer based. Otherwise we run into
     280                 :            :                  * a hrtimer recursion.
     281                 :            :                  *
     282                 :            :                  * local timer_interrupt()
     283                 :            :                  *   local_handler()
     284                 :            :                  *     expire_hrtimers()
     285                 :            :                  *       bc_handler()
     286                 :            :                  *         local_handler()
     287                 :            :                  *           expire_hrtimers()
     288                 :            :                  */
     289                 :          0 :                 local = !(bc->features & CLOCK_EVT_FEAT_HRTIMER);
     290                 :            :         }
     291                 :            : 
     292                 :          0 :         if (!cpumask_empty(mask)) {
     293                 :            :                 /*
     294                 :            :                  * It might be necessary to actually check whether the devices
     295                 :            :                  * have different broadcast functions. For now, just use the
     296                 :            :                  * one of the first device. This works as long as we have this
     297                 :            :                  * misfeature only on x86 (lapic)
     298                 :            :                  */
     299                 :          0 :                 td = &per_cpu(tick_cpu_device, cpumask_first(mask));
     300                 :          0 :                 td->evtdev->broadcast(mask);
     301                 :            :         }
     302                 :          0 :         return local;
     303                 :            : }
     304                 :            : 
     305                 :            : /*
     306                 :            :  * Periodic broadcast:
     307                 :            :  * - invoke the broadcast handlers
     308                 :            :  */
     309                 :          0 : static bool tick_do_periodic_broadcast(void)
     310                 :            : {
     311                 :            :         cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask);
     312                 :          0 :         return tick_do_broadcast(tmpmask);
     313                 :            : }
     314                 :            : 
     315                 :            : /*
     316                 :            :  * Event handler for periodic broadcast ticks
     317                 :            :  */
     318                 :          0 : static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
     319                 :            : {
     320                 :          0 :         struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
     321                 :            :         bool bc_local;
     322                 :            : 
     323                 :          0 :         raw_spin_lock(&tick_broadcast_lock);
     324                 :            : 
     325                 :            :         /* Handle spurious interrupts gracefully */
     326                 :          0 :         if (clockevent_state_shutdown(tick_broadcast_device.evtdev)) {
     327                 :            :                 raw_spin_unlock(&tick_broadcast_lock);
     328                 :          0 :                 return;
     329                 :            :         }
     330                 :            : 
     331                 :          0 :         bc_local = tick_do_periodic_broadcast();
     332                 :            : 
     333                 :          0 :         if (clockevent_state_oneshot(dev)) {
     334                 :          0 :                 ktime_t next = ktime_add(dev->next_event, tick_period);
     335                 :            : 
     336                 :          0 :                 clockevents_program_event(dev, next, true);
     337                 :            :         }
     338                 :            :         raw_spin_unlock(&tick_broadcast_lock);
     339                 :            : 
     340                 :            :         /*
     341                 :            :          * We run the handler of the local cpu after dropping
     342                 :            :          * tick_broadcast_lock because the handler might deadlock when
     343                 :            :          * trying to switch to oneshot mode.
     344                 :            :          */
     345                 :          0 :         if (bc_local)
     346                 :          0 :                 td->evtdev->event_handler(td->evtdev);
     347                 :            : }
     348                 :            : 
     349                 :            : /**
     350                 :            :  * tick_broadcast_control - Enable/disable or force broadcast mode
     351                 :            :  * @mode:       The selected broadcast mode
     352                 :            :  *
     353                 :            :  * Called when the system enters a state where affected tick devices
     354                 :            :  * might stop. Note: TICK_BROADCAST_FORCE cannot be undone.
     355                 :            :  */
     356                 :          0 : void tick_broadcast_control(enum tick_broadcast_mode mode)
     357                 :            : {
     358                 :            :         struct clock_event_device *bc, *dev;
     359                 :            :         struct tick_device *td;
     360                 :            :         int cpu, bc_stopped;
     361                 :            :         unsigned long flags;
     362                 :            : 
     363                 :            :         /* Protects also the local clockevent device. */
     364                 :          0 :         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
     365                 :          0 :         td = this_cpu_ptr(&tick_cpu_device);
     366                 :          0 :         dev = td->evtdev;
     367                 :            : 
     368                 :            :         /*
     369                 :            :          * Is the device not affected by the powerstate ?
     370                 :            :          */
     371                 :          0 :         if (!dev || !(dev->features & CLOCK_EVT_FEAT_C3STOP))
     372                 :            :                 goto out;
     373                 :            : 
     374                 :          0 :         if (!tick_device_is_functional(dev))
     375                 :            :                 goto out;
     376                 :            : 
     377                 :          0 :         cpu = smp_processor_id();
     378                 :          0 :         bc = tick_broadcast_device.evtdev;
     379                 :          0 :         bc_stopped = cpumask_empty(tick_broadcast_mask);
     380                 :            : 
     381                 :          0 :         switch (mode) {
     382                 :            :         case TICK_BROADCAST_FORCE:
     383                 :          0 :                 tick_broadcast_forced = 1;
     384                 :            :                 /* fall through */
     385                 :            :         case TICK_BROADCAST_ON:
     386                 :            :                 cpumask_set_cpu(cpu, tick_broadcast_on);
     387                 :          0 :                 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) {
     388                 :            :                         /*
     389                 :            :                          * Only shutdown the cpu local device, if:
     390                 :            :                          *
     391                 :            :                          * - the broadcast device exists
     392                 :            :                          * - the broadcast device is not a hrtimer based one
     393                 :            :                          * - the broadcast device is in periodic mode to
     394                 :            :                          *   avoid a hickup during switch to oneshot mode
     395                 :            :                          */
     396                 :          0 :                         if (bc && !(bc->features & CLOCK_EVT_FEAT_HRTIMER) &&
     397                 :          0 :                             tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
     398                 :          0 :                                 clockevents_shutdown(dev);
     399                 :            :                 }
     400                 :            :                 break;
     401                 :            : 
     402                 :            :         case TICK_BROADCAST_OFF:
     403                 :          0 :                 if (tick_broadcast_forced)
     404                 :            :                         break;
     405                 :            :                 cpumask_clear_cpu(cpu, tick_broadcast_on);
     406                 :          0 :                 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) {
     407                 :          0 :                         if (tick_broadcast_device.mode ==
     408                 :            :                             TICKDEV_MODE_PERIODIC)
     409                 :          0 :                                 tick_setup_periodic(dev, 0);
     410                 :            :                 }
     411                 :            :                 break;
     412                 :            :         }
     413                 :            : 
     414                 :          0 :         if (bc) {
     415                 :          0 :                 if (cpumask_empty(tick_broadcast_mask)) {
     416                 :          0 :                         if (!bc_stopped)
     417                 :          0 :                                 clockevents_shutdown(bc);
     418                 :          0 :                 } else if (bc_stopped) {
     419                 :          0 :                         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC)
     420                 :            :                                 tick_broadcast_start_periodic(bc);
     421                 :            :                         else
     422                 :          0 :                                 tick_broadcast_setup_oneshot(bc);
     423                 :            :                 }
     424                 :            :         }
     425                 :            : out:
     426                 :          0 :         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
     427                 :          0 : }
     428                 :            : EXPORT_SYMBOL_GPL(tick_broadcast_control);
     429                 :            : 
     430                 :            : /*
     431                 :            :  * Set the periodic handler depending on broadcast on/off
     432                 :            :  */
     433                 :          3 : void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast)
     434                 :            : {
     435                 :          3 :         if (!broadcast)
     436                 :          3 :                 dev->event_handler = tick_handle_periodic;
     437                 :            :         else
     438                 :          0 :                 dev->event_handler = tick_handle_periodic_broadcast;
     439                 :          3 : }
     440                 :            : 
     441                 :            : #ifdef CONFIG_HOTPLUG_CPU
     442                 :            : static void tick_shutdown_broadcast(void)
     443                 :            : {
     444                 :            :         struct clock_event_device *bc = tick_broadcast_device.evtdev;
     445                 :            : 
     446                 :            :         if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
     447                 :            :                 if (bc && cpumask_empty(tick_broadcast_mask))
     448                 :            :                         clockevents_shutdown(bc);
     449                 :            :         }
     450                 :            : }
     451                 :            : 
     452                 :            : /*
     453                 :            :  * Remove a CPU from broadcasting
     454                 :            :  */
     455                 :            : void tick_broadcast_offline(unsigned int cpu)
     456                 :            : {
     457                 :            :         raw_spin_lock(&tick_broadcast_lock);
     458                 :            :         cpumask_clear_cpu(cpu, tick_broadcast_mask);
     459                 :            :         cpumask_clear_cpu(cpu, tick_broadcast_on);
     460                 :            :         tick_broadcast_oneshot_offline(cpu);
     461                 :            :         tick_shutdown_broadcast();
     462                 :            :         raw_spin_unlock(&tick_broadcast_lock);
     463                 :            : }
     464                 :            : 
     465                 :            : #endif
     466                 :            : 
     467                 :          0 : void tick_suspend_broadcast(void)
     468                 :            : {
     469                 :            :         struct clock_event_device *bc;
     470                 :            :         unsigned long flags;
     471                 :            : 
     472                 :          0 :         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
     473                 :            : 
     474                 :          0 :         bc = tick_broadcast_device.evtdev;
     475                 :          0 :         if (bc)
     476                 :          0 :                 clockevents_shutdown(bc);
     477                 :            : 
     478                 :          0 :         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
     479                 :          0 : }
     480                 :            : 
     481                 :            : /*
     482                 :            :  * This is called from tick_resume_local() on a resuming CPU. That's
     483                 :            :  * called from the core resume function, tick_unfreeze() and the magic XEN
     484                 :            :  * resume hackery.
     485                 :            :  *
     486                 :            :  * In none of these cases the broadcast device mode can change and the
     487                 :            :  * bit of the resuming CPU in the broadcast mask is safe as well.
     488                 :            :  */
     489                 :          0 : bool tick_resume_check_broadcast(void)
     490                 :            : {
     491                 :          0 :         if (tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT)
     492                 :            :                 return false;
     493                 :            :         else
     494                 :          0 :                 return cpumask_test_cpu(smp_processor_id(), tick_broadcast_mask);
     495                 :            : }
     496                 :            : 
     497                 :          0 : void tick_resume_broadcast(void)
     498                 :            : {
     499                 :            :         struct clock_event_device *bc;
     500                 :            :         unsigned long flags;
     501                 :            : 
     502                 :          0 :         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
     503                 :            : 
     504                 :          0 :         bc = tick_broadcast_device.evtdev;
     505                 :            : 
     506                 :          0 :         if (bc) {
     507                 :          0 :                 clockevents_tick_resume(bc);
     508                 :            : 
     509                 :          0 :                 switch (tick_broadcast_device.mode) {
     510                 :            :                 case TICKDEV_MODE_PERIODIC:
     511                 :          0 :                         if (!cpumask_empty(tick_broadcast_mask))
     512                 :            :                                 tick_broadcast_start_periodic(bc);
     513                 :            :                         break;
     514                 :            :                 case TICKDEV_MODE_ONESHOT:
     515                 :          0 :                         if (!cpumask_empty(tick_broadcast_mask))
     516                 :            :                                 tick_resume_broadcast_oneshot(bc);
     517                 :            :                         break;
     518                 :            :                 }
     519                 :            :         }
     520                 :          0 :         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
     521                 :          0 : }
     522                 :            : 
     523                 :            : #ifdef CONFIG_TICK_ONESHOT
     524                 :            : 
     525                 :            : static cpumask_var_t tick_broadcast_oneshot_mask __cpumask_var_read_mostly;
     526                 :            : static cpumask_var_t tick_broadcast_pending_mask __cpumask_var_read_mostly;
     527                 :            : static cpumask_var_t tick_broadcast_force_mask __cpumask_var_read_mostly;
     528                 :            : 
     529                 :            : /*
     530                 :            :  * Exposed for debugging: see timer_list.c
     531                 :            :  */
     532                 :          0 : struct cpumask *tick_get_broadcast_oneshot_mask(void)
     533                 :            : {
     534                 :          0 :         return tick_broadcast_oneshot_mask;
     535                 :            : }
     536                 :            : 
     537                 :            : /*
     538                 :            :  * Called before going idle with interrupts disabled. Checks whether a
     539                 :            :  * broadcast event from the other core is about to happen. We detected
     540                 :            :  * that in tick_broadcast_oneshot_control(). The callsite can use this
     541                 :            :  * to avoid a deep idle transition as we are about to get the
     542                 :            :  * broadcast IPI right away.
     543                 :            :  */
     544                 :          3 : int tick_check_broadcast_expired(void)
     545                 :            : {
     546                 :          3 :         return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask);
     547                 :            : }
     548                 :            : 
     549                 :            : /*
     550                 :            :  * Set broadcast interrupt affinity
     551                 :            :  */
     552                 :          0 : static void tick_broadcast_set_affinity(struct clock_event_device *bc,
     553                 :            :                                         const struct cpumask *cpumask)
     554                 :            : {
     555                 :          0 :         if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ))
     556                 :            :                 return;
     557                 :            : 
     558                 :          0 :         if (cpumask_equal(bc->cpumask, cpumask))
     559                 :            :                 return;
     560                 :            : 
     561                 :          0 :         bc->cpumask = cpumask;
     562                 :          0 :         irq_set_affinity(bc->irq, bc->cpumask);
     563                 :            : }
     564                 :            : 
     565                 :          0 : static void tick_broadcast_set_event(struct clock_event_device *bc, int cpu,
     566                 :            :                                      ktime_t expires)
     567                 :            : {
     568                 :          0 :         if (!clockevent_state_oneshot(bc))
     569                 :          0 :                 clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
     570                 :            : 
     571                 :          0 :         clockevents_program_event(bc, expires, 1);
     572                 :          0 :         tick_broadcast_set_affinity(bc, cpumask_of(cpu));
     573                 :          0 : }
     574                 :            : 
     575                 :            : static void tick_resume_broadcast_oneshot(struct clock_event_device *bc)
     576                 :            : {
     577                 :          0 :         clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
     578                 :            : }
     579                 :            : 
     580                 :            : /*
     581                 :            :  * Called from irq_enter() when idle was interrupted to reenable the
     582                 :            :  * per cpu device.
     583                 :            :  */
     584                 :          3 : void tick_check_oneshot_broadcast_this_cpu(void)
     585                 :            : {
     586                 :          3 :         if (cpumask_test_cpu(smp_processor_id(), tick_broadcast_oneshot_mask)) {
     587                 :          0 :                 struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
     588                 :            : 
     589                 :            :                 /*
     590                 :            :                  * We might be in the middle of switching over from
     591                 :            :                  * periodic to oneshot. If the CPU has not yet
     592                 :            :                  * switched over, leave the device alone.
     593                 :            :                  */
     594                 :          0 :                 if (td->mode == TICKDEV_MODE_ONESHOT) {
     595                 :          0 :                         clockevents_switch_state(td->evtdev,
     596                 :            :                                               CLOCK_EVT_STATE_ONESHOT);
     597                 :            :                 }
     598                 :            :         }
     599                 :          3 : }
     600                 :            : 
     601                 :            : /*
     602                 :            :  * Handle oneshot mode broadcasting
     603                 :            :  */
     604                 :          0 : static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
     605                 :            : {
     606                 :            :         struct tick_device *td;
     607                 :            :         ktime_t now, next_event;
     608                 :            :         int cpu, next_cpu = 0;
     609                 :            :         bool bc_local;
     610                 :            : 
     611                 :          0 :         raw_spin_lock(&tick_broadcast_lock);
     612                 :          0 :         dev->next_event = KTIME_MAX;
     613                 :            :         next_event = KTIME_MAX;
     614                 :            :         cpumask_clear(tmpmask);
     615                 :          0 :         now = ktime_get();
     616                 :            :         /* Find all expired events */
     617                 :          0 :         for_each_cpu(cpu, tick_broadcast_oneshot_mask) {
     618                 :            :                 /*
     619                 :            :                  * Required for !SMP because for_each_cpu() reports
     620                 :            :                  * unconditionally CPU0 as set on UP kernels.
     621                 :            :                  */
     622                 :            :                 if (!IS_ENABLED(CONFIG_SMP) &&
     623                 :            :                     cpumask_empty(tick_broadcast_oneshot_mask))
     624                 :            :                         break;
     625                 :            : 
     626                 :          0 :                 td = &per_cpu(tick_cpu_device, cpu);
     627                 :          0 :                 if (td->evtdev->next_event <= now) {
     628                 :            :                         cpumask_set_cpu(cpu, tmpmask);
     629                 :            :                         /*
     630                 :            :                          * Mark the remote cpu in the pending mask, so
     631                 :            :                          * it can avoid reprogramming the cpu local
     632                 :            :                          * timer in tick_broadcast_oneshot_control().
     633                 :            :                          */
     634                 :            :                         cpumask_set_cpu(cpu, tick_broadcast_pending_mask);
     635                 :          0 :                 } else if (td->evtdev->next_event < next_event) {
     636                 :            :                         next_event = td->evtdev->next_event;
     637                 :            :                         next_cpu = cpu;
     638                 :            :                 }
     639                 :            :         }
     640                 :            : 
     641                 :            :         /*
     642                 :            :          * Remove the current cpu from the pending mask. The event is
     643                 :            :          * delivered immediately in tick_do_broadcast() !
     644                 :            :          */
     645                 :          0 :         cpumask_clear_cpu(smp_processor_id(), tick_broadcast_pending_mask);
     646                 :            : 
     647                 :            :         /* Take care of enforced broadcast requests */
     648                 :            :         cpumask_or(tmpmask, tmpmask, tick_broadcast_force_mask);
     649                 :            :         cpumask_clear(tick_broadcast_force_mask);
     650                 :            : 
     651                 :            :         /*
     652                 :            :          * Sanity check. Catch the case where we try to broadcast to
     653                 :            :          * offline cpus.
     654                 :            :          */
     655                 :          0 :         if (WARN_ON_ONCE(!cpumask_subset(tmpmask, cpu_online_mask)))
     656                 :            :                 cpumask_and(tmpmask, tmpmask, cpu_online_mask);
     657                 :            : 
     658                 :            :         /*
     659                 :            :          * Wakeup the cpus which have an expired event.
     660                 :            :          */
     661                 :          0 :         bc_local = tick_do_broadcast(tmpmask);
     662                 :            : 
     663                 :            :         /*
     664                 :            :          * Two reasons for reprogram:
     665                 :            :          *
     666                 :            :          * - The global event did not expire any CPU local
     667                 :            :          * events. This happens in dyntick mode, as the maximum PIT
     668                 :            :          * delta is quite small.
     669                 :            :          *
     670                 :            :          * - There are pending events on sleeping CPUs which were not
     671                 :            :          * in the event mask
     672                 :            :          */
     673                 :          0 :         if (next_event != KTIME_MAX)
     674                 :          0 :                 tick_broadcast_set_event(dev, next_cpu, next_event);
     675                 :            : 
     676                 :            :         raw_spin_unlock(&tick_broadcast_lock);
     677                 :            : 
     678                 :          0 :         if (bc_local) {
     679                 :          0 :                 td = this_cpu_ptr(&tick_cpu_device);
     680                 :          0 :                 td->evtdev->event_handler(td->evtdev);
     681                 :            :         }
     682                 :          0 : }
     683                 :            : 
     684                 :            : static int broadcast_needs_cpu(struct clock_event_device *bc, int cpu)
     685                 :            : {
     686                 :          0 :         if (!(bc->features & CLOCK_EVT_FEAT_HRTIMER))
     687                 :            :                 return 0;
     688                 :          0 :         if (bc->next_event == KTIME_MAX)
     689                 :            :                 return 0;
     690                 :          0 :         return bc->bound_on == cpu ? -EBUSY : 0;
     691                 :            : }
     692                 :            : 
     693                 :          0 : static void broadcast_shutdown_local(struct clock_event_device *bc,
     694                 :            :                                      struct clock_event_device *dev)
     695                 :            : {
     696                 :            :         /*
     697                 :            :          * For hrtimer based broadcasting we cannot shutdown the cpu
     698                 :            :          * local device if our own event is the first one to expire or
     699                 :            :          * if we own the broadcast timer.
     700                 :            :          */
     701                 :          0 :         if (bc->features & CLOCK_EVT_FEAT_HRTIMER) {
     702                 :          0 :                 if (broadcast_needs_cpu(bc, smp_processor_id()))
     703                 :            :                         return;
     704                 :          0 :                 if (dev->next_event < bc->next_event)
     705                 :            :                         return;
     706                 :            :         }
     707                 :          0 :         clockevents_switch_state(dev, CLOCK_EVT_STATE_SHUTDOWN);
     708                 :            : }
     709                 :            : 
     710                 :          0 : int __tick_broadcast_oneshot_control(enum tick_broadcast_state state)
     711                 :            : {
     712                 :            :         struct clock_event_device *bc, *dev;
     713                 :            :         int cpu, ret = 0;
     714                 :            :         ktime_t now;
     715                 :            : 
     716                 :            :         /*
     717                 :            :          * If there is no broadcast device, tell the caller not to go
     718                 :            :          * into deep idle.
     719                 :            :          */
     720                 :          0 :         if (!tick_broadcast_device.evtdev)
     721                 :            :                 return -EBUSY;
     722                 :            : 
     723                 :          0 :         dev = this_cpu_ptr(&tick_cpu_device)->evtdev;
     724                 :            : 
     725                 :          0 :         raw_spin_lock(&tick_broadcast_lock);
     726                 :          0 :         bc = tick_broadcast_device.evtdev;
     727                 :          0 :         cpu = smp_processor_id();
     728                 :            : 
     729                 :          0 :         if (state == TICK_BROADCAST_ENTER) {
     730                 :            :                 /*
     731                 :            :                  * If the current CPU owns the hrtimer broadcast
     732                 :            :                  * mechanism, it cannot go deep idle and we do not add
     733                 :            :                  * the CPU to the broadcast mask. We don't have to go
     734                 :            :                  * through the EXIT path as the local timer is not
     735                 :            :                  * shutdown.
     736                 :            :                  */
     737                 :            :                 ret = broadcast_needs_cpu(bc, cpu);
     738                 :          0 :                 if (ret)
     739                 :            :                         goto out;
     740                 :            : 
     741                 :            :                 /*
     742                 :            :                  * If the broadcast device is in periodic mode, we
     743                 :            :                  * return.
     744                 :            :                  */
     745                 :          0 :                 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
     746                 :            :                         /* If it is a hrtimer based broadcast, return busy */
     747                 :          0 :                         if (bc->features & CLOCK_EVT_FEAT_HRTIMER)
     748                 :            :                                 ret = -EBUSY;
     749                 :            :                         goto out;
     750                 :            :                 }
     751                 :            : 
     752                 :          0 :                 if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) {
     753                 :          0 :                         WARN_ON_ONCE(cpumask_test_cpu(cpu, tick_broadcast_pending_mask));
     754                 :            : 
     755                 :            :                         /* Conditionally shut down the local timer. */
     756                 :          0 :                         broadcast_shutdown_local(bc, dev);
     757                 :            : 
     758                 :            :                         /*
     759                 :            :                          * We only reprogram the broadcast timer if we
     760                 :            :                          * did not mark ourself in the force mask and
     761                 :            :                          * if the cpu local event is earlier than the
     762                 :            :                          * broadcast event. If the current CPU is in
     763                 :            :                          * the force mask, then we are going to be
     764                 :            :                          * woken by the IPI right away; we return
     765                 :            :                          * busy, so the CPU does not try to go deep
     766                 :            :                          * idle.
     767                 :            :                          */
     768                 :          0 :                         if (cpumask_test_cpu(cpu, tick_broadcast_force_mask)) {
     769                 :            :                                 ret = -EBUSY;
     770                 :          0 :                         } else if (dev->next_event < bc->next_event) {
     771                 :          0 :                                 tick_broadcast_set_event(bc, cpu, dev->next_event);
     772                 :            :                                 /*
     773                 :            :                                  * In case of hrtimer broadcasts the
     774                 :            :                                  * programming might have moved the
     775                 :            :                                  * timer to this cpu. If yes, remove
     776                 :            :                                  * us from the broadcast mask and
     777                 :            :                                  * return busy.
     778                 :            :                                  */
     779                 :            :                                 ret = broadcast_needs_cpu(bc, cpu);
     780                 :          0 :                                 if (ret) {
     781                 :            :                                         cpumask_clear_cpu(cpu,
     782                 :            :                                                 tick_broadcast_oneshot_mask);
     783                 :            :                                 }
     784                 :            :                         }
     785                 :            :                 }
     786                 :            :         } else {
     787                 :          0 :                 if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) {
     788                 :          0 :                         clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT);
     789                 :            :                         /*
     790                 :            :                          * The cpu which was handling the broadcast
     791                 :            :                          * timer marked this cpu in the broadcast
     792                 :            :                          * pending mask and fired the broadcast
     793                 :            :                          * IPI. So we are going to handle the expired
     794                 :            :                          * event anyway via the broadcast IPI
     795                 :            :                          * handler. No need to reprogram the timer
     796                 :            :                          * with an already expired event.
     797                 :            :                          */
     798                 :          0 :                         if (cpumask_test_and_clear_cpu(cpu,
     799                 :            :                                        tick_broadcast_pending_mask))
     800                 :            :                                 goto out;
     801                 :            : 
     802                 :            :                         /*
     803                 :            :                          * Bail out if there is no next event.
     804                 :            :                          */
     805                 :          0 :                         if (dev->next_event == KTIME_MAX)
     806                 :            :                                 goto out;
     807                 :            :                         /*
     808                 :            :                          * If the pending bit is not set, then we are
     809                 :            :                          * either the CPU handling the broadcast
     810                 :            :                          * interrupt or we got woken by something else.
     811                 :            :                          *
     812                 :            :                          * We are no longer in the broadcast mask, so
     813                 :            :                          * if the cpu local expiry time is already
     814                 :            :                          * reached, we would reprogram the cpu local
     815                 :            :                          * timer with an already expired event.
     816                 :            :                          *
     817                 :            :                          * This can lead to a ping-pong when we return
     818                 :            :                          * to idle and therefore rearm the broadcast
     819                 :            :                          * timer before the cpu local timer was able
     820                 :            :                          * to fire. This happens because the forced
     821                 :            :                          * reprogramming makes sure that the event
     822                 :            :                          * will happen in the future and depending on
     823                 :            :                          * the min_delta setting this might be far
     824                 :            :                          * enough out that the ping-pong starts.
     825                 :            :                          *
     826                 :            :                          * If the cpu local next_event has expired
     827                 :            :                          * then we know that the broadcast timer
     828                 :            :                          * next_event has expired as well and
     829                 :            :                          * broadcast is about to be handled. So we
     830                 :            :                          * avoid reprogramming and enforce that the
     831                 :            :                          * broadcast handler, which did not run yet,
     832                 :            :                          * will invoke the cpu local handler.
     833                 :            :                          *
     834                 :            :                          * We cannot call the handler directly from
     835                 :            :                          * here, because we might be in a NOHZ phase
     836                 :            :                          * and we did not go through the irq_enter()
     837                 :            :                          * nohz fixups.
     838                 :            :                          */
     839                 :          0 :                         now = ktime_get();
     840                 :          0 :                         if (dev->next_event <= now) {
     841                 :            :                                 cpumask_set_cpu(cpu, tick_broadcast_force_mask);
     842                 :            :                                 goto out;
     843                 :            :                         }
     844                 :            :                         /*
     845                 :            :                          * We got woken by something else. Reprogram
     846                 :            :                          * the cpu local timer device.
     847                 :            :                          */
     848                 :          0 :                         tick_program_event(dev->next_event, 1);
     849                 :            :                 }
     850                 :            :         }
     851                 :            : out:
     852                 :            :         raw_spin_unlock(&tick_broadcast_lock);
     853                 :          0 :         return ret;
     854                 :            : }
     855                 :            : 
     856                 :            : /*
     857                 :            :  * Reset the one shot broadcast for a cpu
     858                 :            :  *
     859                 :            :  * Called with tick_broadcast_lock held
     860                 :            :  */
     861                 :          0 : static void tick_broadcast_clear_oneshot(int cpu)
     862                 :            : {
     863                 :            :         cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
     864                 :            :         cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
     865                 :          0 : }
     866                 :            : 
     867                 :          0 : static void tick_broadcast_init_next_event(struct cpumask *mask,
     868                 :            :                                            ktime_t expires)
     869                 :            : {
     870                 :            :         struct tick_device *td;
     871                 :            :         int cpu;
     872                 :            : 
     873                 :          0 :         for_each_cpu(cpu, mask) {
     874                 :          0 :                 td = &per_cpu(tick_cpu_device, cpu);
     875                 :          0 :                 if (td->evtdev)
     876                 :          0 :                         td->evtdev->next_event = expires;
     877                 :            :         }
     878                 :          0 : }
     879                 :            : 
     880                 :            : /**
     881                 :            :  * tick_broadcast_setup_oneshot - setup the broadcast device
     882                 :            :  */
     883                 :          0 : static void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
     884                 :            : {
     885                 :          0 :         int cpu = smp_processor_id();
     886                 :            : 
     887                 :          0 :         if (!bc)
     888                 :          0 :                 return;
     889                 :            : 
     890                 :            :         /* Set it up only once ! */
     891                 :          0 :         if (bc->event_handler != tick_handle_oneshot_broadcast) {
     892                 :            :                 int was_periodic = clockevent_state_periodic(bc);
     893                 :            : 
     894                 :          0 :                 bc->event_handler = tick_handle_oneshot_broadcast;
     895                 :            : 
     896                 :            :                 /*
     897                 :            :                  * We must be careful here. There might be other CPUs
     898                 :            :                  * waiting for periodic broadcast. We need to set the
     899                 :            :                  * oneshot_mask bits for those and program the
     900                 :            :                  * broadcast device to fire.
     901                 :            :                  */
     902                 :            :                 cpumask_copy(tmpmask, tick_broadcast_mask);
     903                 :            :                 cpumask_clear_cpu(cpu, tmpmask);
     904                 :            :                 cpumask_or(tick_broadcast_oneshot_mask,
     905                 :            :                            tick_broadcast_oneshot_mask, tmpmask);
     906                 :            : 
     907                 :          0 :                 if (was_periodic && !cpumask_empty(tmpmask)) {
     908                 :          0 :                         clockevents_switch_state(bc, CLOCK_EVT_STATE_ONESHOT);
     909                 :          0 :                         tick_broadcast_init_next_event(tmpmask,
     910                 :            :                                                        tick_next_period);
     911                 :          0 :                         tick_broadcast_set_event(bc, cpu, tick_next_period);
     912                 :            :                 } else
     913                 :          0 :                         bc->next_event = KTIME_MAX;
     914                 :            :         } else {
     915                 :            :                 /*
     916                 :            :                  * The first cpu which switches to oneshot mode sets
     917                 :            :                  * the bit for all other cpus which are in the general
     918                 :            :                  * (periodic) broadcast mask. So the bit is set and
     919                 :            :                  * would prevent the first broadcast enter after this
     920                 :            :                  * to program the bc device.
     921                 :            :                  */
     922                 :          0 :                 tick_broadcast_clear_oneshot(cpu);
     923                 :            :         }
     924                 :            : }
     925                 :            : 
     926                 :            : /*
     927                 :            :  * Select oneshot operating mode for the broadcast device
     928                 :            :  */
     929                 :          3 : void tick_broadcast_switch_to_oneshot(void)
     930                 :            : {
     931                 :            :         struct clock_event_device *bc;
     932                 :            :         unsigned long flags;
     933                 :            : 
     934                 :          3 :         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
     935                 :            : 
     936                 :          3 :         tick_broadcast_device.mode = TICKDEV_MODE_ONESHOT;
     937                 :          3 :         bc = tick_broadcast_device.evtdev;
     938                 :          3 :         if (bc)
     939                 :          0 :                 tick_broadcast_setup_oneshot(bc);
     940                 :            : 
     941                 :          3 :         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
     942                 :          3 : }
     943                 :            : 
     944                 :            : #ifdef CONFIG_HOTPLUG_CPU
     945                 :            : void hotplug_cpu__broadcast_tick_pull(int deadcpu)
     946                 :            : {
     947                 :            :         struct clock_event_device *bc;
     948                 :            :         unsigned long flags;
     949                 :            : 
     950                 :            :         raw_spin_lock_irqsave(&tick_broadcast_lock, flags);
     951                 :            :         bc = tick_broadcast_device.evtdev;
     952                 :            : 
     953                 :            :         if (bc && broadcast_needs_cpu(bc, deadcpu)) {
     954                 :            :                 /* This moves the broadcast assignment to this CPU: */
     955                 :            :                 clockevents_program_event(bc, bc->next_event, 1);
     956                 :            :         }
     957                 :            :         raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags);
     958                 :            : }
     959                 :            : 
     960                 :            : /*
     961                 :            :  * Remove a dying CPU from broadcasting
     962                 :            :  */
     963                 :            : static void tick_broadcast_oneshot_offline(unsigned int cpu)
     964                 :            : {
     965                 :            :         /*
     966                 :            :          * Clear the broadcast masks for the dead cpu, but do not stop
     967                 :            :          * the broadcast device!
     968                 :            :          */
     969                 :            :         cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask);
     970                 :            :         cpumask_clear_cpu(cpu, tick_broadcast_pending_mask);
     971                 :            :         cpumask_clear_cpu(cpu, tick_broadcast_force_mask);
     972                 :            : }
     973                 :            : #endif
     974                 :            : 
     975                 :            : /*
     976                 :            :  * Check, whether the broadcast device is in one shot mode
     977                 :            :  */
     978                 :          0 : int tick_broadcast_oneshot_active(void)
     979                 :            : {
     980                 :          0 :         return tick_broadcast_device.mode == TICKDEV_MODE_ONESHOT;
     981                 :            : }
     982                 :            : 
     983                 :            : /*
     984                 :            :  * Check whether the broadcast device supports oneshot.
     985                 :            :  */
     986                 :          0 : bool tick_broadcast_oneshot_available(void)
     987                 :            : {
     988                 :          0 :         struct clock_event_device *bc = tick_broadcast_device.evtdev;
     989                 :            : 
     990                 :          0 :         return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false;
     991                 :            : }
     992                 :            : 
     993                 :            : #else
     994                 :            : int __tick_broadcast_oneshot_control(enum tick_broadcast_state state)
     995                 :            : {
     996                 :            :         struct clock_event_device *bc = tick_broadcast_device.evtdev;
     997                 :            : 
     998                 :            :         if (!bc || (bc->features & CLOCK_EVT_FEAT_HRTIMER))
     999                 :            :                 return -EBUSY;
    1000                 :            : 
    1001                 :            :         return 0;
    1002                 :            : }
    1003                 :            : #endif
    1004                 :            : 
    1005                 :          3 : void __init tick_broadcast_init(void)
    1006                 :            : {
    1007                 :            :         zalloc_cpumask_var(&tick_broadcast_mask, GFP_NOWAIT);
    1008                 :            :         zalloc_cpumask_var(&tick_broadcast_on, GFP_NOWAIT);
    1009                 :            :         zalloc_cpumask_var(&tmpmask, GFP_NOWAIT);
    1010                 :            : #ifdef CONFIG_TICK_ONESHOT
    1011                 :            :         zalloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT);
    1012                 :            :         zalloc_cpumask_var(&tick_broadcast_pending_mask, GFP_NOWAIT);
    1013                 :            :         zalloc_cpumask_var(&tick_broadcast_force_mask, GFP_NOWAIT);
    1014                 :            : #endif
    1015                 :          3 : }
    

Generated by: LCOV version 1.14