Branch data     Line data    Source code

       1                 :            : // SPDX-License-Identifier: GPL-2.0
       2                 :            : /*
       3                 :            :  * local apic based NMI watchdog for various CPUs.
       4                 :            :  *
       5                 :            :  * This file also handles reservation of performance counters for coordination
       6                 :            :  * with other users (like oprofile).
       7                 :            :  *
       8                 :            :  * Note that these events normally don't tick when the CPU idles. This means
       9                 :            :  * the frequency varies with CPU load.
      10                 :            :  *
      11                 :            :  * Original code for K7/P6 written by Keith Owens
      12                 :            :  *
      13                 :            :  */
      14                 :            : 
      15                 :            : #include <linux/percpu.h>
      16                 :            : #include <linux/export.h>
      17                 :            : #include <linux/kernel.h>
      18                 :            : #include <linux/bitops.h>
      19                 :            : #include <linux/smp.h>
      20                 :            : #include <asm/nmi.h>
      21                 :            : #include <linux/kprobes.h>
      22                 :            : 
      23                 :            : #include <asm/apic.h>
      24                 :            : #include <asm/perf_event.h>
      25                 :            : 
      26                 :            : /*
      27                 :            :  * this number is calculated from Intel's MSR_P4_CRU_ESCR5 register and it's
      28                 :            :  * offset from MSR_P4_BSU_ESCR0.
      29                 :            :  *
      30                 :            :  * It will be the max for all platforms (for now)
      31                 :            :  */
      32                 :            : #define NMI_MAX_COUNTER_BITS 66
      33                 :            : 
      34                 :            : /*
      35                 :            :  * perfctr_nmi_owner tracks the ownership of the perfctr registers:
      36                 :            :  * evtsel_nmi_owner tracks the ownership of the event selection
      37                 :            :  * - different performance counters/ event selection may be reserved for
      38                 :            :  *   different subsystems this reservation system just tries to coordinate
      39                 :            :  *   things a little
      40                 :            :  */
      41                 :            : static DECLARE_BITMAP(perfctr_nmi_owner, NMI_MAX_COUNTER_BITS);
      42                 :            : static DECLARE_BITMAP(evntsel_nmi_owner, NMI_MAX_COUNTER_BITS);
      43                 :            : 
      44                 :            : /* converts an msr to an appropriate reservation bit */
      45                 :          0 : static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr)
      46                 :            : {
      47                 :            :         /* returns the bit offset of the performance counter register */
      48      [ #  #  # ]:          0 :         switch (boot_cpu_data.x86_vendor) {
      49                 :          0 :         case X86_VENDOR_HYGON:
      50                 :            :         case X86_VENDOR_AMD:
      51         [ #  # ]:          0 :                 if (msr >= MSR_F15H_PERF_CTR)
      52                 :          0 :                         return (msr - MSR_F15H_PERF_CTR) >> 1;
      53                 :          0 :                 return msr - MSR_K7_PERFCTR0;
      54                 :            :         case X86_VENDOR_INTEL:
      55         [ #  # ]:          0 :                 if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
      56                 :          0 :                         return msr - MSR_ARCH_PERFMON_PERFCTR0;
      57                 :            : 
      58   [ #  #  #  # ]:          0 :                 switch (boot_cpu_data.x86) {
      59                 :          0 :                 case 6:
      60                 :          0 :                         return msr - MSR_P6_PERFCTR0;
      61                 :          0 :                 case 11:
      62                 :          0 :                         return msr - MSR_KNC_PERFCTR0;
      63                 :          0 :                 case 15:
      64                 :          0 :                         return msr - MSR_P4_BPU_PERFCTR0;
      65                 :            :                 }
      66                 :            :         }
      67                 :            :         return 0;
      68                 :            : }
      69                 :            : 
      70                 :            : /*
      71                 :            :  * converts an msr to an appropriate reservation bit
      72                 :            :  * returns the bit offset of the event selection register
      73                 :            :  */
      74                 :          0 : static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr)
      75                 :            : {
      76                 :            :         /* returns the bit offset of the event selection register */
      77      [ #  #  # ]:          0 :         switch (boot_cpu_data.x86_vendor) {
      78                 :          0 :         case X86_VENDOR_HYGON:
      79                 :            :         case X86_VENDOR_AMD:
      80         [ #  # ]:          0 :                 if (msr >= MSR_F15H_PERF_CTL)
      81                 :          0 :                         return (msr - MSR_F15H_PERF_CTL) >> 1;
      82                 :          0 :                 return msr - MSR_K7_EVNTSEL0;
      83                 :            :         case X86_VENDOR_INTEL:
      84         [ #  # ]:          0 :                 if (cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
      85                 :          0 :                         return msr - MSR_ARCH_PERFMON_EVENTSEL0;
      86                 :            : 
      87   [ #  #  #  # ]:          0 :                 switch (boot_cpu_data.x86) {
      88                 :          0 :                 case 6:
      89                 :          0 :                         return msr - MSR_P6_EVNTSEL0;
      90                 :          0 :                 case 11:
      91                 :          0 :                         return msr - MSR_KNC_EVNTSEL0;
      92                 :          0 :                 case 15:
      93                 :          0 :                         return msr - MSR_P4_BSU_ESCR0;
      94                 :            :                 }
      95                 :            :         }
      96                 :            :         return 0;
      97                 :            : 
      98                 :            : }
      99                 :            : 
     100                 :            : /* checks for a bit availability (hack for oprofile) */
     101                 :          0 : int avail_to_resrv_perfctr_nmi_bit(unsigned int counter)
     102                 :            : {
     103         [ #  # ]:          0 :         BUG_ON(counter > NMI_MAX_COUNTER_BITS);
     104                 :            : 
     105                 :          0 :         return !test_bit(counter, perfctr_nmi_owner);
     106                 :            : }
     107                 :            : EXPORT_SYMBOL(avail_to_resrv_perfctr_nmi_bit);
     108                 :            : 
     109                 :          0 : int reserve_perfctr_nmi(unsigned int msr)
     110                 :            : {
     111                 :          0 :         unsigned int counter;
     112                 :            : 
     113                 :          0 :         counter = nmi_perfctr_msr_to_bit(msr);
     114                 :            :         /* register not managed by the allocator? */
     115         [ #  # ]:          0 :         if (counter > NMI_MAX_COUNTER_BITS)
     116                 :            :                 return 1;
     117                 :            : 
     118         [ #  # ]:          0 :         if (!test_and_set_bit(counter, perfctr_nmi_owner))
     119                 :          0 :                 return 1;
     120                 :            :         return 0;
     121                 :            : }
     122                 :            : EXPORT_SYMBOL(reserve_perfctr_nmi);
     123                 :            : 
     124                 :          0 : void release_perfctr_nmi(unsigned int msr)
     125                 :            : {
     126                 :          0 :         unsigned int counter;
     127                 :            : 
     128                 :          0 :         counter = nmi_perfctr_msr_to_bit(msr);
     129                 :            :         /* register not managed by the allocator? */
     130         [ #  # ]:          0 :         if (counter > NMI_MAX_COUNTER_BITS)
     131                 :            :                 return;
     132                 :            : 
     133                 :          0 :         clear_bit(counter, perfctr_nmi_owner);
     134                 :            : }
     135                 :            : EXPORT_SYMBOL(release_perfctr_nmi);
     136                 :            : 
     137                 :          0 : int reserve_evntsel_nmi(unsigned int msr)
     138                 :            : {
     139                 :          0 :         unsigned int counter;
     140                 :            : 
     141                 :          0 :         counter = nmi_evntsel_msr_to_bit(msr);
     142                 :            :         /* register not managed by the allocator? */
     143         [ #  # ]:          0 :         if (counter > NMI_MAX_COUNTER_BITS)
     144                 :            :                 return 1;
     145                 :            : 
     146         [ #  # ]:          0 :         if (!test_and_set_bit(counter, evntsel_nmi_owner))
     147                 :          0 :                 return 1;
     148                 :            :         return 0;
     149                 :            : }
     150                 :            : EXPORT_SYMBOL(reserve_evntsel_nmi);
     151                 :            : 
     152                 :          0 : void release_evntsel_nmi(unsigned int msr)
     153                 :            : {
     154                 :          0 :         unsigned int counter;
     155                 :            : 
     156                 :          0 :         counter = nmi_evntsel_msr_to_bit(msr);
     157                 :            :         /* register not managed by the allocator? */
     158         [ #  # ]:          0 :         if (counter > NMI_MAX_COUNTER_BITS)
     159                 :            :                 return;
     160                 :            : 
     161                 :          0 :         clear_bit(counter, evntsel_nmi_owner);
     162                 :            : }
     163                 :            : EXPORT_SYMBOL(release_evntsel_nmi);