LCOV - code coverage report
Current view: top level - arch/x86/kernel - jump_label.c (source / functions) Hit Total Coverage
Test: combined.info Lines: 51 59 86.4 %
Date: 2022-04-01 14:58:12 Functions: 6 8 75.0 %
Branches: 22 26 84.6 %

           Branch data     Line data    Source code
       1                 :            : // SPDX-License-Identifier: GPL-2.0
       2                 :            : /*
       3                 :            :  * jump label x86 support
       4                 :            :  *
       5                 :            :  * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
       6                 :            :  *
       7                 :            :  */
       8                 :            : #include <linux/jump_label.h>
       9                 :            : #include <linux/memory.h>
      10                 :            : #include <linux/uaccess.h>
      11                 :            : #include <linux/module.h>
      12                 :            : #include <linux/list.h>
      13                 :            : #include <linux/jhash.h>
      14                 :            : #include <linux/cpu.h>
      15                 :            : #include <asm/kprobes.h>
      16                 :            : #include <asm/alternative.h>
      17                 :            : #include <asm/text-patching.h>
      18                 :            : 
      19                 :          0 : static void bug_at(const void *ip, int line)
      20                 :            : {
      21                 :            :         /*
      22                 :            :          * The location is not an op that we were expecting.
      23                 :            :          * Something went wrong. Crash the box, as something could be
      24                 :            :          * corrupting the kernel.
      25                 :            :          */
      26                 :          0 :         pr_crit("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph) %d\n", ip, ip, ip, line);
      27                 :          0 :         BUG();
      28                 :            : }
      29                 :            : 
      30                 :            : static const void *
      31                 :       8496 : __jump_label_set_jump_code(struct jump_entry *entry, enum jump_label_type type, int init)
      32                 :            : {
      33                 :       8496 :         const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
      34                 :       8496 :         const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
      35                 :       8496 :         const void *expect, *code;
      36                 :       8496 :         const void *addr, *dest;
      37                 :       8496 :         int line;
      38                 :            : 
      39         [ +  + ]:       8496 :         addr = (void *)jump_entry_code(entry);
      40         [ +  + ]:       8496 :         dest = (void *)jump_entry_target(entry);
      41                 :            : 
      42         [ +  + ]:       8496 :         code = text_gen_insn(JMP32_INSN_OPCODE, addr, dest);
      43                 :            : 
      44         [ +  + ]:       8496 :         if (init) {
      45                 :            :                 expect = default_nop; line = __LINE__;
      46         [ +  + ]:        183 :         } else if (type == JUMP_LABEL_JMP) {
      47                 :            :                 expect = ideal_nop; line = __LINE__;
      48                 :            :         } else {
      49                 :         60 :                 expect = code; line = __LINE__;
      50                 :            :         }
      51                 :            : 
      52         [ -  + ]:       8496 :         if (memcmp(addr, expect, JUMP_LABEL_NOP_SIZE))
      53                 :          0 :                 bug_at(addr, line);
      54                 :            : 
      55         [ +  + ]:       8496 :         if (type == JUMP_LABEL_NOP)
      56                 :       8373 :                 code = ideal_nop;
      57                 :            : 
      58                 :       8496 :         return code;
      59                 :            : }
      60                 :            : 
      61                 :       8331 : static void inline __jump_label_transform(struct jump_entry *entry,
      62                 :            :                                           enum jump_label_type type,
      63                 :            :                                           int init)
      64                 :            : {
      65                 :       8331 :         const void *opcode = __jump_label_set_jump_code(entry, type, init);
      66                 :            : 
      67                 :            :         /*
      68                 :            :          * As long as only a single processor is running and the code is still
      69                 :            :          * not marked as RO, text_poke_early() can be used; Checking that
      70                 :            :          * system_state is SYSTEM_BOOTING guarantees it. It will be set to
      71                 :            :          * SYSTEM_SCHEDULING before other cores are awaken and before the
      72                 :            :          * code is write-protected.
      73                 :            :          *
      74                 :            :          * At the time the change is being done, just ignore whether we
      75                 :            :          * are doing nop -> jump or jump -> nop transition, and assume
      76                 :            :          * always nop being the 'currently valid' instruction
      77                 :            :          */
      78   [ +  +  +  - ]:       8331 :         if (init || system_state == SYSTEM_BOOTING) {
      79                 :       8331 :                 text_poke_early((void *)jump_entry_code(entry), opcode,
      80                 :            :                                 JUMP_LABEL_NOP_SIZE);
      81                 :       8331 :                 return;
      82                 :            :         }
      83                 :            : 
      84                 :          0 :         text_poke_bp((void *)jump_entry_code(entry), opcode, JUMP_LABEL_NOP_SIZE, NULL);
      85                 :            : }
      86                 :            : 
      87                 :       8331 : static void __ref jump_label_transform(struct jump_entry *entry,
      88                 :            :                                        enum jump_label_type type,
      89                 :            :                                        int init)
      90                 :            : {
      91                 :       8331 :         mutex_lock(&text_mutex);
      92                 :       8331 :         __jump_label_transform(entry, type, init);
      93                 :       8331 :         mutex_unlock(&text_mutex);
      94                 :       8331 : }
      95                 :            : 
      96                 :         18 : void arch_jump_label_transform(struct jump_entry *entry,
      97                 :            :                                enum jump_label_type type)
      98                 :            : {
      99                 :          0 :         jump_label_transform(entry, type, 0);
     100                 :          0 : }
     101                 :            : 
     102                 :        183 : bool arch_jump_label_transform_queue(struct jump_entry *entry,
     103                 :            :                                      enum jump_label_type type)
     104                 :            : {
     105                 :        183 :         const void *opcode;
     106                 :            : 
     107         [ +  + ]:        183 :         if (system_state == SYSTEM_BOOTING) {
     108                 :            :                 /*
     109                 :            :                  * Fallback to the non-batching mode.
     110                 :            :                  */
     111                 :         18 :                 arch_jump_label_transform(entry, type);
     112                 :         18 :                 return true;
     113                 :            :         }
     114                 :            : 
     115                 :        165 :         mutex_lock(&text_mutex);
     116                 :        165 :         opcode = __jump_label_set_jump_code(entry, type, 0);
     117                 :        165 :         text_poke_queue((void *)jump_entry_code(entry),
     118                 :            :                         opcode, JUMP_LABEL_NOP_SIZE, NULL);
     119                 :        165 :         mutex_unlock(&text_mutex);
     120                 :        165 :         return true;
     121                 :            : }
     122                 :            : 
     123                 :         57 : void arch_jump_label_transform_apply(void)
     124                 :            : {
     125                 :         57 :         mutex_lock(&text_mutex);
     126                 :         57 :         text_poke_finish();
     127                 :         57 :         mutex_unlock(&text_mutex);
     128                 :         57 : }
     129                 :            : 
     130                 :            : static enum {
     131                 :            :         JL_STATE_START,
     132                 :            :         JL_STATE_NO_UPDATE,
     133                 :            :         JL_STATE_UPDATE,
     134                 :            : } jlstate __initdata_or_module = JL_STATE_START;
     135                 :            : 
     136                 :       8313 : __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry,
     137                 :            :                                       enum jump_label_type type)
     138                 :            : {
     139                 :            :         /*
     140                 :            :          * This function is called at boot up and when modules are
     141                 :            :          * first loaded. Check if the default nop, the one that is
     142                 :            :          * inserted at compile time, is the ideal nop. If it is, then
     143                 :            :          * we do not need to update the nop, and we can leave it as is.
     144                 :            :          * If it is not, then we need to update the nop to the ideal nop.
     145                 :            :          */
     146         [ +  + ]:       8313 :         if (jlstate == JL_STATE_START) {
     147                 :          3 :                 const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
     148                 :          3 :                 const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
     149                 :            : 
     150         [ +  - ]:          3 :                 if (memcmp(ideal_nop, default_nop, 5) != 0)
     151                 :          3 :                         jlstate = JL_STATE_UPDATE;
     152                 :            :                 else
     153                 :          0 :                         jlstate = JL_STATE_NO_UPDATE;
     154                 :            :         }
     155         [ +  - ]:       8313 :         if (jlstate == JL_STATE_UPDATE)
     156                 :       8313 :                 jump_label_transform(entry, type, 1);
     157                 :       8313 : }

Generated by: LCOV version 1.14