1 // SPDX-License-Identifier: GPL-2.0
3 * jump label x86 support
5 * Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
8 #include <linux/jump_label.h>
9 #include <linux/memory.h>
10 #include <linux/uaccess.h>
11 #include <linux/module.h>
12 #include <linux/list.h>
13 #include <linux/jhash.h>
14 #include <linux/cpu.h>
15 #include <asm/kprobes.h>
16 #include <asm/alternative.h>
17 #include <asm/text-patching.h>
19 union jump_code_union {
20 char code[JUMP_LABEL_NOP_SIZE];
24 } __attribute__((packed));
27 static void bug_at(unsigned char *ip, int line)
30 * The location is not an op that we were expecting.
31 * Something went wrong. Crash the box, as something could be
32 * corrupting the kernel.
34 pr_crit("jump_label: Fatal kernel bug, unexpected op at %pS [%p] (%5ph) %d\n", ip, ip, ip, line);
38 static void __ref __jump_label_transform(struct jump_entry *entry,
39 enum jump_label_type type,
42 union jump_code_union jmp;
43 const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
44 const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
45 const void *expect, *code;
49 jmp.offset = jump_entry_target(entry) -
50 (jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE);
52 if (type == JUMP_LABEL_JMP) {
54 expect = default_nop; line = __LINE__;
56 expect = ideal_nop; line = __LINE__;
62 expect = default_nop; line = __LINE__;
64 expect = &jmp.code; line = __LINE__;
70 if (memcmp((void *)jump_entry_code(entry), expect, JUMP_LABEL_NOP_SIZE))
71 bug_at((void *)jump_entry_code(entry), line);
74 * As long as only a single processor is running and the code is still
75 * not marked as RO, text_poke_early() can be used; Checking that
76 * system_state is SYSTEM_BOOTING guarantees it. It will be set to
77 * SYSTEM_SCHEDULING before other cores are awaken and before the
78 * code is write-protected.
80 * At the time the change is being done, just ignore whether we
81 * are doing nop -> jump or jump -> nop transition, and assume
82 * always nop being the 'currently valid' instruction
84 if (init || system_state == SYSTEM_BOOTING) {
85 text_poke_early((void *)jump_entry_code(entry), code,
90 text_poke_bp((void *)jump_entry_code(entry), code, JUMP_LABEL_NOP_SIZE,
91 (void *)jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE);
94 void arch_jump_label_transform(struct jump_entry *entry,
95 enum jump_label_type type)
97 mutex_lock(&text_mutex);
98 __jump_label_transform(entry, type, 0);
99 mutex_unlock(&text_mutex);
106 } jlstate __initdata_or_module = JL_STATE_START;
108 __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry,
109 enum jump_label_type type)
112 * This function is called at boot up and when modules are
113 * first loaded. Check if the default nop, the one that is
114 * inserted at compile time, is the ideal nop. If it is, then
115 * we do not need to update the nop, and we can leave it as is.
116 * If it is not, then we need to update the nop to the ideal nop.
118 if (jlstate == JL_STATE_START) {
119 const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
120 const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
122 if (memcmp(ideal_nop, default_nop, 5) != 0)
123 jlstate = JL_STATE_UPDATE;
125 jlstate = JL_STATE_NO_UPDATE;
127 if (jlstate == JL_STATE_UPDATE)
128 __jump_label_transform(entry, type, 1);