arm64: entry: Move ct_user_exit before any other exception

When taking an SError or Debug exception from EL0, we run the C
handler for these exceptions before updating the context tracking
code and unmasking lower priority interrupts.

When booting with nohz_full lockdep tells us we got this wrong:
| =============================
| WARNING: suspicious RCU usage
| 5.3.0-rc2-00010-gb4b5e9dcb11b-dirty #11271 Not tainted
| -----------------------------
| include/linux/rcupdate.h:643 rcu_read_unlock() used illegally wh!
|
| other info that might help us debug this:
|
|
| RCU used illegally from idle CPU!
| rcu_scheduler_active = 2, debug_locks = 1
| RCU used illegally from extended quiescent state!
| 1 lock held by a.out/432:
|  #0: 00000000c7a79515 (rcu_read_lock){....}, at: brk_handler+0x00
|
| stack backtrace:
| CPU: 1 PID: 432 Comm: a.out Not tainted 5.3.0-rc2-00010-gb4b5e9d1
| Hardware name: ARM LTD ARM Juno Development Platform/ARM Juno De8
| Call trace:
|  dump_backtrace+0x0/0x140
|  show_stack+0x14/0x20
|  dump_stack+0xbc/0x104
|  lockdep_rcu_suspicious+0xf8/0x108
|  brk_handler+0x164/0x1b0
|  do_debug_exception+0x11c/0x278
|  el0_dbg+0x14/0x20

Moving the ct_user_exit calls to be before do_debug_exception() means
they are also before trace_hardirqs_off() has been updated. Add a new
ct_user_exit_irqoff macro to avoid the context-tracking code using
irqsave/restore before we've updated trace_hardirqs_off(). To be
consistent, do this everywhere.

The C helper is called enter_from_user_mode() to match x86 in the hope
we can merge them into kernel/context_tracking.c later.

Cc: Masami Hiramatsu <mhiramat@kernel.org>
Fixes: 6c81fe7925cc4c42 ("arm64: enable context tracking")
Signed-off-by: James Morse <james.morse@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h
index ed57b760f38cf2497ad37a91041b6f806026f817..a17393ff6677425e676d0e6dd958f8862f87e913 100644 (file)
--- a/arch/arm64/include/asm/exception.h
+++ b/arch/arm64/include/asm/exception.h
@@ -30,4 +30,6 @@ static inline u32 disr_to_esr(u64 disr)
        return esr;
 }
 
+asmlinkage void enter_from_user_mode(void);
+
 #endif /* __ASM_EXCEPTION_H */

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 320a30dbe35efb02e4f7d5bd52c82189e1f4400c..84a822748c84e58c85fc19a62d63053b97f0b2c2 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -30,9 +30,9 @@
  * Context tracking subsystem.  Used to instrument transitions
  * between user and kernel mode.
  */
-       .macro ct_user_exit
+       .macro ct_user_exit_irqoff
 #ifdef CONFIG_CONTEXT_TRACKING
-       bl      context_tracking_user_exit
+       bl      enter_from_user_mode
 #endif
        .endm
 
@@ -792,8 +792,8 @@ el0_cp15:
        /*
         * Trapped CP15 (MRC, MCR, MRRC, MCRR) instructions
         */
+       ct_user_exit_irqoff
        enable_daif
-       ct_user_exit
        mov     x0, x25
        mov     x1, sp
        bl      do_cp15instr
@@ -805,8 +805,8 @@ el0_da:
         * Data abort handling
         */
        mrs     x26, far_el1
+       ct_user_exit_irqoff
        enable_daif
-       ct_user_exit
        clear_address_tag x0, x26
        mov     x1, x25
        mov     x2, sp
@@ -818,11 +818,11 @@ el0_ia:
         */
        mrs     x26, far_el1
        gic_prio_kentry_setup tmp=x0
+       ct_user_exit_irqoff
        enable_da_f
 #ifdef CONFIG_TRACE_IRQFLAGS
        bl      trace_hardirqs_off
 #endif
-       ct_user_exit
        mov     x0, x26
        mov     x1, x25
        mov     x2, sp
@@ -832,8 +832,8 @@ el0_fpsimd_acc:
        /*
         * Floating Point or Advanced SIMD access
         */
+       ct_user_exit_irqoff
        enable_daif
-       ct_user_exit
        mov     x0, x25
        mov     x1, sp
        bl      do_fpsimd_acc
@@ -842,8 +842,8 @@ el0_sve_acc:
        /*
         * Scalable Vector Extension access
         */
+       ct_user_exit_irqoff
        enable_daif
-       ct_user_exit
        mov     x0, x25
        mov     x1, sp
        bl      do_sve_acc
@@ -852,8 +852,8 @@ el0_fpsimd_exc:
        /*
         * Floating Point, Advanced SIMD or SVE exception
         */
+       ct_user_exit_irqoff
        enable_daif
-       ct_user_exit
        mov     x0, x25
        mov     x1, sp
        bl      do_fpsimd_exc
@@ -868,11 +868,11 @@ el0_sp_pc:
         * Stack or PC alignment exception handling
         */
        gic_prio_kentry_setup tmp=x0
+       ct_user_exit_irqoff
        enable_da_f
 #ifdef CONFIG_TRACE_IRQFLAGS
        bl      trace_hardirqs_off
 #endif
-       ct_user_exit
        mov     x0, x26
        mov     x1, x25
        mov     x2, sp
@@ -882,8 +882,8 @@ el0_undef:
        /*
         * Undefined instruction
         */
+       ct_user_exit_irqoff
        enable_daif
-       ct_user_exit
        mov     x0, sp
        bl      do_undefinstr
        b       ret_to_user
@@ -891,8 +891,8 @@ el0_sys:
        /*
         * System instructions, for trapped cache maintenance instructions
         */
+       ct_user_exit_irqoff
        enable_daif
-       ct_user_exit
        mov     x0, x25
        mov     x1, sp
        bl      do_sysinstr
@@ -902,17 +902,18 @@ el0_dbg:
         * Debug exception handling
         */
        tbnz    x24, #0, el0_inv                // EL0 only
+       mrs     x24, far_el1
        gic_prio_kentry_setup tmp=x3
-       mrs     x0, far_el1
+       ct_user_exit_irqoff
+       mov     x0, x24
        mov     x1, x25
        mov     x2, sp
        bl      do_debug_exception
        enable_da_f
-       ct_user_exit
        b       ret_to_user
 el0_inv:
+       ct_user_exit_irqoff
        enable_daif
-       ct_user_exit
        mov     x0, sp
        mov     x1, #BAD_SYNC
        mov     x2, x25
@@ -925,13 +926,13 @@ el0_irq:
        kernel_entry 0
 el0_irq_naked:
        gic_prio_irq_setup pmr=x20, tmp=x0
+       ct_user_exit_irqoff
        enable_da_f
 
 #ifdef CONFIG_TRACE_IRQFLAGS
        bl      trace_hardirqs_off
 #endif
 
-       ct_user_exit
 #ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
        tbz     x22, #55, 1f
        bl      do_el0_irq_bp_hardening
@@ -958,13 +959,14 @@ ENDPROC(el1_error)
 el0_error:
        kernel_entry 0
 el0_error_naked:
-       mrs     x1, esr_el1
+       mrs     x25, esr_el1
        gic_prio_kentry_setup tmp=x2
+       ct_user_exit_irqoff
        enable_dbg
        mov     x0, sp
+       mov     x1, x25
        bl      do_serror
        enable_da_f
-       ct_user_exit
        b       ret_to_user
 ENDPROC(el0_error)
 

diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index a5d7ce4297b03afd1fa27625e83fe10e6141ef23..6e950908eb97409d60d62e034e33f0342e6fbaa5 100644 (file)
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -7,9 +7,11 @@
  */
 
 #include <linux/bug.h>
+#include <linux/context_tracking.h>
 #include <linux/signal.h>
 #include <linux/personality.h>
 #include <linux/kallsyms.h>
+#include <linux/kprobes.h>
 #include <linux/spinlock.h>
 #include <linux/uaccess.h>
 #include <linux/hardirq.h>
@@ -900,6 +902,13 @@ asmlinkage void do_serror(struct pt_regs *regs, unsigned int esr)
                nmi_exit();
 }
 
+asmlinkage void enter_from_user_mode(void)
+{
+       CT_WARN_ON(ct_state() != CONTEXT_USER);
+       user_exit_irqoff();
+}
+NOKPROBE_SYMBOL(enter_from_user_mode);
+
 void __pte_error(const char *file, int line, unsigned long val)
 {
        pr_err("%s:%d: bad pte %016lx.\n", file, line, val);