1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * preemptoff and irqoff tracepoints 4 * 5 * Copyright (C) Joel Fernandes (Google) <joel@joelfernandes.org> 6 */ 7 8 #include <linux/kallsyms.h> 9 #include <linux/uaccess.h> 10 #include <linux/module.h> 11 #include <linux/ftrace.h> 12 #include <linux/kprobes.h> 13 #include "trace.h" 14 15 #define CREATE_TRACE_POINTS 16 #include <trace/events/preemptirq.h> 17 18 #ifdef CONFIG_TRACE_IRQFLAGS 19 /* Per-cpu variable to prevent redundant calls when IRQs already off */ 20 static DEFINE_PER_CPU(int, tracing_irq_cpu); 21 22 /* 23 * Like trace_hardirqs_on() but without the lockdep invocation. This is 24 * used in the low level entry code where the ordering vs. RCU is important 25 * and lockdep uses a staged approach which splits the lockdep hardirq 26 * tracking into a RCU on and a RCU off section. 27 */ 28 void trace_hardirqs_on_prepare(void) 29 { 30 if (this_cpu_read(tracing_irq_cpu)) { 31 trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1); 32 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1); 33 this_cpu_write(tracing_irq_cpu, 0); 34 } 35 } 36 EXPORT_SYMBOL(trace_hardirqs_on_prepare); 37 NOKPROBE_SYMBOL(trace_hardirqs_on_prepare); 38 39 void trace_hardirqs_on(void) 40 { 41 if (this_cpu_read(tracing_irq_cpu)) { 42 trace_irq_enable(CALLER_ADDR0, CALLER_ADDR1); 43 tracer_hardirqs_on(CALLER_ADDR0, CALLER_ADDR1); 44 this_cpu_write(tracing_irq_cpu, 0); 45 } 46 47 lockdep_hardirqs_on_prepare(); 48 lockdep_hardirqs_on(CALLER_ADDR0); 49 } 50 EXPORT_SYMBOL(trace_hardirqs_on); 51 NOKPROBE_SYMBOL(trace_hardirqs_on); 52 53 /* 54 * Like trace_hardirqs_off() but without the lockdep invocation. This is 55 * used in the low level entry code where the ordering vs. RCU is important 56 * and lockdep uses a staged approach which splits the lockdep hardirq 57 * tracking into a RCU on and a RCU off section. 58 */ 59 void trace_hardirqs_off_finish(void) 60 { 61 if (!this_cpu_read(tracing_irq_cpu)) { 62 this_cpu_write(tracing_irq_cpu, 1); 63 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); 64 trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1); 65 } 66 67 } 68 EXPORT_SYMBOL(trace_hardirqs_off_finish); 69 NOKPROBE_SYMBOL(trace_hardirqs_off_finish); 70 71 void trace_hardirqs_off(void) 72 { 73 lockdep_hardirqs_off(CALLER_ADDR0); 74 75 if (!this_cpu_read(tracing_irq_cpu)) { 76 this_cpu_write(tracing_irq_cpu, 1); 77 tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); 78 trace_irq_disable(CALLER_ADDR0, CALLER_ADDR1); 79 } 80 } 81 EXPORT_SYMBOL(trace_hardirqs_off); 82 NOKPROBE_SYMBOL(trace_hardirqs_off); 83 #endif /* CONFIG_TRACE_IRQFLAGS */ 84 85 #ifdef CONFIG_TRACE_PREEMPT_TOGGLE 86 87 void trace_preempt_on(unsigned long a0, unsigned long a1) 88 { 89 trace_preempt_enable(a0, a1); 90 tracer_preempt_on(a0, a1); 91 } 92 93 void trace_preempt_off(unsigned long a0, unsigned long a1) 94 { 95 trace_preempt_disable(a0, a1); 96 tracer_preempt_off(a0, a1); 97 } 98 #endif 99