1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Stage 1 of the trace events. 4 * 5 * Override the macros in the event tracepoint header <trace/events/XXX.h> 6 * to include the following: 7 * 8 * struct trace_event_raw_<call> { 9 * struct trace_entry ent; 10 * <type> <item>; 11 * <type2> <item2>[<len>]; 12 * [...] 13 * }; 14 * 15 * The <type> <item> is created by the __field(type, item) macro or 16 * the __array(type2, item2, len) macro. 17 * We simply do "type item;", and that will create the fields 18 * in the structure. 19 */ 20 21 #include <linux/trace_events.h> 22 23 #ifndef TRACE_SYSTEM_VAR 24 #define TRACE_SYSTEM_VAR TRACE_SYSTEM 25 #endif 26 27 #include "stages/init.h" 28 29 /* 30 * DECLARE_EVENT_CLASS can be used to add a generic function 31 * handlers for events. That is, if all events have the same 32 * parameters and just have distinct trace points. 33 * Each tracepoint can be defined with DEFINE_EVENT and that 34 * will map the DECLARE_EVENT_CLASS to the tracepoint. 35 * 36 * TRACE_EVENT is a one to one mapping between tracepoint and template. 37 */ 38 #undef TRACE_EVENT 39 #define TRACE_EVENT(name, proto, args, tstruct, assign, print) \ 40 DECLARE_EVENT_CLASS(name, \ 41 PARAMS(proto), \ 42 PARAMS(args), \ 43 PARAMS(tstruct), \ 44 PARAMS(assign), \ 45 PARAMS(print)); \ 46 DEFINE_EVENT(name, name, PARAMS(proto), PARAMS(args)); 47 48 #undef TRACE_EVENT_SYSCALL 49 #define TRACE_EVENT_SYSCALL(name, proto, args, tstruct, assign, print, reg, unreg) \ 50 DECLARE_EVENT_SYSCALL_CLASS(name, \ 51 PARAMS(proto), \ 52 PARAMS(args), \ 53 PARAMS(tstruct), \ 54 PARAMS(assign), \ 55 PARAMS(print)); \ 56 DEFINE_EVENT(name, name, PARAMS(proto), PARAMS(args)); 57 58 #include "stages/stage1_struct_define.h" 59 60 #undef DECLARE_EVENT_CLASS 61 #define DECLARE_EVENT_CLASS(name, proto, args, tstruct, assign, print) \ 62 struct trace_event_raw_##name { \ 63 struct trace_entry ent; \ 64 tstruct \ 65 char __data[]; \ 66 }; \ 67 \ 68 static struct trace_event_class event_class_##name; 69 70 #undef DECLARE_EVENT_SYSCALL_CLASS 71 #define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS 72 73 #undef DEFINE_EVENT 74 #define DEFINE_EVENT(template, name, proto, args) \ 75 static struct trace_event_call __used \ 76 __attribute__((__aligned__(4))) event_##name 77 78 #undef DEFINE_EVENT_FN 79 #define DEFINE_EVENT_FN(template, name, proto, args, reg, unreg) \ 80 DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args)) 81 82 #undef DEFINE_EVENT_PRINT 83 #define DEFINE_EVENT_PRINT(template, name, proto, args, print) \ 84 DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args)) 85 86 /* Callbacks are meaningless to ftrace. */ 87 #undef TRACE_EVENT_FN 88 #define TRACE_EVENT_FN(name, proto, args, tstruct, \ 89 assign, print, reg, unreg) \ 90 TRACE_EVENT(name, PARAMS(proto), PARAMS(args), \ 91 PARAMS(tstruct), PARAMS(assign), PARAMS(print)) \ 92 93 #undef TRACE_EVENT_FN_COND 94 #define TRACE_EVENT_FN_COND(name, proto, args, cond, tstruct, \ 95 assign, print, reg, unreg) \ 96 TRACE_EVENT_CONDITION(name, PARAMS(proto), PARAMS(args), PARAMS(cond), \ 97 PARAMS(tstruct), PARAMS(assign), PARAMS(print)) \ 98 99 #undef TRACE_EVENT_FLAGS 100 #define TRACE_EVENT_FLAGS(name, value) \ 101 __TRACE_EVENT_FLAGS(name, value) 102 103 #undef TRACE_EVENT_PERF_PERM 104 #define TRACE_EVENT_PERF_PERM(name, expr...) \ 105 __TRACE_EVENT_PERF_PERM(name, expr) 106 107 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 108 109 /* 110 * Stage 2 of the trace events. 111 * 112 * Include the following: 113 * 114 * struct trace_event_data_offsets_<call> { 115 * u32 <item1>; 116 * u32 <item2>; 117 * [...] 118 * }; 119 * 120 * The __dynamic_array() macro will create each u32 <item>, this is 121 * to keep the offset of each array from the beginning of the event. 122 * The size of an array is also encoded, in the higher 16 bits of <item>. 123 */ 124 125 #include "stages/stage2_data_offsets.h" 126 127 #undef DECLARE_EVENT_CLASS 128 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ 129 struct trace_event_data_offsets_##call { \ 130 tstruct; \ 131 }; 132 133 #undef DECLARE_EVENT_SYSCALL_CLASS 134 #define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS 135 136 #undef DEFINE_EVENT 137 #define DEFINE_EVENT(template, name, proto, args) 138 139 #undef DEFINE_EVENT_PRINT 140 #define DEFINE_EVENT_PRINT(template, name, proto, args, print) 141 142 #undef TRACE_EVENT_FLAGS 143 #define TRACE_EVENT_FLAGS(event, flag) 144 145 #undef TRACE_EVENT_PERF_PERM 146 #define TRACE_EVENT_PERF_PERM(event, expr...) 147 148 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 149 150 /* 151 * Stage 3 of the trace events. 152 * 153 * Override the macros in the event tracepoint header <trace/events/XXX.h> 154 * to include the following: 155 * 156 * enum print_line_t 157 * trace_raw_output_<call>(struct trace_iterator *iter, int flags) 158 * { 159 * struct trace_seq *s = &iter->seq; 160 * struct trace_event_raw_<call> *field; <-- defined in stage 1 161 * struct trace_seq *p = &iter->tmp_seq; 162 * 163 * -------(for event)------- 164 * 165 * struct trace_entry *entry; 166 * 167 * entry = iter->ent; 168 * 169 * if (entry->type != event_<call>->event.type) { 170 * WARN_ON_ONCE(1); 171 * return TRACE_TYPE_UNHANDLED; 172 * } 173 * 174 * field = (typeof(field))entry; 175 * 176 * trace_seq_init(p); 177 * return trace_output_call(iter, <call>, <TP_printk> "\n"); 178 * 179 * ------(or, for event class)------ 180 * 181 * int ret; 182 * 183 * field = (typeof(field))iter->ent; 184 * 185 * ret = trace_raw_output_prep(iter, trace_event); 186 * if (ret != TRACE_TYPE_HANDLED) 187 * return ret; 188 * 189 * trace_event_printf(iter, <TP_printk> "\n"); 190 * 191 * return trace_handle_return(s); 192 * ------- 193 * } 194 * 195 * This is the method used to print the raw event to the trace 196 * output format. Note, this is not needed if the data is read 197 * in binary. 198 */ 199 200 #include "stages/stage3_trace_output.h" 201 202 #undef DECLARE_EVENT_CLASS 203 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ 204 static notrace enum print_line_t \ 205 trace_raw_output_##call(struct trace_iterator *iter, int flags, \ 206 struct trace_event *trace_event) \ 207 { \ 208 struct trace_seq *s = &iter->seq; \ 209 struct trace_seq __maybe_unused *p = &iter->tmp_seq; \ 210 struct trace_event_raw_##call *field; \ 211 int ret; \ 212 \ 213 field = (typeof(field))iter->ent; \ 214 \ 215 ret = trace_raw_output_prep(iter, trace_event); \ 216 if (ret != TRACE_TYPE_HANDLED) \ 217 return ret; \ 218 \ 219 trace_event_printf(iter, print); \ 220 \ 221 return trace_handle_return(s); \ 222 } \ 223 static struct trace_event_functions trace_event_type_funcs_##call = { \ 224 .trace = trace_raw_output_##call, \ 225 }; 226 227 #undef DECLARE_EVENT_SYSCALL_CLASS 228 #define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS 229 230 #undef DEFINE_EVENT_PRINT 231 #define DEFINE_EVENT_PRINT(template, call, proto, args, print) \ 232 static notrace enum print_line_t \ 233 trace_raw_output_##call(struct trace_iterator *iter, int flags, \ 234 struct trace_event *event) \ 235 { \ 236 struct trace_event_raw_##template *field; \ 237 struct trace_entry *entry; \ 238 struct trace_seq *p = &iter->tmp_seq; \ 239 \ 240 entry = iter->ent; \ 241 \ 242 if (entry->type != event_##call.event.type) { \ 243 WARN_ON_ONCE(1); \ 244 return TRACE_TYPE_UNHANDLED; \ 245 } \ 246 \ 247 field = (typeof(field))entry; \ 248 \ 249 trace_seq_init(p); \ 250 return trace_output_call(iter, #call, print); \ 251 } \ 252 static struct trace_event_functions trace_event_type_funcs_##call = { \ 253 .trace = trace_raw_output_##call, \ 254 }; 255 256 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 257 258 #include "stages/stage4_event_fields.h" 259 260 #undef DECLARE_EVENT_CLASS 261 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, func, print) \ 262 static struct trace_event_fields trace_event_fields_##call[] = { \ 263 tstruct \ 264 {} }; 265 266 #undef DECLARE_EVENT_SYSCALL_CLASS 267 #define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS 268 269 #undef DEFINE_EVENT_PRINT 270 #define DEFINE_EVENT_PRINT(template, name, proto, args, print) 271 272 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 273 274 #include "stages/stage5_get_offsets.h" 275 276 #undef DECLARE_EVENT_CLASS 277 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ 278 static inline notrace int trace_event_get_offsets_##call( \ 279 struct trace_event_data_offsets_##call *__data_offsets, proto) \ 280 { \ 281 int __data_size = 0; \ 282 int __maybe_unused __item_length; \ 283 struct trace_event_raw_##call __maybe_unused *entry; \ 284 \ 285 tstruct; \ 286 \ 287 return __data_size; \ 288 } 289 290 #undef DECLARE_EVENT_SYSCALL_CLASS 291 #define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS 292 293 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 294 295 /* 296 * Stage 4 of the trace events. 297 * 298 * Override the macros in the event tracepoint header <trace/events/XXX.h> 299 * to include the following: 300 * 301 * For those macros defined with TRACE_EVENT: 302 * 303 * static struct trace_event_call event_<call>; 304 * 305 * static void trace_event_raw_event_<call>(void *__data, proto) 306 * { 307 * struct trace_event_file *trace_file = __data; 308 * struct trace_event_call *event_call = trace_file->event_call; 309 * struct trace_event_data_offsets_<call> __maybe_unused __data_offsets; 310 * unsigned long eflags = trace_file->flags; 311 * enum event_trigger_type __tt = ETT_NONE; 312 * struct ring_buffer_event *event; 313 * struct trace_event_raw_<call> *entry; <-- defined in stage 1 314 * struct trace_buffer *buffer; 315 * unsigned long irq_flags; 316 * int __data_size; 317 * int pc; 318 * 319 * if (!(eflags & EVENT_FILE_FL_TRIGGER_COND)) { 320 * if (eflags & EVENT_FILE_FL_TRIGGER_MODE) 321 * event_triggers_call(trace_file, NULL); 322 * if (eflags & EVENT_FILE_FL_SOFT_DISABLED) 323 * return; 324 * } 325 * 326 * local_save_flags(irq_flags); 327 * pc = preempt_count(); 328 * 329 * __data_size = trace_event_get_offsets_<call>(&__data_offsets, args); 330 * 331 * event = trace_event_buffer_lock_reserve(&buffer, trace_file, 332 * event_<call>->event.type, 333 * sizeof(*entry) + __data_size, 334 * irq_flags, pc); 335 * if (!event) 336 * return; 337 * entry = ring_buffer_event_data(event); 338 * 339 * { <assign>; } <-- Here we assign the entries by the __field and 340 * __array macros. 341 * 342 * if (eflags & EVENT_FILE_FL_TRIGGER_COND) 343 * __tt = event_triggers_call(trace_file, entry); 344 * 345 * if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, 346 * &trace_file->flags)) 347 * ring_buffer_discard_commit(buffer, event); 348 * else if (!filter_check_discard(trace_file, entry, buffer, event)) 349 * trace_buffer_unlock_commit(buffer, event, irq_flags, pc); 350 * 351 * if (__tt) 352 * event_triggers_post_call(trace_file, __tt); 353 * } 354 * 355 * static struct trace_event ftrace_event_type_<call> = { 356 * .trace = trace_raw_output_<call>, <-- stage 2 357 * }; 358 * 359 * static char print_fmt_<call>[] = <TP_printk>; 360 * 361 * static struct trace_event_class __used event_class_<template> = { 362 * .system = "<system>", 363 * .fields_array = trace_event_fields_<call>, 364 * .fields = LIST_HEAD_INIT(event_class_##call.fields), 365 * .raw_init = trace_event_raw_init, 366 * .probe = trace_event_raw_event_##call, 367 * .reg = trace_event_reg, 368 * }; 369 * 370 * static struct trace_event_call event_<call> = { 371 * .class = event_class_<template>, 372 * { 373 * .tp = &__tracepoint_<call>, 374 * }, 375 * .event = &ftrace_event_type_<call>, 376 * .print_fmt = print_fmt_<call>, 377 * .flags = TRACE_EVENT_FL_TRACEPOINT, 378 * }; 379 * // its only safe to use pointers when doing linker tricks to 380 * // create an array. 381 * static struct trace_event_call __used 382 * __section("_ftrace_events") *__event_<call> = &event_<call>; 383 * 384 */ 385 386 #ifdef CONFIG_PERF_EVENTS 387 388 #define _TRACE_PERF_PROTO(call, proto) \ 389 static notrace void \ 390 perf_trace_##call(void *__data, proto); 391 392 #define _TRACE_PERF_INIT(call) \ 393 .perf_probe = perf_trace_##call, 394 395 #else 396 #define _TRACE_PERF_PROTO(call, proto) 397 #define _TRACE_PERF_INIT(call) 398 #endif /* CONFIG_PERF_EVENTS */ 399 400 #include "stages/stage6_event_callback.h" 401 402 403 #undef __DECLARE_EVENT_CLASS 404 #define __DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ 405 static notrace void \ 406 do_trace_event_raw_event_##call(void *__data, proto) \ 407 { \ 408 struct trace_event_file *trace_file = __data; \ 409 struct trace_event_data_offsets_##call __maybe_unused __data_offsets;\ 410 struct trace_event_buffer fbuffer; \ 411 struct trace_event_raw_##call *entry; \ 412 int __data_size; \ 413 \ 414 if (trace_trigger_soft_disabled(trace_file)) \ 415 return; \ 416 \ 417 __data_size = trace_event_get_offsets_##call(&__data_offsets, args); \ 418 \ 419 entry = trace_event_buffer_reserve(&fbuffer, trace_file, \ 420 sizeof(*entry) + __data_size); \ 421 \ 422 if (!entry) \ 423 return; \ 424 \ 425 tstruct \ 426 \ 427 { assign; } \ 428 \ 429 trace_event_buffer_commit(&fbuffer); \ 430 } 431 432 #undef DECLARE_EVENT_CLASS 433 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ 434 __DECLARE_EVENT_CLASS(call, PARAMS(proto), PARAMS(args), PARAMS(tstruct), \ 435 PARAMS(assign), PARAMS(print)) \ 436 static notrace void \ 437 trace_event_raw_event_##call(void *__data, proto) \ 438 { \ 439 do_trace_event_raw_event_##call(__data, args); \ 440 } 441 442 #undef DECLARE_EVENT_SYSCALL_CLASS 443 #define DECLARE_EVENT_SYSCALL_CLASS(call, proto, args, tstruct, assign, print) \ 444 __DECLARE_EVENT_CLASS(call, PARAMS(proto), PARAMS(args), PARAMS(tstruct), \ 445 PARAMS(assign), PARAMS(print)) \ 446 static notrace void \ 447 trace_event_raw_event_##call(void *__data, proto) \ 448 { \ 449 might_fault(); \ 450 preempt_disable_notrace(); \ 451 do_trace_event_raw_event_##call(__data, args); \ 452 preempt_enable_notrace(); \ 453 } 454 455 /* 456 * The ftrace_test_probe is compiled out, it is only here as a build time check 457 * to make sure that if the tracepoint handling changes, the ftrace probe will 458 * fail to compile unless it too is updated. 459 */ 460 461 #undef DEFINE_EVENT 462 #define DEFINE_EVENT(template, call, proto, args) \ 463 static inline void ftrace_test_probe_##call(void) \ 464 { \ 465 check_trace_callback_type_##call(trace_event_raw_event_##template); \ 466 } 467 468 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 469 470 #undef __DECLARE_EVENT_CLASS 471 472 #include "stages/stage7_class_define.h" 473 474 #undef DECLARE_EVENT_CLASS 475 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ 476 _TRACE_PERF_PROTO(call, PARAMS(proto)); \ 477 static char print_fmt_##call[] = print; \ 478 static struct trace_event_class __used __refdata event_class_##call = { \ 479 .system = TRACE_SYSTEM_STRING, \ 480 .fields_array = trace_event_fields_##call, \ 481 .fields = LIST_HEAD_INIT(event_class_##call.fields),\ 482 .raw_init = trace_event_raw_init, \ 483 .probe = trace_event_raw_event_##call, \ 484 .reg = trace_event_reg, \ 485 _TRACE_PERF_INIT(call) \ 486 }; 487 488 #undef DECLARE_EVENT_SYSCALL_CLASS 489 #define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS 490 491 #undef DEFINE_EVENT 492 #define DEFINE_EVENT(template, call, proto, args) \ 493 \ 494 static struct trace_event_call __used event_##call = { \ 495 .class = &event_class_##template, \ 496 { \ 497 .tp = &__tracepoint_##call, \ 498 }, \ 499 .event.funcs = &trace_event_type_funcs_##template, \ 500 .print_fmt = print_fmt_##template, \ 501 .flags = TRACE_EVENT_FL_TRACEPOINT, \ 502 }; \ 503 static struct trace_event_call __used \ 504 __section("_ftrace_events") *__event_##call = &event_##call 505 506 #undef DEFINE_EVENT_PRINT 507 #define DEFINE_EVENT_PRINT(template, call, proto, args, print) \ 508 \ 509 static char print_fmt_##call[] = print; \ 510 \ 511 static struct trace_event_call __used event_##call = { \ 512 .class = &event_class_##template, \ 513 { \ 514 .tp = &__tracepoint_##call, \ 515 }, \ 516 .event.funcs = &trace_event_type_funcs_##call, \ 517 .print_fmt = print_fmt_##call, \ 518 .flags = TRACE_EVENT_FL_TRACEPOINT, \ 519 }; \ 520 static struct trace_event_call __used \ 521 __section("_ftrace_events") *__event_##call = &event_##call 522 523 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 524