1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Stage 1 of the trace events. 4 * 5 * Override the macros in the event tracepoint header <trace/events/XXX.h> 6 * to include the following: 7 * 8 * struct trace_event_raw_<call> { 9 * struct trace_entry ent; 10 * <type> <item>; 11 * <type2> <item2>[<len>]; 12 * [...] 13 * }; 14 * 15 * The <type> <item> is created by the __field(type, item) macro or 16 * the __array(type2, item2, len) macro. 17 * We simply do "type item;", and that will create the fields 18 * in the structure. 19 */ 20 21 #include <linux/trace_events.h> 22 #include <linux/btf_ids.h> 23 24 #ifndef TRACE_SYSTEM_VAR 25 #define TRACE_SYSTEM_VAR TRACE_SYSTEM 26 #endif 27 28 #include "stages/init.h" 29 30 /* 31 * DECLARE_EVENT_CLASS can be used to add a generic function 32 * handlers for events. That is, if all events have the same 33 * parameters and just have distinct trace points. 34 * Each tracepoint can be defined with DEFINE_EVENT and that 35 * will map the DECLARE_EVENT_CLASS to the tracepoint. 36 * 37 * TRACE_EVENT is a one to one mapping between tracepoint and template. 38 */ 39 #undef TRACE_EVENT 40 #define TRACE_EVENT(name, proto, args, tstruct, assign, print) \ 41 DECLARE_EVENT_CLASS(name, \ 42 PARAMS(proto), \ 43 PARAMS(args), \ 44 PARAMS(tstruct), \ 45 PARAMS(assign), \ 46 PARAMS(print)); \ 47 DEFINE_EVENT(name, name, PARAMS(proto), PARAMS(args)); 48 49 #undef TRACE_EVENT_SYSCALL 50 #define TRACE_EVENT_SYSCALL(name, proto, args, tstruct, assign, print, reg, unreg) \ 51 DECLARE_EVENT_SYSCALL_CLASS(name, \ 52 PARAMS(proto), \ 53 PARAMS(args), \ 54 PARAMS(tstruct), \ 55 PARAMS(assign), \ 56 PARAMS(print)); \ 57 DEFINE_EVENT(name, name, PARAMS(proto), PARAMS(args)); 58 59 #include "stages/stage1_struct_define.h" 60 61 #undef DECLARE_EVENT_CLASS 62 #define DECLARE_EVENT_CLASS(name, proto, args, tstruct, assign, print) \ 63 struct trace_event_raw_##name { \ 64 struct trace_entry ent; \ 65 tstruct \ 66 char __data[]; \ 67 }; \ 68 \ 69 static struct trace_event_class event_class_##name; 70 71 #undef DECLARE_EVENT_SYSCALL_CLASS 72 #define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS 73 74 #undef DEFINE_EVENT 75 #define DEFINE_EVENT(template, name, proto, args) \ 76 static struct trace_event_call __used \ 77 __attribute__((__aligned__(4))) event_##name 78 79 #undef DEFINE_EVENT_FN 80 #define DEFINE_EVENT_FN(template, name, proto, args, reg, unreg) \ 81 DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args)) 82 83 #undef DEFINE_EVENT_PRINT 84 #define DEFINE_EVENT_PRINT(template, name, proto, args, print) \ 85 DEFINE_EVENT(template, name, PARAMS(proto), PARAMS(args)) 86 87 /* Callbacks are meaningless to ftrace. */ 88 #undef TRACE_EVENT_FN 89 #define TRACE_EVENT_FN(name, proto, args, tstruct, \ 90 assign, print, reg, unreg) \ 91 TRACE_EVENT(name, PARAMS(proto), PARAMS(args), \ 92 PARAMS(tstruct), PARAMS(assign), PARAMS(print)) \ 93 94 #undef TRACE_EVENT_FN_COND 95 #define TRACE_EVENT_FN_COND(name, proto, args, cond, tstruct, \ 96 assign, print, reg, unreg) \ 97 TRACE_EVENT_CONDITION(name, PARAMS(proto), PARAMS(args), PARAMS(cond), \ 98 PARAMS(tstruct), PARAMS(assign), PARAMS(print)) \ 99 100 #undef TRACE_EVENT_FLAGS 101 #define TRACE_EVENT_FLAGS(name, value) \ 102 __TRACE_EVENT_FLAGS(name, value) 103 104 #undef TRACE_EVENT_PERF_PERM 105 #define TRACE_EVENT_PERF_PERM(name, expr...) \ 106 __TRACE_EVENT_PERF_PERM(name, expr) 107 108 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 109 110 /* 111 * Stage 2 of the trace events. 112 * 113 * Include the following: 114 * 115 * struct trace_event_data_offsets_<call> { 116 * u32 <item1>; 117 * u32 <item2>; 118 * [...] 119 * }; 120 * 121 * The __dynamic_array() macro will create each u32 <item>, this is 122 * to keep the offset of each array from the beginning of the event. 123 * The size of an array is also encoded, in the higher 16 bits of <item>. 124 */ 125 126 #include "stages/stage2_data_offsets.h" 127 128 #undef DECLARE_EVENT_CLASS 129 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ 130 struct trace_event_data_offsets_##call { \ 131 tstruct; \ 132 }; 133 134 #undef DECLARE_EVENT_SYSCALL_CLASS 135 #define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS 136 137 #undef DEFINE_EVENT 138 #define DEFINE_EVENT(template, name, proto, args) 139 140 #undef DEFINE_EVENT_PRINT 141 #define DEFINE_EVENT_PRINT(template, name, proto, args, print) 142 143 #undef TRACE_EVENT_FLAGS 144 #define TRACE_EVENT_FLAGS(event, flag) 145 146 #undef TRACE_EVENT_PERF_PERM 147 #define TRACE_EVENT_PERF_PERM(event, expr...) 148 149 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 150 151 /* 152 * Stage 3 of the trace events. 153 * 154 * Override the macros in the event tracepoint header <trace/events/XXX.h> 155 * to include the following: 156 * 157 * enum print_line_t 158 * trace_raw_output_<call>(struct trace_iterator *iter, int flags) 159 * { 160 * struct trace_seq *s = &iter->seq; 161 * struct trace_event_raw_<call> *field; <-- defined in stage 1 162 * struct trace_seq *p = &iter->tmp_seq; 163 * 164 * -------(for event)------- 165 * 166 * struct trace_entry *entry; 167 * 168 * entry = iter->ent; 169 * 170 * if (entry->type != event_<call>->event.type) { 171 * WARN_ON_ONCE(1); 172 * return TRACE_TYPE_UNHANDLED; 173 * } 174 * 175 * field = (typeof(field))entry; 176 * 177 * trace_seq_init(p); 178 * return trace_output_call(iter, <call>, <TP_printk> "\n"); 179 * 180 * ------(or, for event class)------ 181 * 182 * int ret; 183 * 184 * field = (typeof(field))iter->ent; 185 * 186 * ret = trace_raw_output_prep(iter, trace_event); 187 * if (ret != TRACE_TYPE_HANDLED) 188 * return ret; 189 * 190 * trace_event_printf(iter, <TP_printk> "\n"); 191 * 192 * return trace_handle_return(s); 193 * ------- 194 * } 195 * 196 * This is the method used to print the raw event to the trace 197 * output format. Note, this is not needed if the data is read 198 * in binary. 199 */ 200 201 #include "stages/stage3_trace_output.h" 202 203 #undef DECLARE_EVENT_CLASS 204 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ 205 static notrace enum print_line_t \ 206 trace_raw_output_##call(struct trace_iterator *iter, int flags, \ 207 struct trace_event *trace_event) \ 208 { \ 209 struct trace_seq *s = &iter->seq; \ 210 struct trace_seq __maybe_unused *p = &iter->tmp_seq; \ 211 struct trace_event_raw_##call *field; \ 212 int ret; \ 213 \ 214 field = (typeof(field))iter->ent; \ 215 \ 216 ret = trace_raw_output_prep(iter, trace_event); \ 217 if (ret != TRACE_TYPE_HANDLED) \ 218 return ret; \ 219 \ 220 trace_event_printf(iter, print); \ 221 \ 222 return trace_handle_return(s); \ 223 } \ 224 static struct trace_event_functions trace_event_type_funcs_##call = { \ 225 .trace = trace_raw_output_##call, \ 226 }; 227 228 #undef DECLARE_EVENT_SYSCALL_CLASS 229 #define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS 230 231 #undef DEFINE_EVENT_PRINT 232 #define DEFINE_EVENT_PRINT(template, call, proto, args, print) \ 233 static notrace enum print_line_t \ 234 trace_raw_output_##call(struct trace_iterator *iter, int flags, \ 235 struct trace_event *event) \ 236 { \ 237 struct trace_event_raw_##template *field; \ 238 struct trace_entry *entry; \ 239 struct trace_seq *p = &iter->tmp_seq; \ 240 \ 241 entry = iter->ent; \ 242 \ 243 if (entry->type != event_##call.event.type) { \ 244 WARN_ON_ONCE(1); \ 245 return TRACE_TYPE_UNHANDLED; \ 246 } \ 247 \ 248 field = (typeof(field))entry; \ 249 \ 250 trace_seq_init(p); \ 251 return trace_output_call(iter, #call, print); \ 252 } \ 253 static struct trace_event_functions trace_event_type_funcs_##call = { \ 254 .trace = trace_raw_output_##call, \ 255 }; 256 257 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 258 259 #include "stages/stage4_event_fields.h" 260 261 #undef DECLARE_EVENT_CLASS 262 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, func, print) \ 263 static struct trace_event_fields trace_event_fields_##call[] = { \ 264 tstruct \ 265 {} }; 266 267 #undef DECLARE_EVENT_SYSCALL_CLASS 268 #define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS 269 270 #undef DEFINE_EVENT_PRINT 271 #define DEFINE_EVENT_PRINT(template, name, proto, args, print) 272 273 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 274 275 #include "stages/stage5_get_offsets.h" 276 277 #undef DECLARE_EVENT_CLASS 278 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ 279 static inline notrace int trace_event_get_offsets_##call( \ 280 struct trace_event_data_offsets_##call *__data_offsets, proto) \ 281 { \ 282 int __data_size = 0; \ 283 int __maybe_unused __item_length; \ 284 struct trace_event_raw_##call __maybe_unused *entry; \ 285 \ 286 tstruct; \ 287 \ 288 return __data_size; \ 289 } 290 291 #undef DECLARE_EVENT_SYSCALL_CLASS 292 #define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS 293 294 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 295 296 /* 297 * Stage 4 of the trace events. 298 * 299 * Override the macros in the event tracepoint header <trace/events/XXX.h> 300 * to include the following: 301 * 302 * For those macros defined with TRACE_EVENT: 303 * 304 * static struct trace_event_call event_<call>; 305 * 306 * static void trace_event_raw_event_<call>(void *__data, proto) 307 * { 308 * struct trace_event_file *trace_file = __data; 309 * struct trace_event_call *event_call = trace_file->event_call; 310 * struct trace_event_data_offsets_<call> __maybe_unused __data_offsets; 311 * unsigned long eflags = trace_file->flags; 312 * enum event_trigger_type __tt = ETT_NONE; 313 * struct ring_buffer_event *event; 314 * struct trace_event_raw_<call> *entry; <-- defined in stage 1 315 * struct trace_buffer *buffer; 316 * unsigned long irq_flags; 317 * int __data_size; 318 * int pc; 319 * 320 * if (!(eflags & EVENT_FILE_FL_TRIGGER_COND)) { 321 * if (eflags & EVENT_FILE_FL_TRIGGER_MODE) 322 * event_triggers_call(trace_file, NULL); 323 * if (eflags & EVENT_FILE_FL_SOFT_DISABLED) 324 * return; 325 * } 326 * 327 * local_save_flags(irq_flags); 328 * pc = preempt_count(); 329 * 330 * __data_size = trace_event_get_offsets_<call>(&__data_offsets, args); 331 * 332 * event = trace_event_buffer_lock_reserve(&buffer, trace_file, 333 * event_<call>->event.type, 334 * sizeof(*entry) + __data_size, 335 * irq_flags, pc); 336 * if (!event) 337 * return; 338 * entry = ring_buffer_event_data(event); 339 * 340 * { <assign>; } <-- Here we assign the entries by the __field and 341 * __array macros. 342 * 343 * if (eflags & EVENT_FILE_FL_TRIGGER_COND) 344 * __tt = event_triggers_call(trace_file, entry); 345 * 346 * if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, 347 * &trace_file->flags)) 348 * ring_buffer_discard_commit(buffer, event); 349 * else if (!filter_check_discard(trace_file, entry, buffer, event)) 350 * trace_buffer_unlock_commit(buffer, event, irq_flags, pc); 351 * 352 * if (__tt) 353 * event_triggers_post_call(trace_file, __tt); 354 * } 355 * 356 * static struct trace_event ftrace_event_type_<call> = { 357 * .trace = trace_raw_output_<call>, <-- stage 2 358 * }; 359 * 360 * static char print_fmt_<call>[] = <TP_printk>; 361 * 362 * static struct trace_event_class __used event_class_<template> = { 363 * .system = "<system>", 364 * .fields_array = trace_event_fields_<call>, 365 * .fields = LIST_HEAD_INIT(event_class_##call.fields), 366 * .raw_init = trace_event_raw_init, 367 * .probe = trace_event_raw_event_##call, 368 * .reg = trace_event_reg, 369 * }; 370 * 371 * static struct trace_event_call event_<call> = { 372 * .class = event_class_<template>, 373 * { 374 * .tp = &__tracepoint_<call>, 375 * }, 376 * .event = &ftrace_event_type_<call>, 377 * .print_fmt = print_fmt_<call>, 378 * .flags = TRACE_EVENT_FL_TRACEPOINT, 379 * }; 380 * // its only safe to use pointers when doing linker tricks to 381 * // create an array. 382 * static struct trace_event_call __used 383 * __section("_ftrace_events") *__event_<call> = &event_<call>; 384 * 385 */ 386 387 #ifdef CONFIG_PERF_EVENTS 388 389 #define _TRACE_PERF_PROTO(call, proto) \ 390 static notrace void \ 391 perf_trace_##call(void *__data, proto); 392 393 #define _TRACE_PERF_INIT(call) \ 394 .perf_probe = perf_trace_##call, 395 396 #else 397 #define _TRACE_PERF_PROTO(call, proto) 398 #define _TRACE_PERF_INIT(call) 399 #endif /* CONFIG_PERF_EVENTS */ 400 401 #if defined(CONFIG_BPF_EVENTS) && defined(CONFIG_DEBUG_INFO_BTF) 402 /* 403 * Per-template BTF id list, populated at link time by resolve_btfids: 404 * [0] FUNC __bpf_trace_<call> (the BPF dispatcher) 405 * [1] STRUCT trace_event_raw_<call> (the ring-buffer record) 406 * Exposed via the events/<sys>/<name>/btf_ids tracefs file. 407 */ 408 #define _TRACE_BTF_IDS_DECLARE(call) \ 409 BTF_ID_LIST(__bpf_trace_btf_ids_##call) \ 410 BTF_ID(func, __bpf_trace_##call) \ 411 BTF_ID(struct, trace_event_raw_##call) 412 413 #define _TRACE_BTF_IDS_INIT(call) \ 414 .btf_ids = __bpf_trace_btf_ids_##call, 415 416 #else 417 #define _TRACE_BTF_IDS_DECLARE(call) 418 #define _TRACE_BTF_IDS_INIT(call) 419 #endif /* CONFIG_BPF_EVENTS && CONFIG_DEBUG_INFO_BTF */ 420 421 #include "stages/stage6_event_callback.h" 422 423 424 #undef __DECLARE_EVENT_CLASS 425 #define __DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ 426 static notrace void \ 427 do_trace_event_raw_event_##call(void *__data, proto) \ 428 { \ 429 struct trace_event_file *trace_file = __data; \ 430 struct trace_event_data_offsets_##call __maybe_unused __data_offsets;\ 431 struct trace_event_buffer fbuffer; \ 432 struct trace_event_raw_##call *entry; \ 433 int __data_size; \ 434 \ 435 if (trace_trigger_soft_disabled(trace_file)) \ 436 return; \ 437 \ 438 __data_size = trace_event_get_offsets_##call(&__data_offsets, args); \ 439 \ 440 entry = trace_event_buffer_reserve(&fbuffer, trace_file, \ 441 sizeof(*entry) + __data_size); \ 442 \ 443 if (!entry) \ 444 return; \ 445 \ 446 tstruct \ 447 \ 448 { assign; } \ 449 \ 450 trace_event_buffer_commit(&fbuffer); \ 451 } 452 453 #undef DECLARE_EVENT_CLASS 454 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ 455 __DECLARE_EVENT_CLASS(call, PARAMS(proto), PARAMS(args), PARAMS(tstruct), \ 456 PARAMS(assign), PARAMS(print)) \ 457 static notrace void \ 458 trace_event_raw_event_##call(void *__data, proto) \ 459 { \ 460 guard(preempt_notrace)(); \ 461 do_trace_event_raw_event_##call(__data, args); \ 462 } 463 464 #undef DECLARE_EVENT_SYSCALL_CLASS 465 #define DECLARE_EVENT_SYSCALL_CLASS(call, proto, args, tstruct, assign, print) \ 466 __DECLARE_EVENT_CLASS(call, PARAMS(proto), PARAMS(args), PARAMS(tstruct), \ 467 PARAMS(assign), PARAMS(print)) \ 468 static notrace void \ 469 trace_event_raw_event_##call(void *__data, proto) \ 470 { \ 471 might_fault(); \ 472 guard(preempt_notrace)(); \ 473 do_trace_event_raw_event_##call(__data, args); \ 474 } 475 476 /* 477 * The ftrace_test_probe is compiled out, it is only here as a build time check 478 * to make sure that if the tracepoint handling changes, the ftrace probe will 479 * fail to compile unless it too is updated. 480 */ 481 482 #undef DEFINE_EVENT 483 #define DEFINE_EVENT(template, call, proto, args) \ 484 static inline void ftrace_test_probe_##call(void) \ 485 { \ 486 check_trace_callback_type_##call(trace_event_raw_event_##template); \ 487 } 488 489 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 490 491 #undef __DECLARE_EVENT_CLASS 492 493 #include "stages/stage7_class_define.h" 494 495 #undef DECLARE_EVENT_CLASS 496 #define DECLARE_EVENT_CLASS(call, proto, args, tstruct, assign, print) \ 497 _TRACE_PERF_PROTO(call, PARAMS(proto)); \ 498 _TRACE_BTF_IDS_DECLARE(call) \ 499 static char print_fmt_##call[] = print; \ 500 static struct trace_event_class __used __refdata event_class_##call = { \ 501 .system = TRACE_SYSTEM_STRING, \ 502 .fields_array = trace_event_fields_##call, \ 503 .fields = LIST_HEAD_INIT(event_class_##call.fields),\ 504 .raw_init = trace_event_raw_init, \ 505 .probe = trace_event_raw_event_##call, \ 506 .reg = trace_event_reg, \ 507 _TRACE_PERF_INIT(call) \ 508 _TRACE_BTF_IDS_INIT(call) \ 509 }; 510 511 #undef DECLARE_EVENT_SYSCALL_CLASS 512 #define DECLARE_EVENT_SYSCALL_CLASS DECLARE_EVENT_CLASS 513 514 #undef DEFINE_EVENT 515 #define DEFINE_EVENT(template, call, proto, args) \ 516 \ 517 static struct trace_event_call __used event_##call = { \ 518 .class = &event_class_##template, \ 519 { \ 520 .tp = &__tracepoint_##call, \ 521 }, \ 522 .event.funcs = &trace_event_type_funcs_##template, \ 523 .print_fmt = print_fmt_##template, \ 524 .flags = TRACE_EVENT_FL_TRACEPOINT, \ 525 }; \ 526 static struct trace_event_call __used \ 527 __section("_ftrace_events") *__event_##call = &event_##call 528 529 #undef DEFINE_EVENT_PRINT 530 #define DEFINE_EVENT_PRINT(template, call, proto, args, print) \ 531 \ 532 static char print_fmt_##call[] = print; \ 533 \ 534 static struct trace_event_call __used event_##call = { \ 535 .class = &event_class_##template, \ 536 { \ 537 .tp = &__tracepoint_##call, \ 538 }, \ 539 .event.funcs = &trace_event_type_funcs_##call, \ 540 .print_fmt = print_fmt_##call, \ 541 .flags = TRACE_EVENT_FL_TRACEPOINT, \ 542 }; \ 543 static struct trace_event_call __used \ 544 __section("_ftrace_events") *__event_##call = &event_##call 545 546 #include TRACE_INCLUDE(TRACE_INCLUDE_FILE) 547