Lines Matching full:tr
102 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) in dummy_set_flag() argument
218 int tracing_set_tracer(struct trace_array *tr, const char *buf);
219 static void ftrace_trace_userstack(struct trace_array *tr,
545 static __always_inline bool printk_binsafe(struct trace_array *tr) in printk_binsafe() argument
553 return !(tr->flags & TRACE_ARRAY_FL_BOOT); in printk_binsafe()
556 static void update_printk_trace(struct trace_array *tr) in update_printk_trace() argument
558 if (printk_trace == tr) in update_printk_trace()
562 printk_trace = tr; in update_printk_trace()
563 tr->trace_flags |= TRACE_ITER_TRACE_PRINTK; in update_printk_trace()
566 /* Returns true if the status of tr changed */
567 static bool update_marker_trace(struct trace_array *tr, int enabled) in update_marker_trace() argument
572 if (!list_empty(&tr->marker_list)) in update_marker_trace()
575 list_add_rcu(&tr->marker_list, &marker_copies); in update_marker_trace()
576 tr->trace_flags |= TRACE_ITER_COPY_MARKER; in update_marker_trace()
580 if (list_empty(&tr->marker_list)) in update_marker_trace()
583 list_del_init(&tr->marker_list); in update_marker_trace()
584 tr->trace_flags &= ~TRACE_ITER_COPY_MARKER; in update_marker_trace()
588 void trace_set_ring_buffer_expanded(struct trace_array *tr) in trace_set_ring_buffer_expanded() argument
590 if (!tr) in trace_set_ring_buffer_expanded()
591 tr = &global_trace; in trace_set_ring_buffer_expanded()
592 tr->ring_buffer_expanded = true; in trace_set_ring_buffer_expanded()
599 struct trace_array *tr; in trace_array_get() local
602 list_for_each_entry(tr, &ftrace_trace_arrays, list) { in trace_array_get()
603 if (tr == this_tr) { in trace_array_get()
604 tr->ref++; in trace_array_get()
637 int tracing_check_open_get_tr(struct trace_array *tr) in tracing_check_open_get_tr() argument
648 if (tr && trace_array_get(tr) < 0) in tracing_check_open_get_tr()
1039 static void __ftrace_trace_stack(struct trace_array *tr,
1043 static inline void ftrace_trace_stack(struct trace_array *tr,
1049 static inline void __ftrace_trace_stack(struct trace_array *tr, in __ftrace_trace_stack() argument
1055 static inline void ftrace_trace_stack(struct trace_array *tr, in ftrace_trace_stack() argument
1088 void tracer_tracing_on(struct trace_array *tr) in tracer_tracing_on() argument
1090 if (tr->array_buffer.buffer) in tracer_tracing_on()
1091 ring_buffer_record_on(tr->array_buffer.buffer); in tracer_tracing_on()
1100 tr->buffer_disabled = 0; in tracer_tracing_on()
1133 int __trace_array_puts(struct trace_array *tr, unsigned long ip, in __trace_array_puts() argument
1142 if (!(tr->trace_flags & TRACE_ITER_PRINTK)) in __trace_array_puts()
1145 if (unlikely(tracing_selftest_running && tr == &global_trace)) in __trace_array_puts()
1154 buffer = tr->array_buffer.buffer; in __trace_array_puts()
1174 ftrace_trace_stack(tr, buffer, trace_ctx, 4, NULL); in __trace_array_puts()
1198 struct trace_array *tr = READ_ONCE(printk_trace); in __trace_bputs() local
1205 if (!printk_binsafe(tr)) in __trace_bputs()
1208 if (!(tr->trace_flags & TRACE_ITER_PRINTK)) in __trace_bputs()
1215 buffer = tr->array_buffer.buffer; in __trace_bputs()
1228 ftrace_trace_stack(tr, buffer, trace_ctx, 4, NULL); in __trace_bputs()
1235 static void tracing_snapshot_instance_cond(struct trace_array *tr, in tracing_snapshot_instance_cond() argument
1238 struct tracer *tracer = tr->current_trace; in tracing_snapshot_instance_cond()
1242 trace_array_puts(tr, "*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n"); in tracing_snapshot_instance_cond()
1243 trace_array_puts(tr, "*** snapshot is being ignored ***\n"); in tracing_snapshot_instance_cond()
1247 if (!tr->allocated_snapshot) { in tracing_snapshot_instance_cond()
1248 trace_array_puts(tr, "*** SNAPSHOT NOT ALLOCATED ***\n"); in tracing_snapshot_instance_cond()
1249 trace_array_puts(tr, "*** stopping trace here! ***\n"); in tracing_snapshot_instance_cond()
1250 tracer_tracing_off(tr); in tracing_snapshot_instance_cond()
1256 trace_array_puts(tr, "*** LATENCY TRACER ACTIVE ***\n"); in tracing_snapshot_instance_cond()
1257 trace_array_puts(tr, "*** Can not use snapshot (sorry) ***\n"); in tracing_snapshot_instance_cond()
1261 if (tr->mapped) { in tracing_snapshot_instance_cond()
1262 trace_array_puts(tr, "*** BUFFER MEMORY MAPPED ***\n"); in tracing_snapshot_instance_cond()
1263 trace_array_puts(tr, "*** Can not use snapshot (sorry) ***\n"); in tracing_snapshot_instance_cond()
1268 update_max_tr(tr, current, smp_processor_id(), cond_data); in tracing_snapshot_instance_cond()
1272 void tracing_snapshot_instance(struct trace_array *tr) in tracing_snapshot_instance() argument
1274 tracing_snapshot_instance_cond(tr, NULL); in tracing_snapshot_instance()
1293 struct trace_array *tr = &global_trace; in tracing_snapshot() local
1295 tracing_snapshot_instance(tr); in tracing_snapshot()
1301 * @tr: The tracing instance to snapshot
1312 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data) in tracing_snapshot_cond() argument
1314 tracing_snapshot_instance_cond(tr, cond_data); in tracing_snapshot_cond()
1320 * @tr: The tracing instance
1327 * the tr->max_lock lock, which the code calling
1332 void *tracing_cond_snapshot_data(struct trace_array *tr) in tracing_cond_snapshot_data() argument
1337 arch_spin_lock(&tr->max_lock); in tracing_cond_snapshot_data()
1339 if (tr->cond_snapshot) in tracing_cond_snapshot_data()
1340 cond_data = tr->cond_snapshot->cond_data; in tracing_cond_snapshot_data()
1342 arch_spin_unlock(&tr->max_lock); in tracing_cond_snapshot_data()
1353 int tracing_alloc_snapshot_instance(struct trace_array *tr) in tracing_alloc_snapshot_instance() argument
1358 if (!tr->allocated_snapshot) { in tracing_alloc_snapshot_instance()
1361 order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); in tracing_alloc_snapshot_instance()
1362 ret = ring_buffer_subbuf_order_set(tr->max_buffer.buffer, order); in tracing_alloc_snapshot_instance()
1367 ret = resize_buffer_duplicate_size(&tr->max_buffer, in tracing_alloc_snapshot_instance()
1368 &tr->array_buffer, RING_BUFFER_ALL_CPUS); in tracing_alloc_snapshot_instance()
1372 tr->allocated_snapshot = true; in tracing_alloc_snapshot_instance()
1378 static void free_snapshot(struct trace_array *tr) in free_snapshot() argument
1385 ring_buffer_subbuf_order_set(tr->max_buffer.buffer, 0); in free_snapshot()
1386 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS); in free_snapshot()
1387 set_buffer_entries(&tr->max_buffer, 1); in free_snapshot()
1388 tracing_reset_online_cpus(&tr->max_buffer); in free_snapshot()
1389 tr->allocated_snapshot = false; in free_snapshot()
1392 static int tracing_arm_snapshot_locked(struct trace_array *tr) in tracing_arm_snapshot_locked() argument
1398 spin_lock(&tr->snapshot_trigger_lock); in tracing_arm_snapshot_locked()
1399 if (tr->snapshot == UINT_MAX || tr->mapped) { in tracing_arm_snapshot_locked()
1400 spin_unlock(&tr->snapshot_trigger_lock); in tracing_arm_snapshot_locked()
1404 tr->snapshot++; in tracing_arm_snapshot_locked()
1405 spin_unlock(&tr->snapshot_trigger_lock); in tracing_arm_snapshot_locked()
1407 ret = tracing_alloc_snapshot_instance(tr); in tracing_arm_snapshot_locked()
1409 spin_lock(&tr->snapshot_trigger_lock); in tracing_arm_snapshot_locked()
1410 tr->snapshot--; in tracing_arm_snapshot_locked()
1411 spin_unlock(&tr->snapshot_trigger_lock); in tracing_arm_snapshot_locked()
1417 int tracing_arm_snapshot(struct trace_array *tr) in tracing_arm_snapshot() argument
1420 return tracing_arm_snapshot_locked(tr); in tracing_arm_snapshot()
1423 void tracing_disarm_snapshot(struct trace_array *tr) in tracing_disarm_snapshot() argument
1425 spin_lock(&tr->snapshot_trigger_lock); in tracing_disarm_snapshot()
1426 if (!WARN_ON(!tr->snapshot)) in tracing_disarm_snapshot()
1427 tr->snapshot--; in tracing_disarm_snapshot()
1428 spin_unlock(&tr->snapshot_trigger_lock); in tracing_disarm_snapshot()
1443 struct trace_array *tr = &global_trace; in tracing_alloc_snapshot() local
1446 ret = tracing_alloc_snapshot_instance(tr); in tracing_alloc_snapshot()
1478 * @tr: The tracing instance
1489 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, in tracing_snapshot_cond_enable() argument
1504 if (tr->current_trace->use_max_tr) in tracing_snapshot_cond_enable()
1515 if (tr->cond_snapshot) in tracing_snapshot_cond_enable()
1518 ret = tracing_arm_snapshot_locked(tr); in tracing_snapshot_cond_enable()
1523 arch_spin_lock(&tr->max_lock); in tracing_snapshot_cond_enable()
1524 tr->cond_snapshot = no_free_ptr(cond_snapshot); in tracing_snapshot_cond_enable()
1525 arch_spin_unlock(&tr->max_lock); in tracing_snapshot_cond_enable()
1534 * @tr: The tracing instance
1542 int tracing_snapshot_cond_disable(struct trace_array *tr) in tracing_snapshot_cond_disable() argument
1547 arch_spin_lock(&tr->max_lock); in tracing_snapshot_cond_disable()
1549 if (!tr->cond_snapshot) in tracing_snapshot_cond_disable()
1552 kfree(tr->cond_snapshot); in tracing_snapshot_cond_disable()
1553 tr->cond_snapshot = NULL; in tracing_snapshot_cond_disable()
1556 arch_spin_unlock(&tr->max_lock); in tracing_snapshot_cond_disable()
1559 tracing_disarm_snapshot(tr); in tracing_snapshot_cond_disable()
1570 void tracing_snapshot_cond(struct trace_array *tr, void *cond_data) in tracing_snapshot_cond() argument
1587 void *tracing_cond_snapshot_data(struct trace_array *tr) in tracing_cond_snapshot_data() argument
1592 int tracing_snapshot_cond_enable(struct trace_array *tr, void *cond_data, cond_update_fn_t update) in tracing_snapshot_cond_enable() argument
1597 int tracing_snapshot_cond_disable(struct trace_array *tr) in tracing_snapshot_cond_disable() argument
1602 #define free_snapshot(tr) do { } while (0) argument
1603 #define tracing_arm_snapshot_locked(tr) ({ -EBUSY; }) argument
1606 void tracer_tracing_off(struct trace_array *tr) in tracer_tracing_off() argument
1608 if (tr->array_buffer.buffer) in tracer_tracing_off()
1609 ring_buffer_record_off(tr->array_buffer.buffer); in tracer_tracing_off()
1618 tr->buffer_disabled = 1; in tracer_tracing_off()
1623 * @tr: The trace array to disable its buffer for
1631 void tracer_tracing_disable(struct trace_array *tr) in tracer_tracing_disable() argument
1633 if (WARN_ON_ONCE(!tr->array_buffer.buffer)) in tracer_tracing_disable()
1636 ring_buffer_record_disable(tr->array_buffer.buffer); in tracer_tracing_disable()
1641 * @tr: The trace array that had tracer_tracincg_disable() called on it
1643 * This is called after tracer_tracing_disable() has been called on @tr,
1646 void tracer_tracing_enable(struct trace_array *tr) in tracer_tracing_enable() argument
1648 if (WARN_ON_ONCE(!tr->array_buffer.buffer)) in tracer_tracing_enable()
1651 ring_buffer_record_enable(tr->array_buffer.buffer); in tracer_tracing_enable()
1679 * @tr : the trace array to know if ring buffer is enabled
1683 bool tracer_tracing_is_on(struct trace_array *tr) in tracer_tracing_is_on() argument
1685 if (tr->array_buffer.buffer) in tracer_tracing_is_on()
1686 return ring_buffer_record_is_set_on(tr->array_buffer.buffer); in tracer_tracing_is_on()
1687 return !tr->buffer_disabled; in tracer_tracing_is_on()
1768 bool trace_clock_in_ns(struct trace_array *tr) in trace_clock_in_ns() argument
1770 if (trace_clocks[tr->clock_id].in_ns) in trace_clock_in_ns()
1916 struct trace_array *tr = container_of(work, struct trace_array, in latency_fsnotify_workfn() local
1918 fsnotify_inode(tr->d_max_latency->d_inode, FS_MODIFY); in latency_fsnotify_workfn()
1923 struct trace_array *tr = container_of(iwork, struct trace_array, in latency_fsnotify_workfn_irq() local
1925 queue_work(fsnotify_wq, &tr->fsnotify_work); in latency_fsnotify_workfn_irq()
1928 static void trace_create_maxlat_file(struct trace_array *tr, in trace_create_maxlat_file() argument
1931 INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn); in trace_create_maxlat_file()
1932 init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq); in trace_create_maxlat_file()
1933 tr->d_max_latency = trace_create_file("tracing_max_latency", in trace_create_maxlat_file()
1935 d_tracer, tr, in trace_create_maxlat_file()
1952 void latency_fsnotify(struct trace_array *tr) in latency_fsnotify() argument
1957 * We cannot call queue_work(&tr->fsnotify_work) from here because it's in latency_fsnotify()
1961 irq_work_queue(&tr->fsnotify_irqwork); in latency_fsnotify()
1966 #define trace_create_maxlat_file(tr, d_tracer) \ argument
1968 d_tracer, tr, &tracing_max_lat_fops)
1978 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) in __update_max_tr() argument
1980 struct array_buffer *trace_buf = &tr->array_buffer; in __update_max_tr()
1981 struct array_buffer *max_buf = &tr->max_buffer; in __update_max_tr()
1988 max_data->saved_latency = tr->max_latency; in __update_max_tr()
2009 latency_fsnotify(tr); in __update_max_tr()
2014 * @tr: tracer
2019 * Flip the buffers between the @tr and the max_tr and record information
2023 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu, in update_max_tr() argument
2026 if (tr->stop_count) in update_max_tr()
2031 if (!tr->allocated_snapshot) { in update_max_tr()
2033 WARN_ON_ONCE(tr->current_trace != &nop_trace); in update_max_tr()
2037 arch_spin_lock(&tr->max_lock); in update_max_tr()
2040 if (ring_buffer_record_is_set_on(tr->array_buffer.buffer)) in update_max_tr()
2041 ring_buffer_record_on(tr->max_buffer.buffer); in update_max_tr()
2043 ring_buffer_record_off(tr->max_buffer.buffer); in update_max_tr()
2046 if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data)) { in update_max_tr()
2047 arch_spin_unlock(&tr->max_lock); in update_max_tr()
2051 swap(tr->array_buffer.buffer, tr->max_buffer.buffer); in update_max_tr()
2053 __update_max_tr(tr, tsk, cpu); in update_max_tr()
2055 arch_spin_unlock(&tr->max_lock); in update_max_tr()
2058 ring_buffer_wake_waiters(tr->array_buffer.buffer, RING_BUFFER_ALL_CPUS); in update_max_tr()
2063 * @tr: tracer
2067 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
2070 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu) in update_max_tr_single() argument
2074 if (tr->stop_count) in update_max_tr_single()
2078 if (!tr->allocated_snapshot) { in update_max_tr_single()
2080 WARN_ON_ONCE(tr->current_trace != &nop_trace); in update_max_tr_single()
2084 arch_spin_lock(&tr->max_lock); in update_max_tr_single()
2086 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->array_buffer.buffer, cpu); in update_max_tr_single()
2096 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_, in update_max_tr_single()
2102 __update_max_tr(tr, tsk, cpu); in update_max_tr_single()
2103 arch_spin_unlock(&tr->max_lock); in update_max_tr_single()
2145 iter->array_buffer = &iter->tr->max_buffer; in wait_on_pipe()
2175 struct trace_array *tr = &global_trace; in run_tracer_selftest() local
2176 struct tracer *saved_tracer = tr->current_trace; in run_tracer_selftest()
2203 tracing_reset_online_cpus(&tr->array_buffer); in run_tracer_selftest()
2205 tr->current_trace = type; in run_tracer_selftest()
2210 if (tr->ring_buffer_expanded) in run_tracer_selftest()
2211 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size, in run_tracer_selftest()
2213 tr->allocated_snapshot = true; in run_tracer_selftest()
2219 ret = type->selftest(type, tr); in run_tracer_selftest()
2221 tr->current_trace = saved_tracer; in run_tracer_selftest()
2229 tracing_reset_online_cpus(&tr->array_buffer); in run_tracer_selftest()
2233 tr->allocated_snapshot = false; in run_tracer_selftest()
2236 if (tr->ring_buffer_expanded) in run_tracer_selftest()
2237 ring_buffer_resize(tr->max_buffer.buffer, 1, in run_tracer_selftest()
2314 static void add_tracer_options(struct trace_array *tr, struct tracer *t);
2462 struct trace_array *tr; in tracing_reset_all_online_cpus_unlocked() local
2466 list_for_each_entry(tr, &ftrace_trace_arrays, list) { in tracing_reset_all_online_cpus_unlocked()
2467 if (!tr->clear_trace) in tracing_reset_all_online_cpus_unlocked()
2469 tr->clear_trace = false; in tracing_reset_all_online_cpus_unlocked()
2470 tracing_reset_online_cpus(&tr->array_buffer); in tracing_reset_all_online_cpus_unlocked()
2472 tracing_reset_online_cpus(&tr->max_buffer); in tracing_reset_all_online_cpus_unlocked()
2488 static void tracing_start_tr(struct trace_array *tr) in tracing_start_tr() argument
2495 guard(raw_spinlock_irqsave)(&tr->start_lock); in tracing_start_tr()
2496 if (--tr->stop_count) { in tracing_start_tr()
2497 if (WARN_ON_ONCE(tr->stop_count < 0)) { in tracing_start_tr()
2499 tr->stop_count = 0; in tracing_start_tr()
2505 arch_spin_lock(&tr->max_lock); in tracing_start_tr()
2507 buffer = tr->array_buffer.buffer; in tracing_start_tr()
2512 buffer = tr->max_buffer.buffer; in tracing_start_tr()
2517 arch_spin_unlock(&tr->max_lock); in tracing_start_tr()
2532 static void tracing_stop_tr(struct trace_array *tr) in tracing_stop_tr() argument
2536 guard(raw_spinlock_irqsave)(&tr->start_lock); in tracing_stop_tr()
2537 if (tr->stop_count++) in tracing_stop_tr()
2541 arch_spin_lock(&tr->max_lock); in tracing_stop_tr()
2543 buffer = tr->array_buffer.buffer; in tracing_stop_tr()
2548 buffer = tr->max_buffer.buffer; in tracing_stop_tr()
2553 arch_spin_unlock(&tr->max_lock); in tracing_stop_tr()
2740 struct trace_array *tr = trace_file->tr; in trace_event_buffer_lock_reserve() local
2743 *current_rb = tr->array_buffer.buffer; in trace_event_buffer_lock_reserve()
2745 if (!tr->no_filter_buffering_ref && in trace_event_buffer_lock_reserve()
2899 trace_buffer_unlock_commit_regs(file->tr, fbuffer->buffer, in trace_event_buffer_commit()
2918 void trace_buffer_unlock_commit_regs(struct trace_array *tr, in trace_buffer_unlock_commit_regs() argument
2932 ftrace_trace_stack(tr, buffer, trace_ctx, regs ? 0 : STACK_SKIP, regs); in trace_buffer_unlock_commit_regs()
2933 ftrace_trace_userstack(tr, buffer, trace_ctx); in trace_buffer_unlock_commit_regs()
2947 trace_function(struct trace_array *tr, unsigned long ip, unsigned long in trace_function() argument
2950 struct trace_buffer *buffer = tr->array_buffer.buffer; in trace_function()
2996 static void __ftrace_trace_stack(struct trace_array *tr, in __ftrace_trace_stack() argument
3045 if (tr->ops && tr->ops->trampoline) { in __ftrace_trace_stack()
3046 unsigned long tramp_start = tr->ops->trampoline; in __ftrace_trace_stack()
3047 unsigned long tramp_end = tramp_start + tr->ops->trampoline_size; in __ftrace_trace_stack()
3076 static inline void ftrace_trace_stack(struct trace_array *tr, in ftrace_trace_stack() argument
3081 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE)) in ftrace_trace_stack()
3084 __ftrace_trace_stack(tr, buffer, trace_ctx, skip, regs); in ftrace_trace_stack()
3087 void __trace_stack(struct trace_array *tr, unsigned int trace_ctx, in __trace_stack() argument
3090 struct trace_buffer *buffer = tr->array_buffer.buffer; in __trace_stack()
3093 __ftrace_trace_stack(tr, buffer, trace_ctx, skip, NULL); in __trace_stack()
3110 __ftrace_trace_stack(tr, buffer, trace_ctx, skip, NULL); in __trace_stack()
3136 ftrace_trace_userstack(struct trace_array *tr, in ftrace_trace_userstack() argument
3142 if (!(tr->trace_flags & TRACE_ITER_USERSTACKTRACE)) in ftrace_trace_userstack()
3178 static void ftrace_trace_userstack(struct trace_array *tr, in ftrace_trace_userstack() argument
3195 void trace_last_func_repeats(struct trace_array *tr, in trace_last_func_repeats() argument
3199 struct trace_buffer *buffer = tr->array_buffer.buffer; in trace_last_func_repeats()
3341 struct trace_array *tr = READ_ONCE(printk_trace); in trace_vbprintk() local
3347 if (!printk_binsafe(tr)) in trace_vbprintk()
3371 buffer = tr->array_buffer.buffer; in trace_vbprintk()
3383 ftrace_trace_stack(tr, buffer, trace_ctx, 6, NULL); in trace_vbprintk()
3445 int trace_array_vprintk(struct trace_array *tr, in trace_array_vprintk() argument
3448 if (tracing_selftest_running && tr == &global_trace) in trace_array_vprintk()
3451 return __trace_array_vprintk(tr->array_buffer.buffer, ip, fmt, args); in trace_array_vprintk()
3456 * @tr: The instance trace_array descriptor
3471 * Note, trace_array_init_printk() must be called on @tr before this
3474 int trace_array_printk(struct trace_array *tr, in trace_array_printk() argument
3480 if (!tr) in trace_array_printk()
3484 if (tr == &global_trace) in trace_array_printk()
3487 if (!(tr->trace_flags & TRACE_ITER_PRINTK)) in trace_array_printk()
3491 ret = trace_array_vprintk(tr, ip, fmt, ap); in trace_array_printk()
3499 * @tr: The trace array to initialize the buffers for
3505 int trace_array_init_printk(struct trace_array *tr) in trace_array_init_printk() argument
3507 if (!tr) in trace_array_init_printk()
3511 if (tr == &global_trace) in trace_array_init_printk()
3641 * iter->tr is NULL when used with tp_printk, which makes in trace_iter_expand_format()
3644 if (!iter->tr || iter->fmt == static_fmt_buf) in trace_iter_expand_format()
3794 if (!iter->tr || iter->tr->trace_flags & TRACE_ITER_HASH_PTR) in trace_event_format()
3957 struct trace_array *tr = iter->tr; in s_start() local
3964 if (unlikely(tr->current_trace != iter->trace)) { in s_start()
3968 iter->trace = tr->current_trace; in s_start()
4065 unsigned long trace_total_entries_cpu(struct trace_array *tr, int cpu) in trace_total_entries_cpu() argument
4069 if (!tr) in trace_total_entries_cpu()
4070 tr = &global_trace; in trace_total_entries_cpu()
4072 get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu); in trace_total_entries_cpu()
4077 unsigned long trace_total_entries(struct trace_array *tr) in trace_total_entries() argument
4081 if (!tr) in trace_total_entries()
4082 tr = &global_trace; in trace_total_entries()
4084 get_total_entries(&tr->array_buffer, &total, &entries); in trace_total_entries()
4198 struct trace_array *tr = iter->tr; in test_cpu_buff_start() local
4200 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE)) in test_cpu_buff_start()
4224 struct trace_array *tr = iter->tr; in print_trace_fmt() local
4226 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK); in print_trace_fmt()
4236 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { in print_trace_fmt()
4247 if (tr->trace_flags & TRACE_ITER_FIELDS) in print_trace_fmt()
4254 if ((tr->text_delta) && in print_trace_fmt()
4268 struct trace_array *tr = iter->tr; in print_raw_fmt() local
4275 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) in print_raw_fmt()
4293 struct trace_array *tr = iter->tr; in print_hex_fmt() local
4301 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { in print_hex_fmt()
4323 struct trace_array *tr = iter->tr; in print_bin_fmt() local
4330 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) { in print_bin_fmt()
4379 struct trace_array *tr = iter->tr; in print_trace_line() local
4380 unsigned long trace_flags = tr->trace_flags; in print_trace_line()
4430 struct trace_array *tr = iter->tr; in trace_latency_header() local
4439 if (!(tr->trace_flags & TRACE_ITER_VERBOSE)) in trace_latency_header()
4446 struct trace_array *tr = iter->tr; in trace_default_header() local
4447 unsigned long trace_flags = tr->trace_flags; in trace_default_header()
4507 if (iter->tr->allocated_snapshot) in print_snapshot_help()
4529 if (iter->tr) { in s_show()
4611 struct trace_array *tr = inode->i_private; in __tracing_open() local
4650 iter->trace = tr->current_trace; in __tracing_open()
4655 iter->tr = tr; in __tracing_open()
4659 if (tr->current_trace->print_max || snapshot) in __tracing_open()
4660 iter->array_buffer = &tr->max_buffer; in __tracing_open()
4663 iter->array_buffer = &tr->array_buffer; in __tracing_open()
4678 if (trace_clocks[tr->clock_id].in_ns) in __tracing_open()
4685 if (!iter->snapshot && (tr->trace_flags & TRACE_ITER_PAUSE_ON_TRACE)) in __tracing_open()
4686 tracing_stop_tr(tr); in __tracing_open()
4738 struct trace_array *tr = inode->i_private; in tracing_open_generic_tr() local
4741 ret = tracing_check_open_get_tr(tr); in tracing_open_generic_tr()
4752 * Update the tr ref count associated to it.
4759 ret = tracing_check_open_get_tr(file->tr); in tracing_open_file_tr()
4767 trace_array_put(file->tr); in tracing_open_file_tr()
4782 trace_array_put(file->tr); in tracing_release_file_tr()
4796 struct trace_array *tr = inode->i_private; in tracing_release() local
4802 trace_array_put(tr); in tracing_release()
4818 if (!iter->snapshot && tr->stop_count) in tracing_release()
4820 tracing_start_tr(tr); in tracing_release()
4822 __trace_array_put(tr); in tracing_release()
4834 struct trace_array *tr = inode->i_private; in tracing_release_generic_tr() local
4836 trace_array_put(tr); in tracing_release_generic_tr()
4842 struct trace_array *tr = inode->i_private; in tracing_single_release_tr() local
4844 trace_array_put(tr); in tracing_single_release_tr()
4851 struct trace_array *tr = inode->i_private; in tracing_open() local
4855 ret = tracing_check_open_get_tr(tr); in tracing_open()
4862 struct array_buffer *trace_buf = &tr->array_buffer; in tracing_open()
4865 if (tr->current_trace->print_max) in tracing_open()
4866 trace_buf = &tr->max_buffer; in tracing_open()
4879 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) in tracing_open()
4884 trace_array_put(tr); in tracing_open()
4895 trace_ok_for_array(struct tracer *t, struct trace_array *tr) in trace_ok_for_array() argument
4899 if (tr->range_addr_start && t->use_max_tr) in trace_ok_for_array()
4902 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances; in trace_ok_for_array()
4907 get_tracer_for_array(struct trace_array *tr, struct tracer *t) in get_tracer_for_array() argument
4909 while (t && !trace_ok_for_array(t, tr)) in get_tracer_for_array()
4918 struct trace_array *tr = m->private; in t_next() local
4924 t = get_tracer_for_array(tr, t->next); in t_next()
4931 struct trace_array *tr = m->private; in t_start() local
4937 t = get_tracer_for_array(tr, trace_types); in t_start()
4974 struct trace_array *tr = inode->i_private; in show_traces_open() local
4978 ret = tracing_check_open_get_tr(tr); in show_traces_open()
4984 trace_array_put(tr); in show_traces_open()
4989 m->private = tr; in show_traces_open()
4996 struct trace_array *tr = inode->i_private; in tracing_seq_release() local
4998 trace_array_put(tr); in tracing_seq_release()
5042 struct trace_array *tr = file_inode(filp)->i_private; in tracing_cpumask_read() local
5047 cpumask_pr_args(tr->tracing_cpumask)) + 1; in tracing_cpumask_read()
5053 cpumask_pr_args(tr->tracing_cpumask)); in tracing_cpumask_read()
5060 int tracing_set_cpumask(struct trace_array *tr, in tracing_set_cpumask() argument
5065 if (!tr) in tracing_set_cpumask()
5069 arch_spin_lock(&tr->max_lock); in tracing_set_cpumask()
5075 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) && in tracing_set_cpumask()
5077 ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu); in tracing_set_cpumask()
5079 ring_buffer_record_disable_cpu(tr->max_buffer.buffer, cpu); in tracing_set_cpumask()
5082 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) && in tracing_set_cpumask()
5084 ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu); in tracing_set_cpumask()
5086 ring_buffer_record_enable_cpu(tr->max_buffer.buffer, cpu); in tracing_set_cpumask()
5090 arch_spin_unlock(&tr->max_lock); in tracing_set_cpumask()
5093 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new); in tracing_set_cpumask()
5102 struct trace_array *tr = file_inode(filp)->i_private; in tracing_cpumask_write() local
5116 err = tracing_set_cpumask(tr, tracing_cpumask_new); in tracing_cpumask_write()
5141 struct trace_array *tr = m->private; in tracing_trace_options_show() local
5147 tracer_flags = tr->current_trace->flags->val; in tracing_trace_options_show()
5148 trace_opts = tr->current_trace->flags->opts; in tracing_trace_options_show()
5151 if (tr->trace_flags & (1 << i)) in tracing_trace_options_show()
5167 static int __set_tracer_option(struct trace_array *tr, in __set_tracer_option() argument
5174 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg); in __set_tracer_option()
5186 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg) in set_tracer_option() argument
5188 struct tracer *trace = tr->current_trace; in set_tracer_option()
5197 return __set_tracer_option(tr, trace->flags, opts, neg); in set_tracer_option()
5212 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled) in set_tracer_flag() argument
5221 if (!!(tr->trace_flags & mask) == !!enabled) in set_tracer_flag()
5225 if (tr->current_trace->flag_changed) in set_tracer_flag()
5226 if (tr->current_trace->flag_changed(tr, mask, !!enabled)) in set_tracer_flag()
5231 update_printk_trace(tr); in set_tracer_flag()
5243 if (printk_trace == tr) in set_tracer_flag()
5249 update_marker_trace(tr, enabled); in set_tracer_flag()
5252 tr->trace_flags |= mask; in set_tracer_flag()
5254 tr->trace_flags &= ~mask; in set_tracer_flag()
5262 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID; in set_tracer_flag()
5270 trace_event_follow_fork(tr, enabled); in set_tracer_flag()
5273 ftrace_pid_follow_fork(tr, enabled); in set_tracer_flag()
5276 ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled); in set_tracer_flag()
5278 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled); in set_tracer_flag()
5290 int trace_set_options(struct trace_array *tr, char *option) in trace_set_options() argument
5312 ret = set_tracer_option(tr, cmp, neg); in trace_set_options()
5314 ret = set_tracer_flag(tr, 1 << ret, !neg); in trace_set_options()
5354 struct trace_array *tr = m->private; in tracing_trace_options_write() local
5366 ret = trace_set_options(tr, buf); in tracing_trace_options_write()
5377 struct trace_array *tr = inode->i_private; in tracing_trace_options_open() local
5380 ret = tracing_check_open_get_tr(tr); in tracing_trace_options_open()
5386 trace_array_put(tr); in tracing_trace_options_open()
5900 struct trace_array *tr = filp->private_data; in tracing_set_trace_read() local
5905 r = sprintf(buf, "%s\n", tr->current_trace->name); in tracing_set_trace_read()
5911 int tracer_init(struct tracer *t, struct trace_array *tr) in tracer_init() argument
5913 tracing_reset_online_cpus(&tr->array_buffer); in tracer_init()
5914 return t->init(tr); in tracer_init()
5935 /* resize @tr's buffer to the size of @size_tr's entries */
5962 static int __tracing_resize_ring_buffer(struct trace_array *tr, in __tracing_resize_ring_buffer() argument
5972 trace_set_ring_buffer_expanded(tr); in __tracing_resize_ring_buffer()
5975 if (!tr->array_buffer.buffer) in __tracing_resize_ring_buffer()
5979 tracing_stop_tr(tr); in __tracing_resize_ring_buffer()
5981 ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu); in __tracing_resize_ring_buffer()
5986 if (!tr->allocated_snapshot) in __tracing_resize_ring_buffer()
5989 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu); in __tracing_resize_ring_buffer()
5991 int r = resize_buffer_duplicate_size(&tr->array_buffer, in __tracing_resize_ring_buffer()
5992 &tr->array_buffer, cpu); in __tracing_resize_ring_buffer()
6014 update_buffer_entries(&tr->max_buffer, cpu); in __tracing_resize_ring_buffer()
6019 update_buffer_entries(&tr->array_buffer, cpu); in __tracing_resize_ring_buffer()
6021 tracing_start_tr(tr); in __tracing_resize_ring_buffer()
6025 ssize_t tracing_resize_ring_buffer(struct trace_array *tr, in tracing_resize_ring_buffer() argument
6036 return __tracing_resize_ring_buffer(tr, size, cpu_id); in tracing_resize_ring_buffer()
6066 * @tr: Persistent ring buffer's trace_array.
6067 * @addr: Address in @tr which is adjusted.
6069 unsigned long trace_adjust_address(struct trace_array *tr, unsigned long addr) in trace_adjust_address() argument
6078 if (!(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) in trace_adjust_address()
6081 /* tr->module_delta must be protected by rcu. */ in trace_adjust_address()
6083 tscratch = tr->scratch; in trace_adjust_address()
6085 module_delta = READ_ONCE(tr->module_delta); in trace_adjust_address()
6088 raddr = addr + tr->text_delta; in trace_adjust_address()
6114 struct trace_array *tr = data; in save_mod() local
6119 tscratch = tr->scratch; in save_mod()
6122 size = tr->scratch_size; in save_mod()
6143 static void update_last_data(struct trace_array *tr) in update_last_data() argument
6148 if (!(tr->flags & TRACE_ARRAY_FL_BOOT)) in update_last_data()
6151 if (!(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) in update_last_data()
6155 tr->flags &= ~TRACE_ARRAY_FL_LAST_BOOT; in update_last_data()
6158 if (tr->scratch) { in update_last_data()
6159 struct trace_scratch *tscratch = tr->scratch; in update_last_data()
6161 tscratch->clock_id = tr->clock_id; in update_last_data()
6167 module_for_each_mod(save_mod, tr); in update_last_data()
6176 tracing_reset_all_cpus(&tr->array_buffer); in update_last_data()
6179 tr->text_delta = 0; in update_last_data()
6181 if (!tr->scratch) in update_last_data()
6184 tscratch = tr->scratch; in update_last_data()
6185 module_delta = READ_ONCE(tr->module_delta); in update_last_data()
6186 WRITE_ONCE(tr->module_delta, NULL); in update_last_data()
6195 * @tr: The tracing instance
6204 int tracing_update_buffers(struct trace_array *tr) in tracing_update_buffers() argument
6210 update_last_data(tr); in tracing_update_buffers()
6212 if (!tr->ring_buffer_expanded) in tracing_update_buffers()
6213 ret = __tracing_resize_ring_buffer(tr, trace_buf_size, in tracing_update_buffers()
6221 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
6227 static void tracing_set_nop(struct trace_array *tr) in tracing_set_nop() argument
6229 if (tr->current_trace == &nop_trace) in tracing_set_nop()
6232 tr->current_trace->enabled--; in tracing_set_nop()
6234 if (tr->current_trace->reset) in tracing_set_nop()
6235 tr->current_trace->reset(tr); in tracing_set_nop()
6237 tr->current_trace = &nop_trace; in tracing_set_nop()
6242 static void add_tracer_options(struct trace_array *tr, struct tracer *t) in add_tracer_options() argument
6245 if (!tr->dir && !(tr->flags & TRACE_ARRAY_FL_GLOBAL)) in add_tracer_options()
6252 create_trace_option_files(tr, t); in add_tracer_options()
6255 int tracing_set_tracer(struct trace_array *tr, const char *buf) in tracing_set_tracer() argument
6265 update_last_data(tr); in tracing_set_tracer()
6267 if (!tr->ring_buffer_expanded) { in tracing_set_tracer()
6268 ret = __tracing_resize_ring_buffer(tr, trace_buf_size, in tracing_set_tracer()
6282 if (t == tr->current_trace) in tracing_set_tracer()
6288 arch_spin_lock(&tr->max_lock); in tracing_set_tracer()
6289 ret = tr->cond_snapshot ? -EBUSY : 0; in tracing_set_tracer()
6290 arch_spin_unlock(&tr->max_lock); in tracing_set_tracer()
6304 if (!trace_ok_for_array(t, tr)) in tracing_set_tracer()
6308 if (tr->trace_ref) in tracing_set_tracer()
6313 tr->current_trace->enabled--; in tracing_set_tracer()
6315 if (tr->current_trace->reset) in tracing_set_tracer()
6316 tr->current_trace->reset(tr); in tracing_set_tracer()
6319 had_max_tr = tr->current_trace->use_max_tr; in tracing_set_tracer()
6322 tr->current_trace = &nop_trace; in tracing_set_tracer()
6333 free_snapshot(tr); in tracing_set_tracer()
6334 tracing_disarm_snapshot(tr); in tracing_set_tracer()
6338 ret = tracing_arm_snapshot_locked(tr); in tracing_set_tracer()
6343 tr->current_trace = &nop_trace; in tracing_set_tracer()
6347 ret = tracer_init(t, tr); in tracing_set_tracer()
6351 tracing_disarm_snapshot(tr); in tracing_set_tracer()
6357 tr->current_trace = t; in tracing_set_tracer()
6358 tr->current_trace->enabled++; in tracing_set_tracer()
6359 trace_branch_enable(tr); in tracing_set_tracer()
6368 struct trace_array *tr = filp->private_data; in tracing_set_trace_write() local
6386 err = tracing_set_tracer(tr, name); in tracing_set_trace_write()
6436 struct trace_array *tr = filp->private_data; in tracing_thresh_write() local
6444 if (tr->current_trace->update_thresh) { in tracing_thresh_write()
6445 ret = tr->current_trace->update_thresh(tr); in tracing_thresh_write()
6459 struct trace_array *tr = filp->private_data; in tracing_max_lat_read() local
6461 return tracing_nsecs_read(&tr->max_latency, ubuf, cnt, ppos); in tracing_max_lat_read()
6468 struct trace_array *tr = filp->private_data; in tracing_max_lat_write() local
6470 return tracing_nsecs_write(&tr->max_latency, ubuf, cnt, ppos); in tracing_max_lat_write()
6475 static int open_pipe_on_cpu(struct trace_array *tr, int cpu) in open_pipe_on_cpu() argument
6478 if (cpumask_empty(tr->pipe_cpumask)) { in open_pipe_on_cpu()
6479 cpumask_setall(tr->pipe_cpumask); in open_pipe_on_cpu()
6482 } else if (!cpumask_test_cpu(cpu, tr->pipe_cpumask)) { in open_pipe_on_cpu()
6483 cpumask_set_cpu(cpu, tr->pipe_cpumask); in open_pipe_on_cpu()
6489 static void close_pipe_on_cpu(struct trace_array *tr, int cpu) in close_pipe_on_cpu() argument
6492 WARN_ON(!cpumask_full(tr->pipe_cpumask)); in close_pipe_on_cpu()
6493 cpumask_clear(tr->pipe_cpumask); in close_pipe_on_cpu()
6495 WARN_ON(!cpumask_test_cpu(cpu, tr->pipe_cpumask)); in close_pipe_on_cpu()
6496 cpumask_clear_cpu(cpu, tr->pipe_cpumask); in close_pipe_on_cpu()
6502 struct trace_array *tr = inode->i_private; in tracing_open_pipe() local
6507 ret = tracing_check_open_get_tr(tr); in tracing_open_pipe()
6513 ret = open_pipe_on_cpu(tr, cpu); in tracing_open_pipe()
6525 iter->trace = tr->current_trace; in tracing_open_pipe()
6535 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) in tracing_open_pipe()
6539 if (trace_clocks[tr->clock_id].in_ns) in tracing_open_pipe()
6542 iter->tr = tr; in tracing_open_pipe()
6543 iter->array_buffer = &tr->array_buffer; in tracing_open_pipe()
6553 tr->trace_ref++; in tracing_open_pipe()
6560 close_pipe_on_cpu(tr, cpu); in tracing_open_pipe()
6562 __trace_array_put(tr); in tracing_open_pipe()
6569 struct trace_array *tr = inode->i_private; in tracing_release_pipe() local
6572 tr->trace_ref--; in tracing_release_pipe()
6576 close_pipe_on_cpu(tr, iter->cpu_file); in tracing_release_pipe()
6582 trace_array_put(tr); in tracing_release_pipe()
6590 struct trace_array *tr = iter->tr; in trace_poll() local
6596 if (tr->trace_flags & TRACE_ITER_BLOCK) in trace_poll()
6603 filp, poll_table, iter->tr->buffer_percent); in trace_poll()
6635 if (!tracer_tracing_is_on(iter->tr) && iter->pos) in tracing_wait_pipe()
6651 static bool update_last_data_if_empty(struct trace_array *tr) in update_last_data_if_empty() argument
6653 if (!(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) in update_last_data_if_empty()
6656 if (!ring_buffer_empty(tr->array_buffer.buffer)) in update_last_data_if_empty()
6663 update_last_data(tr); in update_last_data_if_empty()
6698 if (update_last_data_if_empty(iter->tr)) in tracing_read_pipe()
6712 /* reset all but tr, trace, and overruns */ in tracing_read_pipe()
6919 struct trace_array *tr = inode->i_private; in tracing_entries_read() local
6937 size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries; in tracing_entries_read()
6938 if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) { in tracing_entries_read()
6945 if (!tr->ring_buffer_expanded) in tracing_entries_read()
6954 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10); in tracing_entries_read()
6967 struct trace_array *tr = inode->i_private; in tracing_entries_write() local
6981 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode)); in tracing_entries_write()
6994 struct trace_array *tr = filp->private_data; in tracing_total_entries_read() local
7001 size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10; in tracing_total_entries_read()
7002 if (!tr->ring_buffer_expanded) in tracing_total_entries_read()
7005 if (tr->ring_buffer_expanded) in tracing_total_entries_read()
7018 struct trace_array *tr = m->private; in l_next() local
7019 struct trace_scratch *tscratch = tr->scratch; in l_next()
7028 if (!tscratch || !(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) in l_next()
7052 static void show_last_boot_header(struct seq_file *m, struct trace_array *tr) in show_last_boot_header() argument
7054 struct trace_scratch *tscratch = tr->scratch; in show_last_boot_header()
7063 if (tscratch && (tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) in show_last_boot_header()
7071 struct trace_array *tr = m->private; in l_show() local
7075 show_last_boot_header(m, tr); in l_show()
7092 struct trace_array *tr = inode->i_private; in tracing_last_boot_open() local
7096 ret = tracing_check_open_get_tr(tr); in tracing_last_boot_open()
7102 trace_array_put(tr); in tracing_last_boot_open()
7107 m->private = tr; in tracing_last_boot_open()
7114 struct trace_array *tr = inode->i_private; in tracing_buffer_meta_open() local
7118 ret = tracing_check_open_get_tr(tr); in tracing_buffer_meta_open()
7122 ret = ring_buffer_meta_seq_init(filp, tr->array_buffer.buffer, cpu); in tracing_buffer_meta_open()
7124 __trace_array_put(tr); in tracing_buffer_meta_open()
7145 struct trace_array *tr = inode->i_private; in tracing_free_buffer_release() local
7148 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE) in tracing_free_buffer_release()
7149 tracer_tracing_off(tr); in tracing_free_buffer_release()
7151 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS); in tracing_free_buffer_release()
7153 trace_array_put(tr); in tracing_free_buffer_release()
7160 static ssize_t write_marker_to_buffer(struct trace_array *tr, const char *buf, in write_marker_to_buffer() argument
7175 buffer = tr->array_buffer.buffer; in write_marker_to_buffer()
7200 if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) { in write_marker_to_buffer()
7203 tt = event_triggers_call(tr->trace_marker_file, buffer, entry, event); in write_marker_to_buffer()
7217 event_triggers_post_call(tr->trace_marker_file, tt); in write_marker_to_buffer()
7389 struct trace_array *tr = filp->private_data; in tracing_mark_write() local
7398 if (!(tr->trace_flags & TRACE_ITER_MARKERS)) in tracing_mark_write()
7421 if (tr == &global_trace) { in tracing_mark_write()
7423 list_for_each_entry_rcu(tr, &marker_copies, marker_list) { in tracing_mark_write()
7424 written = write_marker_to_buffer(tr, buf, cnt, ip); in tracing_mark_write()
7429 written = write_marker_to_buffer(tr, buf, cnt, ip); in tracing_mark_write()
7435 static ssize_t write_raw_marker_to_buffer(struct trace_array *tr, in write_raw_marker_to_buffer() argument
7447 buffer = tr->array_buffer.buffer; in write_raw_marker_to_buffer()
7474 struct trace_array *tr = filp->private_data; in tracing_mark_raw_write() local
7482 if (!(tr->trace_flags & TRACE_ITER_MARKERS)) in tracing_mark_raw_write()
7501 if (tr == &global_trace) { in tracing_mark_raw_write()
7503 list_for_each_entry_rcu(tr, &marker_copies, marker_list) { in tracing_mark_raw_write()
7504 written = write_raw_marker_to_buffer(tr, buf, cnt); in tracing_mark_raw_write()
7509 written = write_raw_marker_to_buffer(tr, buf, cnt); in tracing_mark_raw_write()
7538 struct trace_array *tr = m->private; in tracing_clock_show() local
7544 i == tr->clock_id ? "[" : "", trace_clocks[i].name, in tracing_clock_show()
7545 i == tr->clock_id ? "]" : ""); in tracing_clock_show()
7551 int tracing_set_clock(struct trace_array *tr, const char *clockstr) in tracing_set_clock() argument
7564 tr->clock_id = i; in tracing_set_clock()
7566 ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func); in tracing_set_clock()
7572 tracing_reset_online_cpus(&tr->array_buffer); in tracing_set_clock()
7575 if (tr->max_buffer.buffer) in tracing_set_clock()
7576 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func); in tracing_set_clock()
7577 tracing_reset_online_cpus(&tr->max_buffer); in tracing_set_clock()
7580 if (tr->scratch && !(tr->flags & TRACE_ARRAY_FL_LAST_BOOT)) { in tracing_set_clock()
7581 struct trace_scratch *tscratch = tr->scratch; in tracing_set_clock()
7593 struct trace_array *tr = m->private; in tracing_clock_write() local
7608 ret = tracing_set_clock(tr, clockstr); in tracing_clock_write()
7619 struct trace_array *tr = inode->i_private; in tracing_clock_open() local
7622 ret = tracing_check_open_get_tr(tr); in tracing_clock_open()
7628 trace_array_put(tr); in tracing_clock_open()
7635 struct trace_array *tr = m->private; in tracing_time_stamp_mode_show() local
7639 if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer)) in tracing_time_stamp_mode_show()
7649 struct trace_array *tr = inode->i_private; in tracing_time_stamp_mode_open() local
7652 ret = tracing_check_open_get_tr(tr); in tracing_time_stamp_mode_open()
7658 trace_array_put(tr); in tracing_time_stamp_mode_open()
7674 int tracing_set_filter_buffering(struct trace_array *tr, bool set) in tracing_set_filter_buffering() argument
7678 if (set && tr->no_filter_buffering_ref++) in tracing_set_filter_buffering()
7682 if (WARN_ON_ONCE(!tr->no_filter_buffering_ref)) in tracing_set_filter_buffering()
7685 --tr->no_filter_buffering_ref; in tracing_set_filter_buffering()
7702 struct trace_array *tr = inode->i_private; in tracing_snapshot_open() local
7707 ret = tracing_check_open_get_tr(tr); in tracing_snapshot_open()
7728 iter->tr = tr; in tracing_snapshot_open()
7729 iter->array_buffer = &tr->max_buffer; in tracing_snapshot_open()
7736 trace_array_put(tr); in tracing_snapshot_open()
7741 static void tracing_swap_cpu_buffer(void *tr) in tracing_swap_cpu_buffer() argument
7743 update_max_tr_single((struct trace_array *)tr, current, smp_processor_id()); in tracing_swap_cpu_buffer()
7752 struct trace_array *tr = iter->tr; in tracing_snapshot_write() local
7756 ret = tracing_update_buffers(tr); in tracing_snapshot_write()
7766 if (tr->current_trace->use_max_tr) in tracing_snapshot_write()
7770 arch_spin_lock(&tr->max_lock); in tracing_snapshot_write()
7771 if (tr->cond_snapshot) in tracing_snapshot_write()
7773 arch_spin_unlock(&tr->max_lock); in tracing_snapshot_write()
7782 if (tr->allocated_snapshot) in tracing_snapshot_write()
7783 free_snapshot(tr); in tracing_snapshot_write()
7791 if (tr->allocated_snapshot) in tracing_snapshot_write()
7792 ret = resize_buffer_duplicate_size(&tr->max_buffer, in tracing_snapshot_write()
7793 &tr->array_buffer, iter->cpu_file); in tracing_snapshot_write()
7795 ret = tracing_arm_snapshot_locked(tr); in tracing_snapshot_write()
7802 update_max_tr(tr, current, smp_processor_id(), NULL); in tracing_snapshot_write()
7806 (void *)tr, 1); in tracing_snapshot_write()
7808 tracing_disarm_snapshot(tr); in tracing_snapshot_write()
7811 if (tr->allocated_snapshot) { in tracing_snapshot_write()
7813 tracing_reset_online_cpus(&tr->max_buffer); in tracing_snapshot_write()
7815 tracing_reset_cpu(&tr->max_buffer, iter->cpu_file); in tracing_snapshot_write()
7871 info->iter.array_buffer = &info->iter.tr->max_buffer; in snapshot_raw_open()
8121 static struct tracing_log_err *get_tracing_log_err(struct trace_array *tr, in get_tracing_log_err() argument
8127 if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) { in get_tracing_log_err()
8130 tr->n_err_log_entries++; in get_tracing_log_err()
8137 err = list_first_entry(&tr->err_log, struct tracing_log_err, list); in get_tracing_log_err()
8173 * @tr: The associated trace array for the error (NULL for top level array)
8198 void tracing_log_err(struct trace_array *tr, in tracing_log_err() argument
8205 if (!tr) in tracing_log_err()
8206 tr = &global_trace; in tracing_log_err()
8212 err = get_tracing_log_err(tr, len); in tracing_log_err()
8224 list_add_tail(&err->list, &tr->err_log); in tracing_log_err()
8227 static void clear_tracing_err_log(struct trace_array *tr) in clear_tracing_err_log() argument
8233 list_for_each_entry_safe(err, next, &tr->err_log, list) { in clear_tracing_err_log()
8238 tr->n_err_log_entries = 0; in clear_tracing_err_log()
8243 struct trace_array *tr = m->private; in tracing_err_log_seq_start() local
8247 return seq_list_start(&tr->err_log, *pos); in tracing_err_log_seq_start()
8252 struct trace_array *tr = m->private; in tracing_err_log_seq_next() local
8254 return seq_list_next(v, &tr->err_log, pos); in tracing_err_log_seq_next()
8301 struct trace_array *tr = inode->i_private; in tracing_err_log_open() local
8304 ret = tracing_check_open_get_tr(tr); in tracing_err_log_open()
8310 clear_tracing_err_log(tr); in tracing_err_log_open()
8316 m->private = tr; in tracing_err_log_open()
8318 trace_array_put(tr); in tracing_err_log_open()
8333 struct trace_array *tr = inode->i_private; in tracing_err_log_release() local
8335 trace_array_put(tr); in tracing_err_log_release()
8353 struct trace_array *tr = inode->i_private; in tracing_buffers_open() local
8357 ret = tracing_check_open_get_tr(tr); in tracing_buffers_open()
8363 trace_array_put(tr); in tracing_buffers_open()
8369 info->iter.tr = tr; in tracing_buffers_open()
8371 info->iter.trace = tr->current_trace; in tracing_buffers_open()
8372 info->iter.array_buffer = &tr->array_buffer; in tracing_buffers_open()
8379 tr->trace_ref++; in tracing_buffers_open()
8385 trace_array_put(tr); in tracing_buffers_open()
8414 if (iter->snapshot && iter->tr->current_trace->use_max_tr) in tracing_buffers_read()
8457 if (update_last_data_if_empty(iter->tr)) in tracing_buffers_read()
8511 iter->tr->trace_ref--; in tracing_buffers_release()
8513 __trace_array_put(iter->tr); in tracing_buffers_release()
8601 if (iter->snapshot && iter->tr->current_trace->use_max_tr) in tracing_buffers_splice_read()
8680 ret = wait_on_pipe(iter, iter->snapshot ? 0 : iter->tr->buffer_percent); in tracing_buffers_splice_read()
8685 if (!tracer_tracing_is_on(iter->tr)) in tracing_buffers_splice_read()
8711 iter->tr->buffer_percent, in tracing_buffers_ioctl()
8738 static int get_snapshot_map(struct trace_array *tr) in get_snapshot_map() argument
8747 spin_lock(&tr->snapshot_trigger_lock); in get_snapshot_map()
8749 if (tr->snapshot || tr->mapped == UINT_MAX) in get_snapshot_map()
8752 tr->mapped++; in get_snapshot_map()
8754 spin_unlock(&tr->snapshot_trigger_lock); in get_snapshot_map()
8756 /* Wait for update_max_tr() to observe iter->tr->mapped */ in get_snapshot_map()
8757 if (tr->mapped == 1) in get_snapshot_map()
8763 static void put_snapshot_map(struct trace_array *tr) in put_snapshot_map() argument
8765 spin_lock(&tr->snapshot_trigger_lock); in put_snapshot_map()
8766 if (!WARN_ON(!tr->mapped)) in put_snapshot_map()
8767 tr->mapped--; in put_snapshot_map()
8768 spin_unlock(&tr->snapshot_trigger_lock); in put_snapshot_map()
8771 static inline int get_snapshot_map(struct trace_array *tr) { return 0; } in get_snapshot_map() argument
8772 static inline void put_snapshot_map(struct trace_array *tr) { } in put_snapshot_map() argument
8781 put_snapshot_map(iter->tr); in tracing_buffers_mmap_close()
8795 if (iter->tr->flags & TRACE_ARRAY_FL_MEMMAP) in tracing_buffers_mmap()
8798 ret = get_snapshot_map(iter->tr); in tracing_buffers_mmap()
8804 put_snapshot_map(iter->tr); in tracing_buffers_mmap()
8827 struct trace_array *tr = inode->i_private; in tracing_stats_read() local
8828 struct array_buffer *trace_buf = &tr->array_buffer; in tracing_stats_read()
8853 if (trace_clocks[tr->clock_id].in_ns) { in tracing_stats_read()
8935 struct trace_array *tr, struct ftrace_probe_ops *ops, in ftrace_snapshot() argument
8938 tracing_snapshot_instance(tr); in ftrace_snapshot()
8943 struct trace_array *tr, struct ftrace_probe_ops *ops, in ftrace_count_snapshot() argument
8960 tracing_snapshot_instance(tr); in ftrace_count_snapshot()
8986 ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr, in ftrace_snapshot_init() argument
9002 ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr, in ftrace_snapshot_free() argument
9030 ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash, in ftrace_trace_snapshot_callback() argument
9038 if (!tr) in ftrace_trace_snapshot_callback()
9048 ret = unregister_ftrace_function_probe_func(glob+1, tr, ops); in ftrace_trace_snapshot_callback()
9050 tracing_disarm_snapshot(tr); in ftrace_trace_snapshot_callback()
9072 ret = tracing_arm_snapshot(tr); in ftrace_trace_snapshot_callback()
9076 ret = register_ftrace_function_probe(glob, tr, ops, count); in ftrace_trace_snapshot_callback()
9078 tracing_disarm_snapshot(tr); in ftrace_trace_snapshot_callback()
9096 static struct dentry *tracing_get_dentry(struct trace_array *tr) in tracing_get_dentry() argument
9099 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) in tracing_get_dentry()
9102 if (WARN_ON(!tr->dir)) in tracing_get_dentry()
9106 return tr->dir; in tracing_get_dentry()
9109 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu) in tracing_dentry_percpu() argument
9113 if (tr->percpu_dir) in tracing_dentry_percpu()
9114 return tr->percpu_dir; in tracing_dentry_percpu()
9116 d_tracer = tracing_get_dentry(tr); in tracing_dentry_percpu()
9120 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer); in tracing_dentry_percpu()
9122 MEM_FAIL(!tr->percpu_dir, in tracing_dentry_percpu()
9125 return tr->percpu_dir; in tracing_dentry_percpu()
9140 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu) in tracing_init_tracefs_percpu() argument
9142 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu); in tracing_init_tracefs_percpu()
9158 tr, cpu, &tracing_pipe_fops); in tracing_init_tracefs_percpu()
9162 tr, cpu, &tracing_fops); in tracing_init_tracefs_percpu()
9165 tr, cpu, &tracing_buffers_fops); in tracing_init_tracefs_percpu()
9168 tr, cpu, &tracing_stats_fops); in tracing_init_tracefs_percpu()
9171 tr, cpu, &tracing_entries_fops); in tracing_init_tracefs_percpu()
9173 if (tr->range_addr_start) in tracing_init_tracefs_percpu()
9175 tr, cpu, &tracing_buffer_meta_fops); in tracing_init_tracefs_percpu()
9177 if (!tr->range_addr_start) { in tracing_init_tracefs_percpu()
9179 tr, cpu, &snapshot_fops); in tracing_init_tracefs_percpu()
9182 tr, cpu, &snapshot_raw_fops); in tracing_init_tracefs_percpu()
9224 ret = __set_tracer_option(topt->tr, topt->flags, in trace_options_write()
9240 ret = tracing_check_open_get_tr(topt->tr); in tracing_open_options()
9252 trace_array_put(topt->tr); in tracing_release_options()
9302 struct trace_array *tr; in trace_options_core_read() local
9306 get_tr_index(tr_index, &tr, &index); in trace_options_core_read()
9308 if (tr->trace_flags & (1 << index)) in trace_options_core_read()
9321 struct trace_array *tr; in trace_options_core_write() local
9326 get_tr_index(tr_index, &tr, &index); in trace_options_core_write()
9337 ret = set_tracer_flag(tr, 1 << index, val); in trace_options_core_write()
9372 static struct dentry *trace_options_init_dentry(struct trace_array *tr) in trace_options_init_dentry() argument
9376 if (tr->options) in trace_options_init_dentry()
9377 return tr->options; in trace_options_init_dentry()
9379 d_tracer = tracing_get_dentry(tr); in trace_options_init_dentry()
9383 tr->options = tracefs_create_dir("options", d_tracer); in trace_options_init_dentry()
9384 if (!tr->options) { in trace_options_init_dentry()
9389 return tr->options; in trace_options_init_dentry()
9393 create_trace_option_file(struct trace_array *tr, in create_trace_option_file() argument
9400 t_options = trace_options_init_dentry(tr); in create_trace_option_file()
9406 topt->tr = tr; in create_trace_option_file()
9414 create_trace_option_files(struct trace_array *tr, struct tracer *tracer) in create_trace_option_files() argument
9435 if (!trace_ok_for_array(tracer, tr)) in create_trace_option_files()
9438 for (i = 0; i < tr->nr_topts; i++) { in create_trace_option_files()
9440 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags)) in create_trace_option_files()
9453 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1), in create_trace_option_files()
9460 tr->topts = tr_topts; in create_trace_option_files()
9461 tr->topts[tr->nr_topts].tracer = tracer; in create_trace_option_files()
9462 tr->topts[tr->nr_topts].topts = topts; in create_trace_option_files()
9463 tr->nr_topts++; in create_trace_option_files()
9466 create_trace_option_file(tr, &topts[cnt], flags, in create_trace_option_files()
9475 create_trace_option_core_file(struct trace_array *tr, in create_trace_option_core_file() argument
9480 t_options = trace_options_init_dentry(tr); in create_trace_option_core_file()
9485 (void *)&tr->trace_flags_index[index], in create_trace_option_core_file()
9489 static void create_trace_options_dir(struct trace_array *tr) in create_trace_options_dir() argument
9492 bool top_level = tr == &global_trace; in create_trace_options_dir()
9495 t_options = trace_options_init_dentry(tr); in create_trace_options_dir()
9502 create_trace_option_core_file(tr, trace_options[i], i); in create_trace_options_dir()
9510 struct trace_array *tr = filp->private_data; in rb_simple_read() local
9514 r = tracer_tracing_is_on(tr); in rb_simple_read()
9524 struct trace_array *tr = filp->private_data; in rb_simple_write() local
9525 struct trace_buffer *buffer = tr->array_buffer.buffer; in rb_simple_write()
9535 if (!!val == tracer_tracing_is_on(tr)) { in rb_simple_write()
9538 tracer_tracing_on(tr); in rb_simple_write()
9539 if (tr->current_trace->start) in rb_simple_write()
9540 tr->current_trace->start(tr); in rb_simple_write()
9542 tracer_tracing_off(tr); in rb_simple_write()
9543 if (tr->current_trace->stop) in rb_simple_write()
9544 tr->current_trace->stop(tr); in rb_simple_write()
9567 struct trace_array *tr = filp->private_data; in buffer_percent_read() local
9571 r = tr->buffer_percent; in buffer_percent_read()
9581 struct trace_array *tr = filp->private_data; in buffer_percent_write() local
9592 tr->buffer_percent = val; in buffer_percent_write()
9610 struct trace_array *tr = filp->private_data; in buffer_subbuf_size_read() local
9616 order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); in buffer_subbuf_size_read()
9628 struct trace_array *tr = filp->private_data; in buffer_subbuf_size_write() local
9649 tracing_stop_tr(tr); in buffer_subbuf_size_write()
9651 old_order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer); in buffer_subbuf_size_write()
9655 ret = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, order); in buffer_subbuf_size_write()
9661 if (!tr->allocated_snapshot) in buffer_subbuf_size_write()
9664 ret = ring_buffer_subbuf_order_set(tr->max_buffer.buffer, order); in buffer_subbuf_size_write()
9667 cnt = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, old_order); in buffer_subbuf_size_write()
9692 tracing_start_tr(tr); in buffer_subbuf_size_write()
9707 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
9715 struct trace_array *tr = data; in make_mod_delta() local
9718 tscratch = tr->scratch; in make_mod_delta()
9719 module_delta = READ_ONCE(tr->module_delta); in make_mod_delta()
9748 static void setup_trace_scratch(struct trace_array *tr, in setup_trace_scratch() argument
9758 tr->scratch = tscratch; in setup_trace_scratch()
9759 tr->scratch_size = size; in setup_trace_scratch()
9762 tr->text_delta = (unsigned long)_text - tscratch->text_addr; in setup_trace_scratch()
9797 WRITE_ONCE(tr->module_delta, module_delta); in setup_trace_scratch()
9800 module_for_each_mod(make_mod_delta, tr); in setup_trace_scratch()
9803 if (tscratch->clock_id != tr->clock_id) { in setup_trace_scratch()
9805 tracing_set_clock(tr, trace_clocks[tscratch->clock_id].name) < 0) { in setup_trace_scratch()
9817 allocate_trace_buffer(struct trace_array *tr, struct array_buffer *buf, int size) in allocate_trace_buffer() argument
9823 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0; in allocate_trace_buffer()
9825 buf->tr = tr; in allocate_trace_buffer()
9827 if (tr->range_addr_start && tr->range_addr_size) { in allocate_trace_buffer()
9830 tr->range_addr_start, in allocate_trace_buffer()
9831 tr->range_addr_size, in allocate_trace_buffer()
9835 setup_trace_scratch(tr, tscratch, scratch_size); in allocate_trace_buffer()
9841 tr->mapped++; in allocate_trace_buffer()
9856 set_buffer_entries(&tr->array_buffer, in allocate_trace_buffer()
9857 ring_buffer_size(tr->array_buffer.buffer, 0)); in allocate_trace_buffer()
9872 static int allocate_trace_buffers(struct trace_array *tr, int size) in allocate_trace_buffers() argument
9876 ret = allocate_trace_buffer(tr, &tr->array_buffer, size); in allocate_trace_buffers()
9882 if (tr->range_addr_start) in allocate_trace_buffers()
9885 ret = allocate_trace_buffer(tr, &tr->max_buffer, in allocate_trace_buffers()
9888 free_trace_buffer(&tr->array_buffer); in allocate_trace_buffers()
9891 tr->allocated_snapshot = allocate_snapshot; in allocate_trace_buffers()
9899 static void free_trace_buffers(struct trace_array *tr) in free_trace_buffers() argument
9901 if (!tr) in free_trace_buffers()
9904 free_trace_buffer(&tr->array_buffer); in free_trace_buffers()
9905 kfree(tr->module_delta); in free_trace_buffers()
9908 free_trace_buffer(&tr->max_buffer); in free_trace_buffers()
9912 static void init_trace_flags_index(struct trace_array *tr) in init_trace_flags_index() argument
9918 tr->trace_flags_index[i] = i; in init_trace_flags_index()
9921 static void __update_tracer_options(struct trace_array *tr) in __update_tracer_options() argument
9926 add_tracer_options(tr, t); in __update_tracer_options()
9929 static void update_tracer_options(struct trace_array *tr) in update_tracer_options() argument
9933 __update_tracer_options(tr); in update_tracer_options()
9939 struct trace_array *tr, *found = NULL; in trace_array_find() local
9941 list_for_each_entry(tr, &ftrace_trace_arrays, list) { in trace_array_find()
9942 if (tr->name && strcmp(tr->name, instance) == 0) { in trace_array_find()
9943 found = tr; in trace_array_find()
9953 struct trace_array *tr; in trace_array_find_get() local
9956 tr = trace_array_find(instance); in trace_array_find_get()
9957 if (tr) in trace_array_find_get()
9958 tr->ref++; in trace_array_find_get()
9960 return tr; in trace_array_find_get()
9963 static int trace_array_create_dir(struct trace_array *tr) in trace_array_create_dir() argument
9967 tr->dir = tracefs_create_dir(tr->name, trace_instance_dir); in trace_array_create_dir()
9968 if (!tr->dir) in trace_array_create_dir()
9971 ret = event_trace_add_tracer(tr->dir, tr); in trace_array_create_dir()
9973 tracefs_remove(tr->dir); in trace_array_create_dir()
9977 init_tracer_tracefs(tr, tr->dir); in trace_array_create_dir()
9978 __update_tracer_options(tr); in trace_array_create_dir()
9988 struct trace_array *tr; in trace_array_create_systems() local
9992 tr = kzalloc(sizeof(*tr), GFP_KERNEL); in trace_array_create_systems()
9993 if (!tr) in trace_array_create_systems()
9996 tr->name = kstrdup(name, GFP_KERNEL); in trace_array_create_systems()
9997 if (!tr->name) in trace_array_create_systems()
10000 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL)) in trace_array_create_systems()
10003 if (!zalloc_cpumask_var(&tr->pipe_cpumask, GFP_KERNEL)) in trace_array_create_systems()
10007 tr->system_names = kstrdup_const(systems, GFP_KERNEL); in trace_array_create_systems()
10008 if (!tr->system_names) in trace_array_create_systems()
10013 tr->range_addr_start = range_addr_start; in trace_array_create_systems()
10014 tr->range_addr_size = range_addr_size; in trace_array_create_systems()
10016 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS; in trace_array_create_systems()
10018 cpumask_copy(tr->tracing_cpumask, cpu_all_mask); in trace_array_create_systems()
10020 raw_spin_lock_init(&tr->start_lock); in trace_array_create_systems()
10022 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; in trace_array_create_systems()
10024 spin_lock_init(&tr->snapshot_trigger_lock); in trace_array_create_systems()
10026 tr->current_trace = &nop_trace; in trace_array_create_systems()
10028 INIT_LIST_HEAD(&tr->systems); in trace_array_create_systems()
10029 INIT_LIST_HEAD(&tr->events); in trace_array_create_systems()
10030 INIT_LIST_HEAD(&tr->hist_vars); in trace_array_create_systems()
10031 INIT_LIST_HEAD(&tr->err_log); in trace_array_create_systems()
10032 INIT_LIST_HEAD(&tr->marker_list); in trace_array_create_systems()
10035 INIT_LIST_HEAD(&tr->mod_events); in trace_array_create_systems()
10038 if (allocate_trace_buffers(tr, trace_buf_size) < 0) in trace_array_create_systems()
10042 trace_set_ring_buffer_expanded(tr); in trace_array_create_systems()
10044 if (ftrace_allocate_ftrace_ops(tr) < 0) in trace_array_create_systems()
10047 ftrace_init_trace_array(tr); in trace_array_create_systems()
10049 init_trace_flags_index(tr); in trace_array_create_systems()
10052 ret = trace_array_create_dir(tr); in trace_array_create_systems()
10056 __trace_early_add_events(tr); in trace_array_create_systems()
10058 list_add(&tr->list, &ftrace_trace_arrays); in trace_array_create_systems()
10060 tr->ref++; in trace_array_create_systems()
10062 return tr; in trace_array_create_systems()
10065 ftrace_free_ftrace_ops(tr); in trace_array_create_systems()
10066 free_trace_buffers(tr); in trace_array_create_systems()
10067 free_cpumask_var(tr->pipe_cpumask); in trace_array_create_systems()
10068 free_cpumask_var(tr->tracing_cpumask); in trace_array_create_systems()
10069 kfree_const(tr->system_names); in trace_array_create_systems()
10070 kfree(tr->range_name); in trace_array_create_systems()
10071 kfree(tr->name); in trace_array_create_systems()
10072 kfree(tr); in trace_array_create_systems()
10084 struct trace_array *tr; in instance_mkdir() local
10094 tr = trace_array_create(name); in instance_mkdir()
10096 ret = PTR_ERR_OR_ZERO(tr); in instance_mkdir()
10150 struct trace_array *tr; in trace_array_get_by_name() local
10155 list_for_each_entry(tr, &ftrace_trace_arrays, list) { in trace_array_get_by_name()
10156 if (tr->name && strcmp(tr->name, name) == 0) { in trace_array_get_by_name()
10157 tr->ref++; in trace_array_get_by_name()
10158 return tr; in trace_array_get_by_name()
10162 tr = trace_array_create_systems(name, systems, 0, 0); in trace_array_get_by_name()
10164 if (IS_ERR(tr)) in trace_array_get_by_name()
10165 tr = NULL; in trace_array_get_by_name()
10167 tr->ref++; in trace_array_get_by_name()
10169 return tr; in trace_array_get_by_name()
10173 static int __remove_instance(struct trace_array *tr) in __remove_instance() argument
10178 if (tr->ref > 1 || (tr->current_trace && tr->trace_ref)) in __remove_instance()
10181 list_del(&tr->list); in __remove_instance()
10186 set_tracer_flag(tr, 1 << i, 0); in __remove_instance()
10189 if (printk_trace == tr) in __remove_instance()
10192 if (update_marker_trace(tr, 0)) in __remove_instance()
10195 tracing_set_nop(tr); in __remove_instance()
10196 clear_ftrace_function_probes(tr); in __remove_instance()
10197 event_trace_del_tracer(tr); in __remove_instance()
10198 ftrace_clear_pids(tr); in __remove_instance()
10199 ftrace_destroy_function_files(tr); in __remove_instance()
10200 tracefs_remove(tr->dir); in __remove_instance()
10201 free_percpu(tr->last_func_repeats); in __remove_instance()
10202 free_trace_buffers(tr); in __remove_instance()
10203 clear_tracing_err_log(tr); in __remove_instance()
10205 if (tr->range_name) { in __remove_instance()
10206 reserve_mem_release_by_name(tr->range_name); in __remove_instance()
10207 kfree(tr->range_name); in __remove_instance()
10210 for (i = 0; i < tr->nr_topts; i++) { in __remove_instance()
10211 kfree(tr->topts[i].topts); in __remove_instance()
10213 kfree(tr->topts); in __remove_instance()
10215 free_cpumask_var(tr->pipe_cpumask); in __remove_instance()
10216 free_cpumask_var(tr->tracing_cpumask); in __remove_instance()
10217 kfree_const(tr->system_names); in __remove_instance()
10218 kfree(tr->name); in __remove_instance()
10219 kfree(tr); in __remove_instance()
10226 struct trace_array *tr; in trace_array_destroy() local
10236 list_for_each_entry(tr, &ftrace_trace_arrays, list) { in trace_array_destroy()
10237 if (tr == this_tr) in trace_array_destroy()
10238 return __remove_instance(tr); in trace_array_destroy()
10247 struct trace_array *tr; in instance_rmdir() local
10252 tr = trace_array_find(name); in instance_rmdir()
10253 if (!tr) in instance_rmdir()
10256 return __remove_instance(tr); in instance_rmdir()
10261 struct trace_array *tr; in create_trace_instances() local
10272 list_for_each_entry(tr, &ftrace_trace_arrays, list) { in create_trace_instances()
10273 if (!tr->name) in create_trace_instances()
10275 if (MEM_FAIL(trace_array_create_dir(tr) < 0, in create_trace_instances()
10282 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer) in init_tracer_tracefs() argument
10287 tr, &show_traces_fops); in init_tracer_tracefs()
10290 tr, &set_tracer_fops); in init_tracer_tracefs()
10293 tr, &tracing_cpumask_fops); in init_tracer_tracefs()
10296 tr, &tracing_iter_fops); in init_tracer_tracefs()
10299 tr, &tracing_fops); in init_tracer_tracefs()
10302 tr, &tracing_pipe_fops); in init_tracer_tracefs()
10305 tr, &tracing_entries_fops); in init_tracer_tracefs()
10308 tr, &tracing_total_entries_fops); in init_tracer_tracefs()
10311 tr, &tracing_free_buffer_fops); in init_tracer_tracefs()
10314 tr, &tracing_mark_fops); in init_tracer_tracefs()
10316 tr->trace_marker_file = __find_event_file(tr, "ftrace", "print"); in init_tracer_tracefs()
10319 tr, &tracing_mark_raw_fops); in init_tracer_tracefs()
10321 trace_create_file("trace_clock", TRACE_MODE_WRITE, d_tracer, tr, in init_tracer_tracefs()
10325 tr, &rb_simple_fops); in init_tracer_tracefs()
10327 trace_create_file("timestamp_mode", TRACE_MODE_READ, d_tracer, tr, in init_tracer_tracefs()
10330 tr->buffer_percent = 50; in init_tracer_tracefs()
10333 tr, &buffer_percent_fops); in init_tracer_tracefs()
10336 tr, &buffer_subbuf_size_fops); in init_tracer_tracefs()
10338 create_trace_options_dir(tr); in init_tracer_tracefs()
10341 trace_create_maxlat_file(tr, d_tracer); in init_tracer_tracefs()
10344 if (ftrace_create_function_files(tr, d_tracer)) in init_tracer_tracefs()
10347 if (tr->range_addr_start) { in init_tracer_tracefs()
10349 tr, &last_boot_fops); in init_tracer_tracefs()
10353 tr, &snapshot_fops); in init_tracer_tracefs()
10358 tr, &tracing_err_log_fops); in init_tracer_tracefs()
10361 tracing_init_tracefs_percpu(tr, cpu); in init_tracer_tracefs()
10363 ftrace_init_tracefs(tr, d_tracer); in init_tracer_tracefs()
10410 struct trace_array *tr = &global_trace; in tracing_init_dentry() local
10418 if (tr->dir) in tracing_init_dentry()
10431 tr->dir = debugfs_create_automount("tracing", NULL, in tracing_init_dentry()
10546 struct trace_array *tr; in trace_module_record() local
10549 list_for_each_entry(tr, &ftrace_trace_arrays, list) { in trace_module_record()
10550 flags = tr->flags & (TRACE_ARRAY_FL_BOOT | TRACE_ARRAY_FL_LAST_BOOT); in trace_module_record()
10554 save_mod(mod, tr); in trace_module_record()
10557 make_mod_delta(mod, tr); in trace_module_record()
10720 static void trace_init_iter(struct trace_iterator *iter, struct trace_array *tr) in trace_init_iter() argument
10722 iter->tr = tr; in trace_init_iter()
10723 iter->trace = iter->tr->current_trace; in trace_init_iter()
10725 iter->array_buffer = &tr->array_buffer; in trace_init_iter()
10735 if (trace_clocks[iter->tr->clock_id].in_ns) in trace_init_iter()
10750 static void ftrace_dump_one(struct trace_array *tr, enum ftrace_dump_mode dump_mode) in ftrace_dump_one() argument
10766 tracer_tracing_off(tr); in ftrace_dump_one()
10771 trace_init_iter(&iter, tr); in ftrace_dump_one()
10774 tracer_tracing_disable(tr); in ftrace_dump_one()
10776 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ; in ftrace_dump_one()
10779 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ; in ftrace_dump_one()
10786 if (tr == &global_trace) in ftrace_dump_one()
10789 printk(KERN_TRACE "Dumping ftrace instance %s buffer:\n", tr->name); in ftrace_dump_one()
10831 tr->trace_flags |= old_userobj; in ftrace_dump_one()
10833 tracer_tracing_enable(tr); in ftrace_dump_one()
10842 struct trace_array *tr; in ftrace_dump_by_param() local
10864 tr = trace_array_find(inst_name); in ftrace_dump_by_param()
10865 if (!tr) { in ftrace_dump_by_param()
10872 ftrace_dump_one(tr, DUMP_ORIG); in ftrace_dump_by_param()
10874 ftrace_dump_one(tr, DUMP_ALL); in ftrace_dump_by_param()
11013 struct trace_array *tr; in enable_instances() local
11121 tr = trace_array_create_systems(name, NULL, addr, size); in enable_instances()
11122 if (IS_ERR(tr)) { in enable_instances()
11128 tracer_tracing_off(tr); in enable_instances()
11131 update_printk_trace(tr); in enable_instances()
11137 tr->flags |= TRACE_ARRAY_FL_MEMMAP; in enable_instances()
11138 tr->ref++; in enable_instances()
11142 tr->flags |= TRACE_ARRAY_FL_BOOT | TRACE_ARRAY_FL_LAST_BOOT; in enable_instances()
11143 tr->range_name = no_free_ptr(rname); in enable_instances()
11147 early_enable_events(tr, tok, true); in enable_instances()
11302 struct trace_array *tr; in ftrace_boot_snapshot() local
11307 list_for_each_entry(tr, &ftrace_trace_arrays, list) { in ftrace_boot_snapshot()
11308 if (!tr->allocated_snapshot) in ftrace_boot_snapshot()
11311 tracing_snapshot_instance(tr); in ftrace_boot_snapshot()
11312 trace_array_puts(tr, "** Boot snapshot taken **\n"); in ftrace_boot_snapshot()