1 // SPDX-License-Identifier: GPL-2.0
5  * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
12  *  Copyright (C) 2004-2006 Ingo Molnar
61  * A selftest will lurk into the ring-buffer to count the
63  * insertions into the ring-buffer such as trace_printk could occurred
69  * If boot-time tracing including tracers/events via kernel cmdline
122  * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
135  * Set instance name if you want to dump the specific trace instance
225 		strscpy(ftrace_dump_on_oops + 1, str, MAX_TRACER_SIZE - 1);  in set_ftrace_dump_on_oops()
249 	int left = sizeof(boot_snapshot_info) - boot_snapshot_index;  in boot_alloc_snapshot()
255 			return -1;  in boot_alloc_snapshot()
281 	int left = sizeof(boot_instance_info) - boot_instance_index;  in boot_instance()
285 		return -1;  in boot_instance()
355 	if (export->flags & flag) {  in trace_process_export()
358 		export->write(export, entry, size);  in trace_process_export()
372 	if (export->flags & TRACE_EXPORT_FUNCTION)  in ftrace_exports_enable()
375 	if (export->flags & TRACE_EXPORT_EVENT)  in ftrace_exports_enable()
378 	if (export->flags & TRACE_EXPORT_MARKER)  in ftrace_exports_enable()
384 	if (export->flags & TRACE_EXPORT_FUNCTION)  in ftrace_exports_disable()
387 	if (export->flags & TRACE_EXPORT_EVENT)  in ftrace_exports_disable()
390 	if (export->flags & TRACE_EXPORT_MARKER)  in ftrace_exports_disable()
403 		export = rcu_dereference_raw_check(export->next);  in ftrace_exports()
412 	rcu_assign_pointer(export->next, *list);  in add_trace_export()
416 	 * the export->next pointer is valid before another CPU sees  in add_trace_export()
427 	for (p = list; *p != NULL; p = &(*p)->next)  in rm_trace_export()
432 		return -1;  in rm_trace_export()
434 	rcu_assign_pointer(*p, (*p)->next);  in rm_trace_export()
460 	if (WARN_ON_ONCE(!export->write))  in register_ftrace_export()
461 		return -1;  in register_ftrace_export()
505  * The global_trace is the descriptor that holds the top-level tracing
522 	return !(tr->flags & TRACE_ARRAY_FL_BOOT);  in printk_binsafe()
530 	printk_trace->trace_flags &= ~TRACE_ITER_TRACE_PRINTK;  in update_printk_trace()
532 	tr->trace_flags |= TRACE_ITER_TRACE_PRINTK;  in update_printk_trace()
539 	tr->ring_buffer_expanded = true;  in trace_set_ring_buffer_expanded()
551 			tr->ref++;  in trace_array_get()
556 	return -ENODEV;  in trace_array_get()
561 	WARN_ON(!this_tr->ref);  in __trace_array_put()
562 	this_tr->ref--;  in __trace_array_put()
566  * trace_array_put - Decrement the reference counter for this trace array.
594 		return -ENODEV;  in tracing_check_open_get_tr()
597 		return -ENODEV;  in tracing_check_open_get_tr()
603  * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
616  * trace_ignore_this_task - should a task be ignored for tracing
639 		!trace_find_filtered_pid(filtered_pids, task->pid)) ||  in trace_ignore_this_task()
641 		 trace_find_filtered_pid(filtered_no_pids, task->pid));  in trace_ignore_this_task()
645  * trace_filter_add_remove_task - Add or remove a task from a pid_list
665 		if (!trace_find_filtered_pid(pid_list, self->pid))  in trace_filter_add_remove_task()
671 		trace_pid_list_set(pid_list, task->pid);  in trace_filter_add_remove_task()
673 		trace_pid_list_clear(pid_list, task->pid);  in trace_filter_add_remove_task()
677  * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
706  * trace_pid_start - Used for seq_file to start reading pid lists
735  * trace_pid_show - show the current pid in seq_file processing
744 	unsigned long pid = (unsigned long)v - 1;  in trace_pid_show()
755 		    const char __user *ubuf, size_t cnt)  in trace_pid_write()  argument
767 		return -ENOMEM;  in trace_pid_write()
778 		return -ENOMEM;  in trace_pid_write()
792 	while (cnt > 0) {  in trace_pid_write()
796 		ret = trace_get_user(&parser, ubuf, cnt, &pos);  in trace_pid_write()
802 		cnt -= ret;  in trace_pid_write()
807 		ret = -EINVAL;  in trace_pid_write()
814 			ret = -1;  in trace_pid_write()
845 	if (!buf->buffer)  in buffer_ftrace_now()
848 	ts = ring_buffer_time_stamp(buf->buffer);  in buffer_ftrace_now()
849 	ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);  in buffer_ftrace_now()
860  * tracing_is_enabled - Show if global_trace has been enabled
919  * These primitives don't distinguish read-only and read-consume access.
920  * Multi read-only access are also serialized.
1035 	if (tr->array_buffer.buffer)  in tracer_tracing_on()
1036 		ring_buffer_record_on(tr->array_buffer.buffer);  in tracer_tracing_on()
1045 	tr->buffer_disabled = 0;  in tracer_tracing_on()
1051  * tracing_on - enable tracing buffers
1070 		/* Length is in event->array[0] */  in __buffer_unlock_commit()
1071 		ring_buffer_write(buffer, event->array[0], &event->array[1]);  in __buffer_unlock_commit()
1089 	if (!(tr->trace_flags & TRACE_ITER_PRINTK))  in __trace_array_puts()
1101 	buffer = tr->array_buffer.buffer;  in __trace_array_puts()
1111 	entry->ip = ip;  in __trace_array_puts()
1113 	memcpy(&entry->buf, str, size);  in __trace_array_puts()
1116 	if (entry->buf[size - 1] != '\n') {  in __trace_array_puts()
1117 		entry->buf[size] = '\n';  in __trace_array_puts()
1118 		entry->buf[size + 1] = '\0';  in __trace_array_puts()
1120 		entry->buf[size] = '\0';  in __trace_array_puts()
1131  * __trace_puts - write a constant string into the trace buffer.
1143  * __trace_bputs - write the pointer to a constant string into trace buffer
1160 	if (!(tr->trace_flags & TRACE_ITER_PRINTK))  in __trace_bputs()
1167 	buffer = tr->array_buffer.buffer;  in __trace_bputs()
1176 	entry->ip			= ip;  in __trace_bputs()
1177 	entry->str			= str;  in __trace_bputs()
1193 	struct tracer *tracer = tr->current_trace;  in tracing_snapshot_instance_cond()
1202 	if (!tr->allocated_snapshot) {  in tracing_snapshot_instance_cond()
1210 	if (tracer->use_max_tr) {  in tracing_snapshot_instance_cond()
1216 	if (tr->mapped) {  in tracing_snapshot_instance_cond()
1233  * tracing_snapshot - take a snapshot of the current buffer.
1255  * tracing_snapshot_cond - conditionally take a snapshot of the current buffer.
1260  * conditional - the snapshot will only happen if the
1274  * tracing_cond_snapshot_data - get the user data associated with a snapshot
1278  * tracing_snapshot_cond_enable(), the user-defined cond_data is saved
1282  * the tr->max_lock lock, which the code calling
1292 	arch_spin_lock(&tr->max_lock);  in tracing_cond_snapshot_data()
1294 	if (tr->cond_snapshot)  in tracing_cond_snapshot_data()
1295 		cond_data = tr->cond_snapshot->cond_data;  in tracing_cond_snapshot_data()
1297 	arch_spin_unlock(&tr->max_lock);  in tracing_cond_snapshot_data()
1313 	if (!tr->allocated_snapshot) {  in tracing_alloc_snapshot_instance()
1316 		order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer);  in tracing_alloc_snapshot_instance()
1317 		ret = ring_buffer_subbuf_order_set(tr->max_buffer.buffer, order);  in tracing_alloc_snapshot_instance()
1322 		ret = resize_buffer_duplicate_size(&tr->max_buffer,  in tracing_alloc_snapshot_instance()
1323 				   &tr->array_buffer, RING_BUFFER_ALL_CPUS);  in tracing_alloc_snapshot_instance()
1327 		tr->allocated_snapshot = true;  in tracing_alloc_snapshot_instance()
1337 	 * The max_tr ring buffer has some state (e.g. ring->clock) and  in free_snapshot()
1340 	ring_buffer_subbuf_order_set(tr->max_buffer.buffer, 0);  in free_snapshot()
1341 	ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);  in free_snapshot()
1342 	set_buffer_entries(&tr->max_buffer, 1);  in free_snapshot()
1343 	tracing_reset_online_cpus(&tr->max_buffer);  in free_snapshot()
1344 	tr->allocated_snapshot = false;  in free_snapshot()
1353 	spin_lock(&tr->snapshot_trigger_lock);  in tracing_arm_snapshot_locked()
1354 	if (tr->snapshot == UINT_MAX || tr->mapped) {  in tracing_arm_snapshot_locked()
1355 		spin_unlock(&tr->snapshot_trigger_lock);  in tracing_arm_snapshot_locked()
1356 		return -EBUSY;  in tracing_arm_snapshot_locked()
1359 	tr->snapshot++;  in tracing_arm_snapshot_locked()
1360 	spin_unlock(&tr->snapshot_trigger_lock);  in tracing_arm_snapshot_locked()
1364 		spin_lock(&tr->snapshot_trigger_lock);  in tracing_arm_snapshot_locked()
1365 		tr->snapshot--;  in tracing_arm_snapshot_locked()
1366 		spin_unlock(&tr->snapshot_trigger_lock);  in tracing_arm_snapshot_locked()
1385 	spin_lock(&tr->snapshot_trigger_lock);  in tracing_disarm_snapshot()
1386 	if (!WARN_ON(!tr->snapshot))  in tracing_disarm_snapshot()
1387 		tr->snapshot--;  in tracing_disarm_snapshot()
1388 	spin_unlock(&tr->snapshot_trigger_lock);  in tracing_disarm_snapshot()
1392  * tracing_alloc_snapshot - allocate snapshot buffer.
1395  * allocated - it doesn't also take a snapshot.
1414  * tracing_snapshot_alloc - allocate and take a snapshot of the current buffer.
1437  * tracing_snapshot_cond_enable - enable conditional snapshot for an instance
1444  * snapshot; if so, return -EBUSY, else create a cond_snapshot and
1457 		return -ENOMEM;  in tracing_snapshot_cond_enable()
1459 	cond_snapshot->cond_data = cond_data;  in tracing_snapshot_cond_enable()
1460 	cond_snapshot->update = update;  in tracing_snapshot_cond_enable()
1464 	if (tr->current_trace->use_max_tr)  in tracing_snapshot_cond_enable()
1465 		return -EBUSY;  in tracing_snapshot_cond_enable()
1475 	if (tr->cond_snapshot)  in tracing_snapshot_cond_enable()
1476 		return -EBUSY;  in tracing_snapshot_cond_enable()
1483 	arch_spin_lock(&tr->max_lock);  in tracing_snapshot_cond_enable()
1484 	tr->cond_snapshot = no_free_ptr(cond_snapshot);  in tracing_snapshot_cond_enable()
1485 	arch_spin_unlock(&tr->max_lock);  in tracing_snapshot_cond_enable()
1493  * tracing_snapshot_cond_disable - disable conditional snapshot for an instance
1498  * otherwise return -EINVAL.
1507 	arch_spin_lock(&tr->max_lock);  in tracing_snapshot_cond_disable()
1509 	if (!tr->cond_snapshot)  in tracing_snapshot_cond_disable()
1510 		ret = -EINVAL;  in tracing_snapshot_cond_disable()
1512 		kfree(tr->cond_snapshot);  in tracing_snapshot_cond_disable()
1513 		tr->cond_snapshot = NULL;  in tracing_snapshot_cond_disable()
1516 	arch_spin_unlock(&tr->max_lock);  in tracing_snapshot_cond_disable()
1538 	return -ENODEV;  in tracing_alloc_snapshot()
1554 	return -ENODEV;  in tracing_snapshot_cond_enable()
1563 #define tracing_arm_snapshot_locked(tr) ({ -EBUSY; })
1568 	if (tr->array_buffer.buffer)  in tracer_tracing_off()
1569 		ring_buffer_record_off(tr->array_buffer.buffer);  in tracer_tracing_off()
1578 	tr->buffer_disabled = 1;  in tracer_tracing_off()
1584  * tracing_off - turn off tracing buffers
1607  * tracer_tracing_is_on - show real state of ring buffer enabled
1614 	if (tr->array_buffer.buffer)  in tracer_tracing_is_on()
1615 		return ring_buffer_record_is_set_on(tr->array_buffer.buffer);  in tracer_tracing_is_on()
1616 	return !tr->buffer_disabled;  in tracer_tracing_is_on()
1620  * tracing_is_on - show state of ring buffers enabled
1667  * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1682 	const char *name;  member
1699 	if (trace_clocks[tr->clock_id].in_ns)  in trace_clock_in_ns()
1706  * trace_parser_get_init - gets the buffer for trace parser
1712 	parser->buffer = kmalloc(size, GFP_KERNEL);  in trace_parser_get_init()
1713 	if (!parser->buffer)  in trace_parser_get_init()
1716 	parser->size = size;  in trace_parser_get_init()
1721  * trace_parser_put - frees the buffer for trace parser
1725 	kfree(parser->buffer);  in trace_parser_put()
1726 	parser->buffer = NULL;  in trace_parser_put()
1730  * trace_get_user - reads the user input string separated by  space
1741 	size_t cnt, loff_t *ppos)  in trace_get_user()  argument
1755 	cnt--;  in trace_get_user()
1761 	if (!parser->cont) {  in trace_get_user()
1763 		while (cnt && isspace(ch)) {  in trace_get_user()
1768 			cnt--;  in trace_get_user()
1771 		parser->idx = 0;  in trace_get_user()
1781 	/* read the non-space input */  in trace_get_user()
1782 	while (cnt && !isspace(ch) && ch) {  in trace_get_user()
1783 		if (parser->idx < parser->size - 1)  in trace_get_user()
1784 			parser->buffer[parser->idx++] = ch;  in trace_get_user()
1786 			ret = -EINVAL;  in trace_get_user()
1793 		cnt--;  in trace_get_user()
1798 		parser->buffer[parser->idx] = 0;  in trace_get_user()
1799 		parser->cont = false;  in trace_get_user()
1800 	} else if (parser->idx < parser->size - 1) {  in trace_get_user()
1801 		parser->cont = true;  in trace_get_user()
1802 		parser->buffer[parser->idx++] = ch;  in trace_get_user()
1804 		parser->buffer[parser->idx] = 0;  in trace_get_user()
1806 		ret = -EINVAL;  in trace_get_user()
1818 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)  in trace_seq_to_buffer()  argument
1822 	if (trace_seq_used(s) <= s->readpos)  in trace_seq_to_buffer()
1823 		return -EBUSY;  in trace_seq_to_buffer()
1825 	len = trace_seq_used(s) - s->readpos;  in trace_seq_to_buffer()
1826 	if (cnt > len)  in trace_seq_to_buffer()
1827 		cnt = len;  in trace_seq_to_buffer()
1828 	memcpy(buf, s->buffer + s->readpos, cnt);  in trace_seq_to_buffer()
1830 	s->readpos += cnt;  in trace_seq_to_buffer()
1831 	return cnt;  in trace_seq_to_buffer()
1847 	fsnotify_inode(tr->d_max_latency->d_inode, FS_MODIFY);  in latency_fsnotify_workfn()
1854 	queue_work(fsnotify_wq, &tr->fsnotify_work);  in latency_fsnotify_workfn_irq()
1860 	INIT_WORK(&tr->fsnotify_work, latency_fsnotify_workfn);  in trace_create_maxlat_file()
1861 	init_irq_work(&tr->fsnotify_irqwork, latency_fsnotify_workfn_irq);  in trace_create_maxlat_file()
1862 	tr->d_max_latency = trace_create_file("tracing_max_latency",  in trace_create_maxlat_file()
1874 		return -ENOMEM;  in latency_fsnotify_init()
1886 	 * We cannot call queue_work(&tr->fsnotify_work) from here because it's  in latency_fsnotify()
1890 	irq_work_queue(&tr->fsnotify_irqwork);  in latency_fsnotify()
1902  * Copy the new maximum trace into the separate maximum-trace
1909 	struct array_buffer *trace_buf = &tr->array_buffer;  in __update_max_tr()
1910 	struct array_buffer *max_buf = &tr->max_buffer;  in __update_max_tr()
1911 	struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);  in __update_max_tr()
1912 	struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);  in __update_max_tr()
1914 	max_buf->cpu = cpu;  in __update_max_tr()
1915 	max_buf->time_start = data->preempt_timestamp;  in __update_max_tr()
1917 	max_data->saved_latency = tr->max_latency;  in __update_max_tr()
1918 	max_data->critical_start = data->critical_start;  in __update_max_tr()
1919 	max_data->critical_end = data->critical_end;  in __update_max_tr()
1921 	strscpy(max_data->comm, tsk->comm);  in __update_max_tr()
1922 	max_data->pid = tsk->pid;  in __update_max_tr()
1928 		max_data->uid = current_uid();  in __update_max_tr()
1930 		max_data->uid = task_uid(tsk);  in __update_max_tr()
1932 	max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;  in __update_max_tr()
1933 	max_data->policy = tsk->policy;  in __update_max_tr()
1934 	max_data->rt_priority = tsk->rt_priority;  in __update_max_tr()
1942  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1955 	if (tr->stop_count)  in update_max_tr()
1960 	if (!tr->allocated_snapshot) {  in update_max_tr()
1962 		WARN_ON_ONCE(tr->current_trace != &nop_trace);  in update_max_tr()
1966 	arch_spin_lock(&tr->max_lock);  in update_max_tr()
1969 	if (ring_buffer_record_is_set_on(tr->array_buffer.buffer))  in update_max_tr()
1970 		ring_buffer_record_on(tr->max_buffer.buffer);  in update_max_tr()
1972 		ring_buffer_record_off(tr->max_buffer.buffer);  in update_max_tr()
1975 	if (tr->cond_snapshot && !tr->cond_snapshot->update(tr, cond_data)) {  in update_max_tr()
1976 		arch_spin_unlock(&tr->max_lock);  in update_max_tr()
1980 	swap(tr->array_buffer.buffer, tr->max_buffer.buffer);  in update_max_tr()
1984 	arch_spin_unlock(&tr->max_lock);  in update_max_tr()
1987 	ring_buffer_wake_waiters(tr->array_buffer.buffer, RING_BUFFER_ALL_CPUS);  in update_max_tr()
1991  * update_max_tr_single - only copy one trace over, and reset the rest
2003 	if (tr->stop_count)  in update_max_tr_single()
2007 	if (!tr->allocated_snapshot) {  in update_max_tr_single()
2009 		WARN_ON_ONCE(tr->current_trace != &nop_trace);  in update_max_tr_single()
2013 	arch_spin_lock(&tr->max_lock);  in update_max_tr_single()
2015 	ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->array_buffer.buffer, cpu);  in update_max_tr_single()
2017 	if (ret == -EBUSY) {  in update_max_tr_single()
2025 		trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,  in update_max_tr_single()
2029 	WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);  in update_max_tr_single()
2032 	arch_spin_unlock(&tr->max_lock);  in update_max_tr_single()
2045 	struct trace_iterator *iter = pwait->iter;  in wait_pipe_cond()
2047 	if (atomic_read_acquire(&iter->wait_index) != pwait->wait_index)  in wait_pipe_cond()
2050 	return iter->closed;  in wait_pipe_cond()
2059 	if (trace_buffer_iter(iter, iter->cpu_file))  in wait_on_pipe()
2062 	pwait.wait_index = atomic_read_acquire(&iter->wait_index);  in wait_on_pipe()
2065 	ret = ring_buffer_wait(iter->array_buffer->buffer, iter->cpu_file, full,  in wait_on_pipe()
2073 	if (iter->snapshot)  in wait_on_pipe()
2074 		iter->array_buffer = &iter->tr->max_buffer;  in wait_on_pipe()
2095 		return -ENOMEM;  in save_selftest()
2097 	selftest->type = type;  in save_selftest()
2098 	list_add(&selftest->list, &postponed_selftests);  in save_selftest()
2105 	struct tracer *saved_tracer = tr->current_trace;  in run_tracer_selftest()
2108 	if (!type->selftest || tracing_selftest_disabled)  in run_tracer_selftest()
2121 			type->name);  in run_tracer_selftest()
2132 	tracing_reset_online_cpus(&tr->array_buffer);  in run_tracer_selftest()
2134 	tr->current_trace = type;  in run_tracer_selftest()
2137 	if (type->use_max_tr) {  in run_tracer_selftest()
2139 		if (tr->ring_buffer_expanded)  in run_tracer_selftest()
2140 			ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,  in run_tracer_selftest()
2142 		tr->allocated_snapshot = true;  in run_tracer_selftest()
2147 	pr_info("Testing tracer %s: ", type->name);  in run_tracer_selftest()
2148 	ret = type->selftest(type, tr);  in run_tracer_selftest()
2150 	tr->current_trace = saved_tracer;  in run_tracer_selftest()
2155 		return -1;  in run_tracer_selftest()
2158 	tracing_reset_online_cpus(&tr->array_buffer);  in run_tracer_selftest()
2161 	if (type->use_max_tr) {  in run_tracer_selftest()
2162 		tr->allocated_snapshot = false;  in run_tracer_selftest()
2165 		if (tr->ring_buffer_expanded)  in run_tracer_selftest()
2166 			ring_buffer_resize(tr->max_buffer.buffer, 1,  in run_tracer_selftest()
2214 		ret = run_tracer_selftest(p->type);  in init_trace_selftests()
2218 			     p->type->name);  in init_trace_selftests()
2220 			for (t = trace_types; t; t = t->next) {  in init_trace_selftests()
2221 				if (t == p->type) {  in init_trace_selftests()
2222 					*last = t->next;  in init_trace_selftests()
2225 				last = &t->next;  in init_trace_selftests()
2228 		list_del(&p->list);  in init_trace_selftests()
2248  * register_tracer - register a tracer with the ftrace system.
2258 	if (!type->name) {  in register_tracer()
2259 		pr_info("Tracer must have a name\n");  in register_tracer()
2260 		return -1;  in register_tracer()
2263 	if (strlen(type->name) >= MAX_TRACER_SIZE) {  in register_tracer()
2264 		pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);  in register_tracer()
2265 		return -1;  in register_tracer()
2270 			   type->name);  in register_tracer()
2271 		return -EPERM;  in register_tracer()
2276 	for (t = trace_types; t; t = t->next) {  in register_tracer()
2277 		if (strcmp(type->name, t->name) == 0) {  in register_tracer()
2280 				type->name);  in register_tracer()
2281 			ret = -1;  in register_tracer()
2286 	if (!type->set_flag)  in register_tracer()
2287 		type->set_flag = &dummy_set_flag;  in register_tracer()
2288 	if (!type->flags) {  in register_tracer()
2290 		type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);  in register_tracer()
2291 		if (!type->flags) {  in register_tracer()
2292 			ret = -ENOMEM;  in register_tracer()
2295 		type->flags->val = 0;  in register_tracer()
2296 		type->flags->opts = dummy_tracer_opt;  in register_tracer()
2298 		if (!type->flags->opts)  in register_tracer()
2299 			type->flags->opts = dummy_tracer_opt;  in register_tracer()
2302 	type->flags->trace = type;  in register_tracer()
2308 	type->next = trace_types;  in register_tracer()
2318 	if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))  in register_tracer()
2321 	printk(KERN_INFO "Starting tracer '%s'\n", type->name);  in register_tracer()
2323 	tracing_set_tracer(&global_trace, type->name);  in register_tracer()
2337 	struct trace_buffer *buffer = buf->buffer;  in tracing_reset_cpu()
2353 	struct trace_buffer *buffer = buf->buffer;  in tracing_reset_online_cpus()
2363 	buf->time_start = buffer_ftrace_now(buf, buf->cpu);  in tracing_reset_online_cpus()
2372 	struct trace_buffer *buffer = buf->buffer;  in tracing_reset_all_cpus()
2382 	buf->time_start = buffer_ftrace_now(buf, buf->cpu);  in tracing_reset_all_cpus()
2397 		if (!tr->clear_trace)  in tracing_reset_all_online_cpus_unlocked()
2399 		tr->clear_trace = false;  in tracing_reset_all_online_cpus_unlocked()
2400 		tracing_reset_online_cpus(&tr->array_buffer);  in tracing_reset_all_online_cpus_unlocked()
2402 		tracing_reset_online_cpus(&tr->max_buffer);  in tracing_reset_all_online_cpus_unlocked()
2427 	raw_spin_lock_irqsave(&tr->start_lock, flags);  in tracing_start_tr()
2428 	if (--tr->stop_count) {  in tracing_start_tr()
2429 		if (WARN_ON_ONCE(tr->stop_count < 0)) {  in tracing_start_tr()
2431 			tr->stop_count = 0;  in tracing_start_tr()
2437 	arch_spin_lock(&tr->max_lock);  in tracing_start_tr()
2439 	buffer = tr->array_buffer.buffer;  in tracing_start_tr()
2444 	buffer = tr->max_buffer.buffer;  in tracing_start_tr()
2449 	arch_spin_unlock(&tr->max_lock);  in tracing_start_tr()
2452 	raw_spin_unlock_irqrestore(&tr->start_lock, flags);  in tracing_start_tr()
2456  * tracing_start - quick start of the tracer
2472 	raw_spin_lock_irqsave(&tr->start_lock, flags);  in tracing_stop_tr()
2473 	if (tr->stop_count++)  in tracing_stop_tr()
2477 	arch_spin_lock(&tr->max_lock);  in tracing_stop_tr()
2479 	buffer = tr->array_buffer.buffer;  in tracing_stop_tr()
2484 	buffer = tr->max_buffer.buffer;  in tracing_stop_tr()
2489 	arch_spin_unlock(&tr->max_lock);  in tracing_stop_tr()
2492 	raw_spin_unlock_irqrestore(&tr->start_lock, flags);  in tracing_stop_tr()
2496  * tracing_stop - quick stop of the tracer
2521 	return current->migration_disabled;  in migration_disable_value()
2567  * trace_buffered_event_enable - enable buffering events
2627  * trace_buffered_event_disable - disable buffering events
2643 	if (--trace_buffered_event_ref)  in trace_buffered_event_disable()
2663 	 * could wrongly decide to use the pointed-to buffer which is now freed.  in trace_buffered_event_disable()
2681 	struct trace_array *tr = trace_file->tr;  in trace_event_buffer_lock_reserve()
2684 	*current_rb = tr->array_buffer.buffer;  in trace_event_buffer_lock_reserve()
2686 	if (!tr->no_filter_buffering_ref &&  in trace_event_buffer_lock_reserve()
2687 	    (trace_file->flags & (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED))) {  in trace_event_buffer_lock_reserve()
2694 		 * (see include/linux/ring-buffer.h for details on  in trace_event_buffer_lock_reserve()
2707 			int max_len = PAGE_SIZE - struct_size(entry, array, 1);  in trace_event_buffer_lock_reserve()
2730 				entry->array[0] = len;  in trace_event_buffer_lock_reserve()
2748 	if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {  in trace_event_buffer_lock_reserve()
2772 	event_call = fbuffer->trace_file->event_call;  in output_printk()
2773 	if (!event_call || !event_call->event.funcs ||  in output_printk()
2774 	    !event_call->event.funcs->trace)  in output_printk()
2777 	file = fbuffer->trace_file;  in output_printk()
2778 	if (test_bit(EVENT_FILE_FL_SOFT_DISABLED_BIT, &file->flags) ||  in output_printk()
2779 	    (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&  in output_printk()
2780 	     !filter_match_preds(file->filter, fbuffer->entry)))  in output_printk()
2783 	event = &fbuffer->trace_file->event_call->event;  in output_printk()
2786 	trace_seq_init(&iter->seq);  in output_printk()
2787 	iter->ent = fbuffer->entry;  in output_printk()
2788 	event_call->event.funcs->trace(iter, 0, event);  in output_printk()
2789 	trace_seq_putc(&iter->seq, 0);  in output_printk()
2790 	printk("%s", iter->seq.buffer);  in output_printk()
2828 	struct trace_event_file *file = fbuffer->trace_file;  in trace_event_buffer_commit()
2830 	if (__event_trigger_test_discard(file, fbuffer->buffer, fbuffer->event,  in trace_event_buffer_commit()
2831 			fbuffer->entry, &tt))  in trace_event_buffer_commit()
2838 		ftrace_exports(fbuffer->event, TRACE_EXPORT_EVENT);  in trace_event_buffer_commit()
2840 	trace_buffer_unlock_commit_regs(file->tr, fbuffer->buffer,  in trace_event_buffer_commit()
2841 			fbuffer->event, fbuffer->trace_ctx, fbuffer->regs);  in trace_event_buffer_commit()
2891 	struct trace_buffer *buffer = tr->array_buffer.buffer;  in trace_function()
2903 	entry->ip			= ip;  in trace_function()
2904 	entry->parent_ip		= parent_ip;  in trace_function()
2909 			entry->args[i] = ftrace_regs_get_argument(fregs, i);  in trace_function()
2959 	stackidx = __this_cpu_inc_return(ftrace_stack_reserve) - 1;  in __ftrace_trace_stack()
2975 	size = ARRAY_SIZE(fstack->calls);  in __ftrace_trace_stack()
2978 		nr_entries = stack_trace_save_regs(regs, fstack->calls,  in __ftrace_trace_stack()
2981 		nr_entries = stack_trace_save(fstack->calls, size, skip);  in __ftrace_trace_stack()
2986 	if (tr->ops && tr->ops->trampoline) {  in __ftrace_trace_stack()
2987 		unsigned long tramp_start = tr->ops->trampoline;  in __ftrace_trace_stack()
2988 		unsigned long tramp_end = tramp_start + tr->ops->trampoline_size;  in __ftrace_trace_stack()
2989 		unsigned long *calls = fstack->calls;  in __ftrace_trace_stack()
3005 	entry->size = nr_entries;  in __ftrace_trace_stack()
3006 	memcpy(&entry->caller, fstack->calls,  in __ftrace_trace_stack()
3024 	if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))  in ftrace_trace_stack()
3033 	struct trace_buffer *buffer = tr->array_buffer.buffer;  in __trace_stack()
3058  * trace_dump_stack - record a stack back trace in the trace buffer
3070 	__ftrace_trace_stack(printk_trace, printk_trace->array_buffer.buffer,  in trace_dump_stack()
3085 	if (!(tr->trace_flags & TRACE_ITER_USERSTACKTRACE))  in ftrace_trace_userstack()
3111 	entry->tgid		= current->tgid;  in ftrace_trace_userstack()
3112 	memset(&entry->caller, 0, sizeof(entry->caller));  in ftrace_trace_userstack()
3114 	stack_trace_save_user(entry->caller, FTRACE_STACK_ENTRIES);  in ftrace_trace_userstack()
3136 	entry->bottom_delta_ts = delta & U32_MAX;  in func_repeats_set_delta_ts()
3137 	entry->top_delta_ts = (delta >> 32);  in func_repeats_set_delta_ts()
3144 	struct trace_buffer *buffer = tr->array_buffer.buffer;  in trace_last_func_repeats()
3154 	delta = ring_buffer_event_time_stamp(buffer, event) -  in trace_last_func_repeats()
3155 		last_info->ts_last_call;  in trace_last_func_repeats()
3158 	entry->ip = last_info->ip;  in trace_last_func_repeats()
3159 	entry->parent_ip = last_info->parent_ip;  in trace_last_func_repeats()
3160 	entry->count = last_info->count;  in trace_last_func_repeats()
3182 	if (!trace_percpu_buffer || buffer->nesting >= 4)  in get_trace_buf()
3185 	buffer->nesting++;  in get_trace_buf()
3189 	return &buffer->buffer[buffer->nesting - 1][0];  in get_trace_buf()
3196 	this_cpu_dec(trace_percpu_buffer->nesting);  in put_trace_buf()
3208 		return -ENOMEM;  in alloc_percpu_trace_buffer()
3277  * trace_vbprintk - write binary msg to tracing buffer
3316 	buffer = tr->array_buffer.buffer;  in trace_vbprintk()
3323 	entry->ip			= ip;  in trace_vbprintk()
3324 	entry->fmt			= fmt;  in trace_vbprintk()
3326 	memcpy(entry->buf, tbuffer, sizeof(u32) * len);  in trace_vbprintk()
3379 	entry->ip = ip;  in __trace_array_vprintk()
3381 	memcpy(&entry->buf, tbuffer, len + 1);  in __trace_array_vprintk()
3403 	return __trace_array_vprintk(tr->array_buffer.buffer, ip, fmt, args);  in trace_array_vprintk()
3407  * trace_array_printk - Print a message to a specific instance
3434 		return -ENOENT;  in trace_array_printk()
3440 	if (!(tr->trace_flags & TRACE_ITER_PRINTK))  in trace_array_printk()
3451  * trace_array_init_printk - Initialize buffers for trace_array_printk()
3461 		return -ENOENT;  in trace_array_init_printk()
3465 		return -EINVAL;  in trace_array_init_printk()
3478 	if (!(printk_trace->trace_flags & TRACE_ITER_PRINTK))  in trace_array_printk_buf()
3496 	struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);  in trace_iterator_increment()
3498 	iter->idx++;  in trace_iterator_increment()
3514 				(unsigned long)-1 : 0;  in peek_next_entry()
3516 		event = ring_buffer_peek(iter->array_buffer->buffer, cpu, ts,  in peek_next_entry()
3521 		iter->ent_size = ring_buffer_event_length(event);  in peek_next_entry()
3524 	iter->ent_size = 0;  in peek_next_entry()
3532 	struct trace_buffer *buffer = iter->array_buffer->buffer;  in __find_next_entry()
3535 	int cpu_file = iter->cpu_file;  in __find_next_entry()
3537 	int next_cpu = -1;  in __find_next_entry()
3570 			next_size = iter->ent_size;  in __find_next_entry()
3574 	iter->ent_size = next_size;  in __find_next_entry()
3596 	 * iter->tr is NULL when used with tp_printk, which makes  in trace_iter_expand_format()
3599 	if (!iter->tr || iter->fmt == static_fmt_buf)  in trace_iter_expand_format()
3602 	tmp = krealloc(iter->fmt, iter->fmt_size + STATIC_FMT_BUF_SIZE,  in trace_iter_expand_format()
3605 		iter->fmt_size += STATIC_FMT_BUF_SIZE;  in trace_iter_expand_format()
3606 		iter->fmt = tmp;  in trace_iter_expand_format()
3620 	if ((addr >= (unsigned long)iter->ent) &&  in trace_safe_str()
3621 	    (addr < (unsigned long)iter->ent + iter->ent_size))  in trace_safe_str()
3625 	if ((addr >= (unsigned long)iter->tmp_seq.buffer) &&  in trace_safe_str()
3626 	    (addr < (unsigned long)iter->tmp_seq.buffer + TRACE_SEQ_BUFFER_SIZE))  in trace_safe_str()
3640 	if (!iter->ent)  in trace_safe_str()
3643 	trace_event = ftrace_find_event(iter->ent->type);  in trace_safe_str()
3648 	if ((event->flags & TRACE_EVENT_FL_DYNAMIC) || !event->module)  in trace_safe_str()
3652 	if (within_module_core(addr, event->module))  in trace_safe_str()
3659  * ignore_event - Check dereferenced fields while writing to the seq buffer
3673  * If it is found that a field is unsafe, it will write into the @iter->seq
3688 	trace_event = ftrace_find_event(iter->ent->type);  in ignore_event()
3690 	seq = &iter->seq;  in ignore_event()
3693 		trace_seq_printf(seq, "EVENT ID %d NOT FOUND?\n", iter->ent->type);  in ignore_event()
3698 	if (!(event->flags & TRACE_EVENT_FL_TEST_STR))  in ignore_event()
3708 	/* Offsets are from the iter->ent that points to the raw event */  in ignore_event()
3709 	ptr = iter->ent;  in ignore_event()
3715 		if (!field->needs_test)  in ignore_event()
3718 		str = *(const char **)(ptr + field->offset);  in ignore_event()
3728 		 * instead. See samples/trace_events/trace-events-sample.h  in ignore_event()
3732 			      trace_event_name(event), field->name)) {  in ignore_event()
3734 					 trace_event_name(event), field->name);  in ignore_event()
3749 	if (!iter->tr || iter->tr->trace_flags & TRACE_ITER_HASH_PTR)  in trace_event_format()
3753 	new_fmt = q = iter->fmt;  in trace_event_format()
3755 		if (unlikely(q - new_fmt + 3 > iter->fmt_size)) {  in trace_event_format()
3759 			q += iter->fmt - new_fmt;  in trace_event_format()
3760 			new_fmt = iter->fmt;  in trace_event_format()
3766 		if (p[-1] == '%') {  in trace_event_format()
3788 	int ent_size = iter->ent_size;  in trace_find_next_entry()
3792 	 * If called from ftrace_dump(), then the iter->temp buffer  in trace_find_next_entry()
3799 	if (iter->temp == static_temp_buf &&  in trace_find_next_entry()
3805 	 * call ring_buffer_peek() that may make the contents of iter->ent  in trace_find_next_entry()
3806 	 * undefined. Need to copy iter->ent now.  in trace_find_next_entry()
3808 	if (iter->ent && iter->ent != iter->temp) {  in trace_find_next_entry()
3809 		if ((!iter->temp || iter->temp_size < iter->ent_size) &&  in trace_find_next_entry()
3810 		    !WARN_ON_ONCE(iter->temp == static_temp_buf)) {  in trace_find_next_entry()
3812 			temp = kmalloc(iter->ent_size, GFP_KERNEL);  in trace_find_next_entry()
3815 			kfree(iter->temp);  in trace_find_next_entry()
3816 			iter->temp = temp;  in trace_find_next_entry()
3817 			iter->temp_size = iter->ent_size;  in trace_find_next_entry()
3819 		memcpy(iter->temp, iter->ent, iter->ent_size);  in trace_find_next_entry()
3820 		iter->ent = iter->temp;  in trace_find_next_entry()
3824 	iter->ent_size = ent_size;  in trace_find_next_entry()
3832 	iter->ent = __find_next_entry(iter, &iter->cpu,  in trace_find_next_entry_inc()
3833 				      &iter->lost_events, &iter->ts);  in trace_find_next_entry_inc()
3835 	if (iter->ent)  in trace_find_next_entry_inc()
3838 	return iter->ent ? iter : NULL;  in trace_find_next_entry_inc()
3843 	ring_buffer_consume(iter->array_buffer->buffer, iter->cpu, &iter->ts,  in trace_consume()
3844 			    &iter->lost_events);  in trace_consume()
3849 	struct trace_iterator *iter = m->private;  in s_next()
3853 	WARN_ON_ONCE(iter->leftover);  in s_next()
3858 	if (iter->idx > i)  in s_next()
3861 	if (iter->idx < 0)  in s_next()
3866 	while (ent && iter->idx < i)  in s_next()
3869 	iter->pos = *pos;  in s_next()
3880 	per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = 0;  in tracing_iter_reset()
3894 		if (ts >= iter->array_buffer->time_start)  in tracing_iter_reset()
3902 	per_cpu_ptr(iter->array_buffer->data, cpu)->skipped_entries = entries;  in tracing_iter_reset()
3911 	struct trace_iterator *iter = m->private;  in s_start()
3912 	struct trace_array *tr = iter->tr;  in s_start()
3913 	int cpu_file = iter->cpu_file;  in s_start()
3919 	if (unlikely(tr->current_trace != iter->trace)) {  in s_start()
3920 		/* Close iter->trace before switching to the new current tracer */  in s_start()
3921 		if (iter->trace->close)  in s_start()
3922 			iter->trace->close(iter);  in s_start()
3923 		iter->trace = tr->current_trace;  in s_start()
3925 		if (iter->trace->open)  in s_start()
3926 			iter->trace->open(iter);  in s_start()
3931 	if (iter->snapshot && iter->trace->use_max_tr)  in s_start()
3932 		return ERR_PTR(-EBUSY);  in s_start()
3935 	if (*pos != iter->pos) {  in s_start()
3936 		iter->ent = NULL;  in s_start()
3937 		iter->cpu = 0;  in s_start()
3938 		iter->idx = -1;  in s_start()
3946 		iter->leftover = 0;  in s_start()
3955 		if (iter->leftover)  in s_start()
3958 			l = *pos - 1;  in s_start()
3970 	struct trace_iterator *iter = m->private;  in s_stop()
3973 	if (iter->snapshot && iter->trace->use_max_tr)  in s_stop()
3977 	trace_access_unlock(iter->cpu_file);  in s_stop()
3987 	count = ring_buffer_entries_cpu(buf->buffer, cpu);  in get_total_entries_cpu()
3993 	if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {  in get_total_entries_cpu()
3994 		count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;  in get_total_entries_cpu()
3999 			ring_buffer_overrun_cpu(buf->buffer, cpu);  in get_total_entries_cpu()
4027 	get_total_entries_cpu(&tr->array_buffer, &total, &entries, cpu);  in trace_total_entries_cpu()
4039 	get_total_entries(&tr->array_buffer, &total, &entries);  in trace_total_entries()
4046 	seq_puts(m, "#                    _------=> CPU#            \n"  in print_lat_help_header()
4047 		    "#                   / _-----=> irqs-off/BH-disabled\n"  in print_lat_help_header()
4048 		    "#                  | / _----=> need-resched    \n"  in print_lat_help_header()
4049 		    "#                  || / _---=> hardirq/softirq \n"  in print_lat_help_header()
4050 		    "#                  ||| / _--=> preempt-depth   \n"  in print_lat_help_header()
4051 		    "#                  |||| / _-=> migrate-disable \n"  in print_lat_help_header()
4053 		    "#  cmd     pid     |||||| time  |   caller     \n"  in print_lat_help_header()
4063 	seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu   #P:%d\n",  in print_event_info()
4075 …seq_printf(m, "#           TASK-PID    %s CPU#     TIMESTAMP  FUNCTION\n", tgid ? "   TGID   " : "…  in print_func_help_header()
4088 	seq_printf(m, "#                            %.*s  _-----=> irqs-off/BH-disabled\n", prec, space);  in print_func_help_header_irq()
4089 	seq_printf(m, "#                            %.*s / _----=> need-resched\n", prec, space);  in print_func_help_header_irq()
4090 	seq_printf(m, "#                            %.*s| / _---=> hardirq/softirq\n", prec, space);  in print_func_help_header_irq()
4091 	seq_printf(m, "#                            %.*s|| / _--=> preempt-depth\n", prec, space);  in print_func_help_header_irq()
4092 	seq_printf(m, "#                            %.*s||| / _-=> migrate-disable\n", prec, space);  in print_func_help_header_irq()
4094 …seq_printf(m, "#           TASK-PID  %.*s CPU#  |||||  TIMESTAMP  FUNCTION\n", prec, "     TGID   …  in print_func_help_header_irq()
4102 	struct array_buffer *buf = iter->array_buffer;  in print_trace_header()
4103 	struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);  in print_trace_header()
4104 	struct tracer *type = iter->trace;  in print_trace_header()
4107 	const char *name = type->name;  in print_trace_header()  local
4112 		   name, init_utsname()->release);  in print_trace_header()
4113 	seq_puts(m, "# -----------------------------------"  in print_trace_header()
4114 		 "---------------------------------\n");  in print_trace_header()
4117 		   nsecs_to_usecs(data->saved_latency),  in print_trace_header()
4120 		   buf->cpu,  in print_trace_header()
4129 	seq_puts(m, "#    -----------------\n");  in print_trace_header()
4130 	seq_printf(m, "#    | task: %.16s-%d "  in print_trace_header()
4132 		   data->comm, data->pid,  in print_trace_header()
4133 		   from_kuid_munged(seq_user_ns(m), data->uid), data->nice,  in print_trace_header()
4134 		   data->policy, data->rt_priority);  in print_trace_header()
4135 	seq_puts(m, "#    -----------------\n");  in print_trace_header()
4137 	if (data->critical_start) {  in print_trace_header()
4139 		seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);  in print_trace_header()
4140 		trace_print_seq(m, &iter->seq);  in print_trace_header()
4142 		seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);  in print_trace_header()
4143 		trace_print_seq(m, &iter->seq);  in print_trace_header()
4152 	struct trace_seq *s = &iter->seq;  in test_cpu_buff_start()
4153 	struct trace_array *tr = iter->tr;  in test_cpu_buff_start()
4155 	if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))  in test_cpu_buff_start()
4158 	if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))  in test_cpu_buff_start()
4161 	if (cpumask_available(iter->started) &&  in test_cpu_buff_start()
4162 	    cpumask_test_cpu(iter->cpu, iter->started))  in test_cpu_buff_start()
4165 	if (per_cpu_ptr(iter->array_buffer->data, iter->cpu)->skipped_entries)  in test_cpu_buff_start()
4168 	if (cpumask_available(iter->started))  in test_cpu_buff_start()
4169 		cpumask_set_cpu(iter->cpu, iter->started);  in test_cpu_buff_start()
4172 	if (iter->idx > 1)  in test_cpu_buff_start()
4174 				iter->cpu);  in test_cpu_buff_start()
4179 	struct trace_array *tr = iter->tr;  in print_trace_fmt()
4180 	struct trace_seq *s = &iter->seq;  in print_trace_fmt()
4181 	unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);  in print_trace_fmt()
4185 	entry = iter->ent;  in print_trace_fmt()
4189 	event = ftrace_find_event(entry->type);  in print_trace_fmt()
4191 	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {  in print_trace_fmt()
4192 		if (iter->iter_flags & TRACE_FILE_LAT_FMT)  in print_trace_fmt()
4202 		if (tr->trace_flags & TRACE_ITER_FIELDS)  in print_trace_fmt()
4209 		if ((tr->text_delta || tr->data_delta) &&  in print_trace_fmt()
4210 		    event->type > __TRACE_LAST_TYPE)  in print_trace_fmt()
4213 		return event->funcs->trace(iter, sym_flags, event);  in print_trace_fmt()
4216 	trace_seq_printf(s, "Unknown type %d\n", entry->type);  in print_trace_fmt()
4223 	struct trace_array *tr = iter->tr;  in print_raw_fmt()
4224 	struct trace_seq *s = &iter->seq;  in print_raw_fmt()
4228 	entry = iter->ent;  in print_raw_fmt()
4230 	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)  in print_raw_fmt()
4232 				 entry->pid, iter->cpu, iter->ts);  in print_raw_fmt()
4237 	event = ftrace_find_event(entry->type);  in print_raw_fmt()
4239 		return event->funcs->raw(iter, 0, event);  in print_raw_fmt()
4241 	trace_seq_printf(s, "%d ?\n", entry->type);  in print_raw_fmt()
4248 	struct trace_array *tr = iter->tr;  in print_hex_fmt()
4249 	struct trace_seq *s = &iter->seq;  in print_hex_fmt()
4254 	entry = iter->ent;  in print_hex_fmt()
4256 	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {  in print_hex_fmt()
4257 		SEQ_PUT_HEX_FIELD(s, entry->pid);  in print_hex_fmt()
4258 		SEQ_PUT_HEX_FIELD(s, iter->cpu);  in print_hex_fmt()
4259 		SEQ_PUT_HEX_FIELD(s, iter->ts);  in print_hex_fmt()
4264 	event = ftrace_find_event(entry->type);  in print_hex_fmt()
4266 		enum print_line_t ret = event->funcs->hex(iter, 0, event);  in print_hex_fmt()
4278 	struct trace_array *tr = iter->tr;  in print_bin_fmt()
4279 	struct trace_seq *s = &iter->seq;  in print_bin_fmt()
4283 	entry = iter->ent;  in print_bin_fmt()
4285 	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {  in print_bin_fmt()
4286 		SEQ_PUT_FIELD(s, entry->pid);  in print_bin_fmt()
4287 		SEQ_PUT_FIELD(s, iter->cpu);  in print_bin_fmt()
4288 		SEQ_PUT_FIELD(s, iter->ts);  in print_bin_fmt()
4293 	event = ftrace_find_event(entry->type);  in print_bin_fmt()
4294 	return event ? event->funcs->binary(iter, 0, event) :  in print_bin_fmt()
4304 	if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {  in trace_empty()
4305 		cpu = iter->cpu_file;  in trace_empty()
4311 			if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))  in trace_empty()
4323 			if (!ring_buffer_empty_cpu(iter->array_buffer->buffer, cpu))  in trace_empty()
4334 	struct trace_array *tr = iter->tr;  in print_trace_line()
4335 	unsigned long trace_flags = tr->trace_flags;  in print_trace_line()
4338 	if (iter->lost_events) {  in print_trace_line()
4339 		if (iter->lost_events == (unsigned long)-1)  in print_trace_line()
4340 			trace_seq_printf(&iter->seq, "CPU:%d [LOST EVENTS]\n",  in print_trace_line()
4341 					 iter->cpu);  in print_trace_line()
4343 			trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",  in print_trace_line()
4344 					 iter->cpu, iter->lost_events);  in print_trace_line()
4345 		if (trace_seq_has_overflowed(&iter->seq))  in print_trace_line()
4349 	if (iter->trace && iter->trace->print_line) {  in print_trace_line()
4350 		ret = iter->trace->print_line(iter);  in print_trace_line()
4355 	if (iter->ent->type == TRACE_BPUTS &&  in print_trace_line()
4360 	if (iter->ent->type == TRACE_BPRINT &&  in print_trace_line()
4365 	if (iter->ent->type == TRACE_PRINT &&  in print_trace_line()
4384 	struct trace_iterator *iter = m->private;  in trace_latency_header()
4385 	struct trace_array *tr = iter->tr;  in trace_latency_header()
4391 	if (iter->iter_flags & TRACE_FILE_LAT_FMT)  in trace_latency_header()
4394 	if (!(tr->trace_flags & TRACE_ITER_VERBOSE))  in trace_latency_header()
4400 	struct trace_iterator *iter = m->private;  in trace_default_header()
4401 	struct trace_array *tr = iter->tr;  in trace_default_header()
4402 	unsigned long trace_flags = tr->trace_flags;  in trace_default_header()
4407 	if (iter->iter_flags & TRACE_FILE_LAT_FMT) {  in trace_default_header()
4417 				print_func_help_header_irq(iter->array_buffer,  in trace_default_header()
4420 				print_func_help_header(iter->array_buffer, m,  in trace_default_header()
4462 	if (iter->tr->allocated_snapshot)  in print_snapshot_help()
4468 	if (iter->cpu_file == RING_BUFFER_ALL_CPUS)  in print_snapshot_help()
4483 	if (iter->ent == NULL) {  in s_show()
4484 		if (iter->tr) {  in s_show()
4485 			seq_printf(m, "# tracer: %s\n", iter->trace->name);  in s_show()
4489 		if (iter->snapshot && trace_empty(iter))  in s_show()
4491 		else if (iter->trace && iter->trace->print_header)  in s_show()
4492 			iter->trace->print_header(m);  in s_show()
4496 	} else if (iter->leftover) {  in s_show()
4501 		ret = trace_print_seq(m, &iter->seq);  in s_show()
4504 		iter->leftover = ret;  in s_show()
4509 			iter->seq.full = 0;  in s_show()
4510 			trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n");  in s_show()
4512 		ret = trace_print_seq(m, &iter->seq);  in s_show()
4518 		 *        -1 otherwise.  in s_show()
4520 		iter->leftover = ret;  in s_show()
4532 	if (inode->i_cdev) /* See trace_create_cpu_file() */  in tracing_get_cpu()
4533 		return (long)inode->i_cdev - 1;  in tracing_get_cpu()
4554 	if (iter->fmt != static_fmt_buf)  in free_trace_iter_content()
4555 		kfree(iter->fmt);  in free_trace_iter_content()
4557 	kfree(iter->temp);  in free_trace_iter_content()
4558 	kfree(iter->buffer_iter);  in free_trace_iter_content()
4559 	mutex_destroy(&iter->mutex);  in free_trace_iter_content()
4560 	free_cpumask_var(iter->started);  in free_trace_iter_content()
4566 	struct trace_array *tr = inode->i_private;  in __tracing_open()
4571 		return ERR_PTR(-ENODEV);  in __tracing_open()
4575 		return ERR_PTR(-ENOMEM);  in __tracing_open()
4577 	iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),  in __tracing_open()
4579 	if (!iter->buffer_iter)  in __tracing_open()
4583 	 * trace_find_next_entry() may need to save off iter->ent.  in __tracing_open()
4584 	 * It will place it into the iter->temp buffer. As most  in __tracing_open()
4587 	 * allocate a new buffer to adjust for the bigger iter->ent.  in __tracing_open()
4590 	iter->temp = kmalloc(128, GFP_KERNEL);  in __tracing_open()
4591 	if (iter->temp)  in __tracing_open()
4592 		iter->temp_size = 128;  in __tracing_open()
4601 	iter->fmt = NULL;  in __tracing_open()
4602 	iter->fmt_size = 0;  in __tracing_open()
4605 	iter->trace = tr->current_trace;  in __tracing_open()
4607 	if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))  in __tracing_open()
4610 	iter->tr = tr;  in __tracing_open()
4614 	if (tr->current_trace->print_max || snapshot)  in __tracing_open()
4615 		iter->array_buffer = &tr->max_buffer;  in __tracing_open()
4618 		iter->array_buffer = &tr->array_buffer;  in __tracing_open()
4619 	iter->snapshot = snapshot;  in __tracing_open()
4620 	iter->pos = -1;  in __tracing_open()
4621 	iter->cpu_file = tracing_get_cpu(inode);  in __tracing_open()
4622 	mutex_init(&iter->mutex);  in __tracing_open()
4625 	if (iter->trace->open)  in __tracing_open()
4626 		iter->trace->open(iter);  in __tracing_open()
4629 	if (ring_buffer_overruns(iter->array_buffer->buffer))  in __tracing_open()
4630 		iter->iter_flags |= TRACE_FILE_ANNOTATE;  in __tracing_open()
4633 	if (trace_clocks[tr->clock_id].in_ns)  in __tracing_open()
4634 		iter->iter_flags |= TRACE_FILE_TIME_IN_NS;  in __tracing_open()
4637 	 * If pause-on-trace is enabled, then stop the trace while  in __tracing_open()
4640 	if (!iter->snapshot && (tr->trace_flags & TRACE_ITER_PAUSE_ON_TRACE))  in __tracing_open()
4643 	if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {  in __tracing_open()
4645 			iter->buffer_iter[cpu] =  in __tracing_open()
4646 				ring_buffer_read_prepare(iter->array_buffer->buffer,  in __tracing_open()
4651 			ring_buffer_read_start(iter->buffer_iter[cpu]);  in __tracing_open()
4655 		cpu = iter->cpu_file;  in __tracing_open()
4656 		iter->buffer_iter[cpu] =  in __tracing_open()
4657 			ring_buffer_read_prepare(iter->array_buffer->buffer,  in __tracing_open()
4660 		ring_buffer_read_start(iter->buffer_iter[cpu]);  in __tracing_open()
4673 	return ERR_PTR(-ENOMEM);  in __tracing_open()
4684 	filp->private_data = inode->i_private;  in tracing_open_generic()
4699 	struct trace_array *tr = inode->i_private;  in tracing_open_generic_tr()
4706 	filp->private_data = inode->i_private;  in tracing_open_generic_tr()
4717 	struct trace_event_file *file = inode->i_private;  in tracing_open_file_tr()
4720 	ret = tracing_check_open_get_tr(file->tr);  in tracing_open_file_tr()
4727 	if (file->flags & EVENT_FILE_FL_FREED) {  in tracing_open_file_tr()
4728 		trace_array_put(file->tr);  in tracing_open_file_tr()
4729 		ret = -ENODEV;  in tracing_open_file_tr()
4738 	filp->private_data = inode->i_private;  in tracing_open_file_tr()
4745 	struct trace_event_file *file = inode->i_private;  in tracing_release_file_tr()
4747 	trace_array_put(file->tr);  in tracing_release_file_tr()
4767 	struct trace_array *tr = inode->i_private;  in tracing_release()
4768 	struct seq_file *m = file->private_data;  in tracing_release()
4772 	if (!(file->f_mode & FMODE_READ)) {  in tracing_release()
4778 	iter = m->private;  in tracing_release()
4782 		if (iter->buffer_iter[cpu])  in tracing_release()
4783 			ring_buffer_read_finish(iter->buffer_iter[cpu]);  in tracing_release()
4786 	if (iter->trace && iter->trace->close)  in tracing_release()
4787 		iter->trace->close(iter);  in tracing_release()
4789 	if (!iter->snapshot && tr->stop_count)  in tracing_release()
4805 	struct trace_array *tr = inode->i_private;  in tracing_release_generic_tr()
4813 	struct trace_array *tr = inode->i_private;  in tracing_single_release_tr()
4822 	struct trace_array *tr = inode->i_private;  in tracing_open()
4831 	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {  in tracing_open()
4833 		struct array_buffer *trace_buf = &tr->array_buffer;  in tracing_open()
4836 		if (tr->current_trace->print_max)  in tracing_open()
4837 			trace_buf = &tr->max_buffer;  in tracing_open()
4846 	if (file->f_mode & FMODE_READ) {  in tracing_open()
4850 		else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)  in tracing_open()
4851 			iter->iter_flags |= TRACE_FILE_LAT_FMT;  in tracing_open()
4870 	if (tr->range_addr_start && t->use_max_tr)  in trace_ok_for_array()
4873 	return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;  in trace_ok_for_array()
4881 		t = t->next;  in get_tracer_for_array()
4889 	struct trace_array *tr = m->private;  in t_next()
4895 		t = get_tracer_for_array(tr, t->next);  in t_next()
4902 	struct trace_array *tr = m->private;  in t_start()
4927 	seq_puts(m, t->name);  in t_show()
4928 	if (t->next)  in t_show()
4945 	struct trace_array *tr = inode->i_private;  in show_traces_open()
4959 	m = file->private_data;  in show_traces_open()
4960 	m->private = tr;  in show_traces_open()
4967 	struct trace_array *tr = inode->i_private;  in tracing_seq_release()
4984 	if (file->f_mode & FMODE_READ)  in tracing_lseek()
4987 		file->f_pos = ret = 0;  in tracing_lseek()
5013 	struct trace_array *tr = file_inode(filp)->i_private;  in tracing_cpumask_read()
5018 		       cpumask_pr_args(tr->tracing_cpumask)) + 1;  in tracing_cpumask_read()
5021 		return -ENOMEM;  in tracing_cpumask_read()
5024 		       cpumask_pr_args(tr->tracing_cpumask));  in tracing_cpumask_read()
5026 		count = -EINVAL;  in tracing_cpumask_read()
5043 		return -EINVAL;  in tracing_set_cpumask()
5046 	arch_spin_lock(&tr->max_lock);  in tracing_set_cpumask()
5052 		if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&  in tracing_set_cpumask()
5054 			atomic_inc(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);  in tracing_set_cpumask()
5055 			ring_buffer_record_disable_cpu(tr->array_buffer.buffer, cpu);  in tracing_set_cpumask()
5057 			ring_buffer_record_disable_cpu(tr->max_buffer.buffer, cpu);  in tracing_set_cpumask()
5060 		if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&  in tracing_set_cpumask()
5062 			atomic_dec(&per_cpu_ptr(tr->array_buffer.data, cpu)->disabled);  in tracing_set_cpumask()
5063 			ring_buffer_record_enable_cpu(tr->array_buffer.buffer, cpu);  in tracing_set_cpumask()
5065 			ring_buffer_record_enable_cpu(tr->max_buffer.buffer, cpu);  in tracing_set_cpumask()
5069 	arch_spin_unlock(&tr->max_lock);  in tracing_set_cpumask()
5072 	cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);  in tracing_set_cpumask()
5081 	struct trace_array *tr = file_inode(filp)->i_private;  in tracing_cpumask_write()
5086 		return -EINVAL;  in tracing_cpumask_write()
5089 		return -ENOMEM;  in tracing_cpumask_write()
5120 	struct trace_array *tr = m->private;  in tracing_trace_options_show()
5126 	tracer_flags = tr->current_trace->flags->val;  in tracing_trace_options_show()
5127 	trace_opts = tr->current_trace->flags->opts;  in tracing_trace_options_show()
5130 		if (tr->trace_flags & (1 << i))  in tracing_trace_options_show()
5136 	for (i = 0; trace_opts[i].name; i++) {  in tracing_trace_options_show()
5138 			seq_printf(m, "%s\n", trace_opts[i].name);  in tracing_trace_options_show()
5140 			seq_printf(m, "no%s\n", trace_opts[i].name);  in tracing_trace_options_show()
5150 	struct tracer *trace = tracer_flags->trace;  in __set_tracer_option()
5153 	ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);  in __set_tracer_option()
5158 		tracer_flags->val &= ~opts->bit;  in __set_tracer_option()
5160 		tracer_flags->val |= opts->bit;  in __set_tracer_option()
5167 	struct tracer *trace = tr->current_trace;  in set_tracer_option()
5168 	struct tracer_flags *tracer_flags = trace->flags;  in set_tracer_option()
5172 	for (i = 0; tracer_flags->opts[i].name; i++) {  in set_tracer_option()
5173 		opts = &tracer_flags->opts[i];  in set_tracer_option()
5175 		if (strcmp(cmp, opts->name) == 0)  in set_tracer_option()
5176 			return __set_tracer_option(tr, trace->flags, opts, neg);  in set_tracer_option()
5179 	return -EINVAL;  in set_tracer_option()
5185 	if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)  in trace_keep_overwrite()
5186 		return -1;  in trace_keep_overwrite()
5199 	if (!!(tr->trace_flags & mask) == !!enabled)  in set_tracer_flag()
5203 	if (tr->current_trace->flag_changed)  in set_tracer_flag()
5204 		if (tr->current_trace->flag_changed(tr, mask, !!enabled))  in set_tracer_flag()
5205 			return -EINVAL;  in set_tracer_flag()
5216 				return -EINVAL;  in set_tracer_flag()
5227 		tr->trace_flags |= mask;  in set_tracer_flag()
5229 		tr->trace_flags &= ~mask;  in set_tracer_flag()
5237 			tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;  in set_tracer_flag()
5238 			return -ENOMEM;  in set_tracer_flag()
5251 		ring_buffer_change_overwrite(tr->array_buffer.buffer, enabled);  in set_tracer_flag()
5253 		ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);  in set_tracer_flag()
5284 	ret = match_string(trace_options, -1, cmp);  in trace_set_options()
5320 			*(buf - 1) = ',';  in apply_trace_boot_options()
5326 			size_t cnt, loff_t *ppos)  in tracing_trace_options_write()  argument
5328 	struct seq_file *m = filp->private_data;  in tracing_trace_options_write()
5329 	struct trace_array *tr = m->private;  in tracing_trace_options_write()
5333 	if (cnt >= sizeof(buf))  in tracing_trace_options_write()
5334 		return -EINVAL;  in tracing_trace_options_write()
5336 	if (copy_from_user(buf, ubuf, cnt))  in tracing_trace_options_write()
5337 		return -EFAULT;  in tracing_trace_options_write()
5339 	buf[cnt] = 0;  in tracing_trace_options_write()
5345 	*ppos += cnt;  in tracing_trace_options_write()
5347 	return cnt;  in tracing_trace_options_write()
5352 	struct trace_array *tr = inode->i_private;  in tracing_trace_options_open()
5359 	ret = single_open(file, tracing_trace_options_show, inode->i_private);  in tracing_trace_options_open()
5375 	"tracing mini-HOWTO:\n\n"
5379 	"\t e.g. mount -t tracefs [-o [gid=<gid>]] nodev /sys/kernel/tracing\n\n"
5381 	"# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
5383 	"  trace\t\t\t- The static contents of the buffer\n"
5385 	"  trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
5386 	"  current_tracer\t- function and latency tracers\n"
5387 	"  available_tracers\t- list of configured tracers for current_tracer\n"
5388 	"  error_log\t- error log for failed commands (that support it)\n"
5389 	"  buffer_size_kb\t- view and modify size of per cpu buffer\n"
5390 	"  buffer_total_size_kb  - view total size of all cpu buffers\n\n"
5391 	"  trace_clock\t\t- change the clock used to order events\n"
5398 	"     x86-tsc:   TSC cycle counter\n"
5400 	"\n  timestamp_mode\t- view the mode used to timestamp events\n"
5401 	"       delta:   Delta difference against a buffer-wide timestamp\n"
5403 	"\n  trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
5404 	"\n  trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
5405 	"  tracing_cpumask\t- Limit which CPUs to trace\n"
5406 	"  instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
5407 	"\t\t\t  Remove sub-buffer with rmdir\n"
5408 	"  trace_options\t\t- Set format or modify how tracing happens\n"
5410 	"\t\t\t  option name\n"
5411 	"  saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
5413 	"\n  available_filter_functions - list of functions that can be filtered on\n"
5414 	"  set_ftrace_filter\t- echo function name in here to only trace these\n"
5416 	"\t     accepts: func_full_name or glob-matching-pattern\n"
5418 	"\t      Format: :mod:<module-name>\n"
5445 	"  set_ftrace_notrace\t- echo function name in here to never trace.\n"
5451 	"  set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
5453 	"  set_ftrace_notrace_pid\t- Write pid(s) to not function trace those pids\n"
5457 	"  set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
5458 	"  set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
5459 	"  max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
5462 	"\n  snapshot\t\t- Like 'trace' but shows the content of the static\n"
5467 	"  stack_trace\t\t- Shows the max stack trace when active\n"
5468 	"  stack_max_size\t- Shows current max stack size that was traced\n"
5472 	"  stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
5477 	"  dynamic_events\t\t- Create/append/remove/show the generic dynamic events\n"
5481 	"  kprobe_events\t\t- Create/append/remove/show the kernel dynamic events\n"
5485 	"  uprobe_events\t\t- Create/append/remove/show the userspace dynamic events\n"
5490 	"\t  accepts: event-definitions (one definition per line)\n"
5496 	"\t           f[:[<group>/][<event>]] <func-name>[%return] [<args>]\n"
5502 	"\t           e[:[<group>/][<event>]] <attached-group>.<attached-event> [<args>] [if <filter>]\n"
5503 	"\t           -:[<group>/][<event>]\n"
5511 	"\t     args: <name>=fetcharg[:type]\n"
5512 	"\t fetcharg: (%<register>|$<efield>), @<address>, @<symbol>[+|-<offset>],\n"
5516 	"\t           <argname>[->field[->field|.field...]],\n"
5521 	"\t           +|-[u]<offset>(<fetcharg>), \\imm-value, \\\"imm-string\"\n"
5524 	"\t           b<bit-width>@<bit-offset>/<container-size>, ustring,\n"
5525 	"\t           symstr, %pd/%pD, <type>\\[<array-size>\\]\n"
5527 	"\t    field: <stype> <name>;\n"
5532 	"\t            of the <attached-group>/<attached-event>.\n"
5534 	"  set_event\t\t- Enables events by name written into it\n"
5536 	"  events/\t\t- Directory containing all trace event subsystems:\n"
5537 	"      enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
5538 	"  events/<system>/\t- Directory containing all trace events for <system>:\n"
5539 	"      enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
5541 	"      filter\t\t- If set, only events passing filter are traced\n"
5542 	"  events/<system>/<event>/\t- Directory containing control files for\n"
5544 	"      enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
5545 	"      filter\t\t- If set, only events passing filter are traced\n"
5546 	"      trigger\t\t- If set, a command to perform when event is hit\n"
5580 	"      hist trigger\t- If set, event hits are aggregated into a hash table\n"
5587 	"\t            [:name=histname1]\n"
5592 	"\t            common_timestamp - to record current timestamp\n"
5593 	"\t            common_cpu - to record the CPU the event happened on\n"
5596 	"\t        - a reference to a field e.g. x=current_timestamp,\n"
5597 	"\t        - a reference to another variable e.g. y=$x,\n"
5598 	"\t        - a numeric literal: e.g. ms_per_sec=1000,\n"
5599 	"\t        - an arithmetic expression: e.g. time_secs=current_timestamp/1000\n"
5601 	"\t    hist trigger arithmetic expressions support addition(+), subtraction(-),\n"
5617 	"\t    If a hist trigger is given a name using the 'name' parameter,\n"
5619 	"\t    same name, and trigger hits will update this common data.\n\n"
5625 	"\t    same name.  The default format used to display a given field\n"
5627 	"\t    to the field name, as applicable:\n\n"
5630 	"\t            .sym-offset display an address as a symbol and offset\n"
5631 	"\t            .execname   display a common_pid as a program name\n"
5632 	"\t            .syscall    display a syscall id as a syscall name\n"
5637 	"\t            .graph      display a bar-graph of a value\n\n"
5649 	"\t    already-attached hist trigger.  The syntax is analogous to\n"
5655 	"\t        onmatch(matching.event)  - invoke on addition or update\n"
5656 	"\t        onmax(var)               - invoke if var exceeds current max\n"
5657 	"\t        onchange(var)            - invoke action if var changes\n\n"
5659 	"\t        trace(<synthetic_event>,param list)  - generate synthetic event\n"
5660 	"\t        save(field,...)                      - save current event fields\n"
5662 	"\t        snapshot()                           - snapshot the trace buffer\n\n"
5665 	"  events/synthetic_events\t- Create/append/remove/show synthetic events\n"
5667 	"\t     example: echo 'myevent u64 lat; char name[]; long[] stack' >> synthetic_events\n"
5674 		       size_t cnt, loff_t *ppos)  in tracing_readme_read()  argument
5676 	return simple_read_from_buffer(ubuf, cnt, ppos,  in tracing_readme_read()
5690 	if (!ptr->map.eval_string) {  in update_eval_map()
5691 		if (ptr->tail.next) {  in update_eval_map()
5692 			ptr = ptr->tail.next;  in update_eval_map()
5748 		   ptr->map.eval_string, ptr->map.eval_value,  in eval_map_show()
5749 		   ptr->map.system);  in eval_map_show()
5783 	return ptr + ptr->head.length + 1;  in trace_eval_jmp_to_tail()
5816 			if (!ptr->tail.next)  in trace_insert_eval_map_file()
5818 			ptr = ptr->tail.next;  in trace_insert_eval_map_file()
5821 		ptr->tail.next = map_array;  in trace_insert_eval_map_file()
5823 	map_array->head.mod = mod;  in trace_insert_eval_map_file()
5824 	map_array->head.length = len;  in trace_insert_eval_map_file()
5828 		map_array->map = **map;  in trace_insert_eval_map_file()
5863 		       size_t cnt, loff_t *ppos)  in tracing_set_trace_read()  argument
5865 	struct trace_array *tr = filp->private_data;  in tracing_set_trace_read()
5870 	r = sprintf(buf, "%s\n", tr->current_trace->name);  in tracing_set_trace_read()
5873 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);  in tracing_set_trace_read()
5878 	tracing_reset_online_cpus(&tr->array_buffer);  in tracer_init()
5879 	return t->init(tr);  in tracer_init()
5887 		per_cpu_ptr(buf->data, cpu)->entries = val;  in set_buffer_entries()
5893 		set_buffer_entries(buf, ring_buffer_size(buf->buffer, 0));  in update_buffer_entries()
5895 		per_cpu_ptr(buf->data, cpu)->entries = ring_buffer_size(buf->buffer, cpu);  in update_buffer_entries()
5908 			ret = ring_buffer_resize(trace_buf->buffer,  in resize_buffer_duplicate_size()
5909 				 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);  in resize_buffer_duplicate_size()
5912 			per_cpu_ptr(trace_buf->data, cpu)->entries =  in resize_buffer_duplicate_size()
5913 				per_cpu_ptr(size_buf->data, cpu)->entries;  in resize_buffer_duplicate_size()
5916 		ret = ring_buffer_resize(trace_buf->buffer,  in resize_buffer_duplicate_size()
5917 				 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);  in resize_buffer_duplicate_size()
5919 			per_cpu_ptr(trace_buf->data, cpu_id)->entries =  in resize_buffer_duplicate_size()
5920 				per_cpu_ptr(size_buf->data, cpu_id)->entries;  in resize_buffer_duplicate_size()
5940 	if (!tr->array_buffer.buffer)  in __tracing_resize_ring_buffer()
5946 	ret = ring_buffer_resize(tr->array_buffer.buffer, size, cpu);  in __tracing_resize_ring_buffer()
5951 	if (!tr->allocated_snapshot)  in __tracing_resize_ring_buffer()
5954 	ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);  in __tracing_resize_ring_buffer()
5956 		int r = resize_buffer_duplicate_size(&tr->array_buffer,  in __tracing_resize_ring_buffer()
5957 						     &tr->array_buffer, cpu);  in __tracing_resize_ring_buffer()
5979 	update_buffer_entries(&tr->max_buffer, cpu);  in __tracing_resize_ring_buffer()
5984 	update_buffer_entries(&tr->array_buffer, cpu);  in __tracing_resize_ring_buffer()
5998 			return -EINVAL;  in tracing_resize_ring_buffer()
6006 	if (!tr->text_delta && !tr->data_delta)  in update_last_data()
6015 	tracing_reset_all_cpus(&tr->array_buffer);  in update_last_data()
6018 	tr->text_delta = 0;  in update_last_data()
6019 	tr->data_delta = 0;  in update_last_data()
6023  * tracing_update_buffers - used by tracing facility to expand ring buffers
6041 	if (!tr->ring_buffer_expanded)  in tracing_update_buffers()
6060 	if (tr->current_trace == &nop_trace)  in tracing_set_nop()
6063 	tr->current_trace->enabled--;  in tracing_set_nop()
6065 	if (tr->current_trace->reset)  in tracing_set_nop()
6066 		tr->current_trace->reset(tr);  in tracing_set_nop()
6068 	tr->current_trace = &nop_trace;  in tracing_set_nop()
6076 	if (!tr->dir)  in add_tracer_options()
6098 	if (!tr->ring_buffer_expanded) {  in tracing_set_tracer()
6106 	for (t = trace_types; t; t = t->next) {  in tracing_set_tracer()
6107 		if (strcmp(t->name, buf) == 0)  in tracing_set_tracer()
6111 		return -EINVAL;  in tracing_set_tracer()
6113 	if (t == tr->current_trace)  in tracing_set_tracer()
6117 	if (t->use_max_tr) {  in tracing_set_tracer()
6119 		arch_spin_lock(&tr->max_lock);  in tracing_set_tracer()
6120 		ret = tr->cond_snapshot ? -EBUSY : 0;  in tracing_set_tracer()
6121 		arch_spin_unlock(&tr->max_lock);  in tracing_set_tracer()
6128 	if (system_state < SYSTEM_RUNNING && t->noboot) {  in tracing_set_tracer()
6130 			t->name);  in tracing_set_tracer()
6131 		return -EINVAL;  in tracing_set_tracer()
6136 		return -EINVAL;  in tracing_set_tracer()
6139 	if (tr->trace_ref)  in tracing_set_tracer()
6140 		return -EBUSY;  in tracing_set_tracer()
6144 	tr->current_trace->enabled--;  in tracing_set_tracer()
6146 	if (tr->current_trace->reset)  in tracing_set_tracer()
6147 		tr->current_trace->reset(tr);  in tracing_set_tracer()
6150 	had_max_tr = tr->current_trace->use_max_tr;  in tracing_set_tracer()
6153 	tr->current_trace = &nop_trace;  in tracing_set_tracer()
6155 	if (had_max_tr && !t->use_max_tr) {  in tracing_set_tracer()
6168 	if (!had_max_tr && t->use_max_tr) {  in tracing_set_tracer()
6174 	tr->current_trace = &nop_trace;  in tracing_set_tracer()
6177 	if (t->init) {  in tracing_set_tracer()
6181 			if (t->use_max_tr)  in tracing_set_tracer()
6188 	tr->current_trace = t;  in tracing_set_tracer()
6189 	tr->current_trace->enabled++;  in tracing_set_tracer()
6197 			size_t cnt, loff_t *ppos)  in tracing_set_trace_write()  argument
6199 	struct trace_array *tr = filp->private_data;  in tracing_set_trace_write()
6201 	char *name;  in tracing_set_trace_write()  local
6205 	ret = cnt;  in tracing_set_trace_write()
6207 	if (cnt > MAX_TRACER_SIZE)  in tracing_set_trace_write()
6208 		cnt = MAX_TRACER_SIZE;  in tracing_set_trace_write()
6210 	if (copy_from_user(buf, ubuf, cnt))  in tracing_set_trace_write()
6211 		return -EFAULT;  in tracing_set_trace_write()
6213 	buf[cnt] = 0;  in tracing_set_trace_write()
6215 	name = strim(buf);  in tracing_set_trace_write()
6217 	err = tracing_set_tracer(tr, name);  in tracing_set_trace_write()
6228 		   size_t cnt, loff_t *ppos)  in tracing_nsecs_read()  argument
6234 		     *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));  in tracing_nsecs_read()
6237 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);  in tracing_nsecs_read()
6242 		    size_t cnt, loff_t *ppos)  in tracing_nsecs_write()  argument
6247 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);  in tracing_nsecs_write()
6253 	return cnt;  in tracing_nsecs_write()
6258 		    size_t cnt, loff_t *ppos)  in tracing_thresh_read()  argument
6260 	return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);  in tracing_thresh_read()
6265 		     size_t cnt, loff_t *ppos)  in tracing_thresh_write()  argument
6267 	struct trace_array *tr = filp->private_data;  in tracing_thresh_write()
6271 	ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);  in tracing_thresh_write()
6275 	if (tr->current_trace->update_thresh) {  in tracing_thresh_write()
6276 		ret = tr->current_trace->update_thresh(tr);  in tracing_thresh_write()
6281 	return cnt;  in tracing_thresh_write()
6288 		     size_t cnt, loff_t *ppos)  in tracing_max_lat_read()  argument
6290 	struct trace_array *tr = filp->private_data;  in tracing_max_lat_read()
6292 	return tracing_nsecs_read(&tr->max_latency, ubuf, cnt, ppos);  in tracing_max_lat_read()
6297 		      size_t cnt, loff_t *ppos)  in tracing_max_lat_write()  argument
6299 	struct trace_array *tr = filp->private_data;  in tracing_max_lat_write()
6301 	return tracing_nsecs_write(&tr->max_latency, ubuf, cnt, ppos);  in tracing_max_lat_write()
6309 		if (cpumask_empty(tr->pipe_cpumask)) {  in open_pipe_on_cpu()
6310 			cpumask_setall(tr->pipe_cpumask);  in open_pipe_on_cpu()
6313 	} else if (!cpumask_test_cpu(cpu, tr->pipe_cpumask)) {  in open_pipe_on_cpu()
6314 		cpumask_set_cpu(cpu, tr->pipe_cpumask);  in open_pipe_on_cpu()
6317 	return -EBUSY;  in open_pipe_on_cpu()
6323 		WARN_ON(!cpumask_full(tr->pipe_cpumask));  in close_pipe_on_cpu()
6324 		cpumask_clear(tr->pipe_cpumask);  in close_pipe_on_cpu()
6326 		WARN_ON(!cpumask_test_cpu(cpu, tr->pipe_cpumask));  in close_pipe_on_cpu()
6327 		cpumask_clear_cpu(cpu, tr->pipe_cpumask);  in close_pipe_on_cpu()
6333 	struct trace_array *tr = inode->i_private;  in tracing_open_pipe()
6351 		ret = -ENOMEM;  in tracing_open_pipe()
6355 	trace_seq_init(&iter->seq);  in tracing_open_pipe()
6356 	iter->trace = tr->current_trace;  in tracing_open_pipe()
6358 	if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {  in tracing_open_pipe()
6359 		ret = -ENOMEM;  in tracing_open_pipe()
6364 	cpumask_setall(iter->started);  in tracing_open_pipe()
6366 	if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)  in tracing_open_pipe()
6367 		iter->iter_flags |= TRACE_FILE_LAT_FMT;  in tracing_open_pipe()
6370 	if (trace_clocks[tr->clock_id].in_ns)  in tracing_open_pipe()
6371 		iter->iter_flags |= TRACE_FILE_TIME_IN_NS;  in tracing_open_pipe()
6373 	iter->tr = tr;  in tracing_open_pipe()
6374 	iter->array_buffer = &tr->array_buffer;  in tracing_open_pipe()
6375 	iter->cpu_file = cpu;  in tracing_open_pipe()
6376 	mutex_init(&iter->mutex);  in tracing_open_pipe()
6377 	filp->private_data = iter;  in tracing_open_pipe()
6379 	if (iter->trace->pipe_open)  in tracing_open_pipe()
6380 		iter->trace->pipe_open(iter);  in tracing_open_pipe()
6384 	tr->trace_ref++;  in tracing_open_pipe()
6401 	struct trace_iterator *iter = file->private_data;  in tracing_release_pipe()
6402 	struct trace_array *tr = inode->i_private;  in tracing_release_pipe()
6406 	tr->trace_ref--;  in tracing_release_pipe()
6408 	if (iter->trace->pipe_close)  in tracing_release_pipe()
6409 		iter->trace->pipe_close(iter);  in tracing_release_pipe()
6410 	close_pipe_on_cpu(tr, iter->cpu_file);  in tracing_release_pipe()
6424 	struct trace_array *tr = iter->tr;  in trace_poll()
6427 	if (trace_buffer_iter(iter, iter->cpu_file))  in trace_poll()
6430 	if (tr->trace_flags & TRACE_ITER_BLOCK)  in trace_poll()
6436 		return ring_buffer_poll_wait(iter->array_buffer->buffer, iter->cpu_file,  in trace_poll()
6437 					     filp, poll_table, iter->tr->buffer_percent);  in trace_poll()
6443 	struct trace_iterator *iter = filp->private_data;  in tracing_poll_pipe()
6448 /* Must be called with iter->mutex held. */
6451 	struct trace_iterator *iter = filp->private_data;  in tracing_wait_pipe()
6456 		if ((filp->f_flags & O_NONBLOCK)) {  in tracing_wait_pipe()
6457 			return -EAGAIN;  in tracing_wait_pipe()
6467 		 * iter->pos will be 0 if we haven't read anything.  in tracing_wait_pipe()
6469 		if (!tracer_tracing_is_on(iter->tr) && iter->pos)  in tracing_wait_pipe()
6472 		mutex_unlock(&iter->mutex);  in tracing_wait_pipe()
6476 		mutex_lock(&iter->mutex);  in tracing_wait_pipe()
6490 		  size_t cnt, loff_t *ppos)  in tracing_read_pipe()  argument
6492 	struct trace_iterator *iter = filp->private_data;  in tracing_read_pipe()
6500 	guard(mutex)(&iter->mutex);  in tracing_read_pipe()
6503 	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);  in tracing_read_pipe()
6504 	if (sret != -EBUSY)  in tracing_read_pipe()
6507 	trace_seq_init(&iter->seq);  in tracing_read_pipe()
6509 	if (iter->trace->read) {  in tracing_read_pipe()
6510 		sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);  in tracing_read_pipe()
6524 	if (cnt >= TRACE_SEQ_BUFFER_SIZE)  in tracing_read_pipe()
6525 		cnt = TRACE_SEQ_BUFFER_SIZE - 1;  in tracing_read_pipe()
6529 	cpumask_clear(iter->started);  in tracing_read_pipe()
6530 	trace_seq_init(&iter->seq);  in tracing_read_pipe()
6533 	trace_access_lock(iter->cpu_file);  in tracing_read_pipe()
6536 		int save_len = iter->seq.seq.len;  in tracing_read_pipe()
6542 			 * trace_seq_to_user() will returns -EBUSY because save_len == 0,  in tracing_read_pipe()
6547 				iter->seq.full = 0;  in tracing_read_pipe()
6548 				trace_seq_puts(&iter->seq, "[LINE TOO BIG]\n");  in tracing_read_pipe()
6554 			iter->seq.seq.len = save_len;  in tracing_read_pipe()
6560 		if (trace_seq_used(&iter->seq) >= cnt)  in tracing_read_pipe()
6568 		WARN_ONCE(iter->seq.full, "full flag set for trace type %d",  in tracing_read_pipe()
6569 			  iter->ent->type);  in tracing_read_pipe()
6571 	trace_access_unlock(iter->cpu_file);  in tracing_read_pipe()
6575 	sret = trace_seq_to_user(&iter->seq, ubuf, cnt);  in tracing_read_pipe()
6576 	if (iter->seq.readpos >= trace_seq_used(&iter->seq))  in tracing_read_pipe()
6577 		trace_seq_init(&iter->seq);  in tracing_read_pipe()
6583 	if (sret == -EBUSY)  in tracing_read_pipe()
6592 	__free_page(spd->pages[idx]);  in tracing_spd_release_pipe()
6602 	/* Seq buffer is page-sized, exactly what we need. */  in tracing_fill_pipe_page()
6604 		save_len = iter->seq.seq.len;  in tracing_fill_pipe_page()
6607 		if (trace_seq_has_overflowed(&iter->seq)) {  in tracing_fill_pipe_page()
6608 			iter->seq.seq.len = save_len;  in tracing_fill_pipe_page()
6614 		 * be set if the iter->seq overflowed. But check it  in tracing_fill_pipe_page()
6618 			iter->seq.seq.len = save_len;  in tracing_fill_pipe_page()
6622 		count = trace_seq_used(&iter->seq) - save_len;  in tracing_fill_pipe_page()
6625 			iter->seq.seq.len = save_len;  in tracing_fill_pipe_page()
6631 		rem -= count;  in tracing_fill_pipe_page()
6634 			iter->ent = NULL;  in tracing_fill_pipe_page()
6650 	struct trace_iterator *iter = filp->private_data;  in tracing_splice_read_pipe()
6664 		return -ENOMEM;  in tracing_splice_read_pipe()
6666 	mutex_lock(&iter->mutex);  in tracing_splice_read_pipe()
6668 	if (iter->trace->splice_read) {  in tracing_splice_read_pipe()
6669 		ret = iter->trace->splice_read(iter, filp,  in tracing_splice_read_pipe()
6679 	if (!iter->ent && !trace_find_next_entry_inc(iter)) {  in tracing_splice_read_pipe()
6680 		ret = -EFAULT;  in tracing_splice_read_pipe()
6685 	trace_access_lock(iter->cpu_file);  in tracing_splice_read_pipe()
6696 		ret = trace_seq_to_buffer(&iter->seq,  in tracing_splice_read_pipe()
6698 					  trace_seq_used(&iter->seq));  in tracing_splice_read_pipe()
6704 		spd.partial[i].len = trace_seq_used(&iter->seq);  in tracing_splice_read_pipe()
6706 		trace_seq_init(&iter->seq);  in tracing_splice_read_pipe()
6709 	trace_access_unlock(iter->cpu_file);  in tracing_splice_read_pipe()
6711 	mutex_unlock(&iter->mutex);  in tracing_splice_read_pipe()
6724 	mutex_unlock(&iter->mutex);  in tracing_splice_read_pipe()
6730 		     size_t cnt, loff_t *ppos)  in tracing_entries_read()  argument
6733 	struct trace_array *tr = inode->i_private;  in tracing_entries_read()
6751 				size = per_cpu_ptr(tr->array_buffer.data, cpu)->entries;  in tracing_entries_read()
6752 			if (size != per_cpu_ptr(tr->array_buffer.data, cpu)->entries) {  in tracing_entries_read()
6759 			if (!tr->ring_buffer_expanded)  in tracing_entries_read()
6768 		r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10);  in tracing_entries_read()
6772 	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);  in tracing_entries_read()
6778 		      size_t cnt, loff_t *ppos)  in tracing_entries_write()  argument
6781 	struct trace_array *tr = inode->i_private;  in tracing_entries_write()
6785 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);  in tracing_entries_write()
6791 		return -EINVAL;  in tracing_entries_write()
6799 	*ppos += cnt;  in tracing_entries_write()
6801 	return cnt;  in tracing_entries_write()
6806 				size_t cnt, loff_t *ppos)  in tracing_total_entries_read()  argument
6808 	struct trace_array *tr = filp->private_data;  in tracing_total_entries_read()
6815 		size += per_cpu_ptr(tr->array_buffer.data, cpu)->entries >> 10;  in tracing_total_entries_read()
6816 		if (!tr->ring_buffer_expanded)  in tracing_total_entries_read()
6819 	if (tr->ring_buffer_expanded)  in tracing_total_entries_read()
6825 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);  in tracing_total_entries_read()
6829 tracing_last_boot_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)  in tracing_last_boot_read()  argument
6831 	struct trace_array *tr = filp->private_data;  in tracing_last_boot_read()
6837 	seq_buf_printf(&seq, "text delta:\t%ld\n", tr->text_delta);  in tracing_last_boot_read()
6838 	seq_buf_printf(&seq, "data delta:\t%ld\n", tr->data_delta);  in tracing_last_boot_read()
6840 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, seq_buf_used(&seq));  in tracing_last_boot_read()
6845 	struct trace_array *tr = inode->i_private;  in tracing_buffer_meta_open()
6853 	ret = ring_buffer_meta_seq_init(filp, tr->array_buffer.buffer, cpu);  in tracing_buffer_meta_open()
6861 			  size_t cnt, loff_t *ppos)  in tracing_free_buffer_write()  argument
6868 	*ppos += cnt;  in tracing_free_buffer_write()
6870 	return cnt;  in tracing_free_buffer_write()
6876 	struct trace_array *tr = inode->i_private;  in tracing_free_buffer_release()
6879 	if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)  in tracing_free_buffer_release()
6893 					size_t cnt, loff_t *fpos)  in tracing_mark_write()  argument
6895 	struct trace_array *tr = filp->private_data;  in tracing_mark_write()
6907 #define FAULTED_SIZE (sizeof(FAULTED_STR) - 1) /* '\0' is already accounted for */  in tracing_mark_write()
6910 		return -EINVAL;  in tracing_mark_write()
6912 	if (!(tr->trace_flags & TRACE_ITER_MARKERS))  in tracing_mark_write()
6913 		return -EINVAL;  in tracing_mark_write()
6915 	if ((ssize_t)cnt < 0)  in tracing_mark_write()
6916 		return -EINVAL;  in tracing_mark_write()
6918 	if (cnt > TRACE_MARKER_MAX_SIZE)  in tracing_mark_write()
6919 		cnt = TRACE_MARKER_MAX_SIZE;  in tracing_mark_write()
6923 	size = cnt + meta_size;  in tracing_mark_write()
6926 	if (cnt < FAULTED_SIZE)  in tracing_mark_write()
6927 		size += FAULTED_SIZE - cnt;  in tracing_mark_write()
6929 	buffer = tr->array_buffer.buffer;  in tracing_mark_write()
6938 			/* cnt < FAULTED size should never be bigger than max */  in tracing_mark_write()
6939 			if (WARN_ON_ONCE(cnt < FAULTED_SIZE))  in tracing_mark_write()
6940 				return -EBADF;  in tracing_mark_write()
6941 			cnt = ring_buffer_max_event_size(buffer) - meta_size;  in tracing_mark_write()
6943 			if (WARN_ON_ONCE(cnt + meta_size == size))  in tracing_mark_write()
6944 				return -EBADF;  in tracing_mark_write()
6949 		return -EBADF;  in tracing_mark_write()
6953 	entry->ip = _THIS_IP_;  in tracing_mark_write()
6955 	len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);  in tracing_mark_write()
6957 		memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);  in tracing_mark_write()
6958 		cnt = FAULTED_SIZE;  in tracing_mark_write()
6959 		written = -EFAULT;  in tracing_mark_write()
6961 		written = cnt;  in tracing_mark_write()
6963 	if (tr->trace_marker_file && !list_empty(&tr->trace_marker_file->triggers)) {  in tracing_mark_write()
6965 		entry->buf[cnt] = '\0';  in tracing_mark_write()
6966 		tt = event_triggers_call(tr->trace_marker_file, buffer, entry, event);  in tracing_mark_write()
6969 	if (entry->buf[cnt - 1] != '\n') {  in tracing_mark_write()
6970 		entry->buf[cnt] = '\n';  in tracing_mark_write()
6971 		entry->buf[cnt + 1] = '\0';  in tracing_mark_write()
6973 		entry->buf[cnt] = '\0';  in tracing_mark_write()
6980 		event_triggers_post_call(tr->trace_marker_file, tt);  in tracing_mark_write()
6987 					size_t cnt, loff_t *fpos)  in tracing_mark_raw_write()  argument
6989 	struct trace_array *tr = filp->private_data;  in tracing_mark_raw_write()
7000 		return -EINVAL;  in tracing_mark_raw_write()
7002 	if (!(tr->trace_flags & TRACE_ITER_MARKERS))  in tracing_mark_raw_write()
7003 		return -EINVAL;  in tracing_mark_raw_write()
7006 	if (cnt < sizeof(unsigned int))  in tracing_mark_raw_write()
7007 		return -EINVAL;  in tracing_mark_raw_write()
7009 	size = sizeof(*entry) + cnt;  in tracing_mark_raw_write()
7010 	if (cnt < FAULT_SIZE_ID)  in tracing_mark_raw_write()
7011 		size += FAULT_SIZE_ID - cnt;  in tracing_mark_raw_write()
7013 	buffer = tr->array_buffer.buffer;  in tracing_mark_raw_write()
7016 		return -EINVAL;  in tracing_mark_raw_write()
7022 		return -EBADF;  in tracing_mark_raw_write()
7026 	len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);  in tracing_mark_raw_write()
7028 		entry->id = -1;  in tracing_mark_raw_write()
7029 		memcpy(&entry->buf, FAULTED_STR, FAULTED_SIZE);  in tracing_mark_raw_write()
7030 		written = -EFAULT;  in tracing_mark_raw_write()
7032 		written = cnt;  in tracing_mark_raw_write()
7041 	struct trace_array *tr = m->private;  in tracing_clock_show()
7047 			i == tr->clock_id ? "[" : "", trace_clocks[i].name,  in tracing_clock_show()
7048 			i == tr->clock_id ? "]" : "");  in tracing_clock_show()
7059 		if (strcmp(trace_clocks[i].name, clockstr) == 0)  in tracing_set_clock()
7063 		return -EINVAL;  in tracing_set_clock()
7067 	tr->clock_id = i;  in tracing_set_clock()
7069 	ring_buffer_set_clock(tr->array_buffer.buffer, trace_clocks[i].func);  in tracing_set_clock()
7075 	tracing_reset_online_cpus(&tr->array_buffer);  in tracing_set_clock()
7078 	if (tr->max_buffer.buffer)  in tracing_set_clock()
7079 		ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);  in tracing_set_clock()
7080 	tracing_reset_online_cpus(&tr->max_buffer);  in tracing_set_clock()
7089 				   size_t cnt, loff_t *fpos)  in tracing_clock_write()  argument
7091 	struct seq_file *m = filp->private_data;  in tracing_clock_write()
7092 	struct trace_array *tr = m->private;  in tracing_clock_write()
7097 	if (cnt >= sizeof(buf))  in tracing_clock_write()
7098 		return -EINVAL;  in tracing_clock_write()
7100 	if (copy_from_user(buf, ubuf, cnt))  in tracing_clock_write()
7101 		return -EFAULT;  in tracing_clock_write()
7103 	buf[cnt] = 0;  in tracing_clock_write()
7111 	*fpos += cnt;  in tracing_clock_write()
7113 	return cnt;  in tracing_clock_write()
7118 	struct trace_array *tr = inode->i_private;  in tracing_clock_open()
7125 	ret = single_open(file, tracing_clock_show, inode->i_private);  in tracing_clock_open()
7134 	struct trace_array *tr = m->private;  in tracing_time_stamp_mode_show()
7138 	if (ring_buffer_time_stamp_abs(tr->array_buffer.buffer))  in tracing_time_stamp_mode_show()
7150 	struct trace_array *tr = inode->i_private;  in tracing_time_stamp_mode_open()
7157 	ret = single_open(file, tracing_time_stamp_mode_show, inode->i_private);  in tracing_time_stamp_mode_open()
7179 	if (set && tr->no_filter_buffering_ref++)  in tracing_set_filter_buffering()
7183 		if (WARN_ON_ONCE(!tr->no_filter_buffering_ref))  in tracing_set_filter_buffering()
7184 			return -EINVAL;  in tracing_set_filter_buffering()
7186 		--tr->no_filter_buffering_ref;  in tracing_set_filter_buffering()
7203 	struct trace_array *tr = inode->i_private;  in tracing_snapshot_open()
7212 	if (file->f_mode & FMODE_READ) {  in tracing_snapshot_open()
7218 		ret = -ENOMEM;  in tracing_snapshot_open()
7229 		iter->tr = tr;  in tracing_snapshot_open()
7230 		iter->array_buffer = &tr->max_buffer;  in tracing_snapshot_open()
7231 		iter->cpu_file = tracing_get_cpu(inode);  in tracing_snapshot_open()
7232 		m->private = iter;  in tracing_snapshot_open()
7233 		file->private_data = m;  in tracing_snapshot_open()
7248 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,  in tracing_snapshot_write()  argument
7251 	struct seq_file *m = filp->private_data;  in tracing_snapshot_write()
7252 	struct trace_iterator *iter = m->private;  in tracing_snapshot_write()
7253 	struct trace_array *tr = iter->tr;  in tracing_snapshot_write()
7261 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);  in tracing_snapshot_write()
7267 	if (tr->current_trace->use_max_tr)  in tracing_snapshot_write()
7268 		return -EBUSY;  in tracing_snapshot_write()
7271 	arch_spin_lock(&tr->max_lock);  in tracing_snapshot_write()
7272 	if (tr->cond_snapshot)  in tracing_snapshot_write()
7273 		ret = -EBUSY;  in tracing_snapshot_write()
7274 	arch_spin_unlock(&tr->max_lock);  in tracing_snapshot_write()
7281 		if (iter->cpu_file != RING_BUFFER_ALL_CPUS)  in tracing_snapshot_write()
7282 			return -EINVAL;  in tracing_snapshot_write()
7283 		if (tr->allocated_snapshot)  in tracing_snapshot_write()
7287 /* Only allow per-cpu swap if the ring buffer supports it */  in tracing_snapshot_write()
7289 		if (iter->cpu_file != RING_BUFFER_ALL_CPUS)  in tracing_snapshot_write()
7290 			return -EINVAL;  in tracing_snapshot_write()
7292 		if (tr->allocated_snapshot)  in tracing_snapshot_write()
7293 			ret = resize_buffer_duplicate_size(&tr->max_buffer,  in tracing_snapshot_write()
7294 					&tr->array_buffer, iter->cpu_file);  in tracing_snapshot_write()
7301 		if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {  in tracing_snapshot_write()
7306 			smp_call_function_single(iter->cpu_file, tracing_swap_cpu_buffer,  in tracing_snapshot_write()
7312 		if (tr->allocated_snapshot) {  in tracing_snapshot_write()
7313 			if (iter->cpu_file == RING_BUFFER_ALL_CPUS)  in tracing_snapshot_write()
7314 				tracing_reset_online_cpus(&tr->max_buffer);  in tracing_snapshot_write()
7316 				tracing_reset_cpu(&tr->max_buffer, iter->cpu_file);  in tracing_snapshot_write()
7322 		*ppos += cnt;  in tracing_snapshot_write()
7323 		ret = cnt;  in tracing_snapshot_write()
7331 	struct seq_file *m = file->private_data;  in tracing_snapshot_release()
7336 	if (file->f_mode & FMODE_READ)  in tracing_snapshot_release()
7341 		kfree(m->private);  in tracing_snapshot_release()
7364 	info = filp->private_data;  in snapshot_raw_open()
7366 	if (info->iter.trace->use_max_tr) {  in snapshot_raw_open()
7368 		return -EBUSY;  in snapshot_raw_open()
7371 	info->iter.snapshot = true;  in snapshot_raw_open()
7372 	info->iter.array_buffer = &info->iter.tr->max_buffer;  in snapshot_raw_open()
7494  * trace_min_max_write - Write a u64 value to a trace_min_max_param struct
7497  * @cnt: The maximum number of bytes to read
7501  * The filp->private_data must point to a trace_min_max_param structure that
7506 trace_min_max_write(struct file *filp, const char __user *ubuf, size_t cnt, loff_t *ppos)  in trace_min_max_write()  argument
7508 	struct trace_min_max_param *param = filp->private_data;  in trace_min_max_write()
7513 		return -EFAULT;  in trace_min_max_write()
7515 	err = kstrtoull_from_user(ubuf, cnt, 10, &val);  in trace_min_max_write()
7519 	if (param->lock)  in trace_min_max_write()
7520 		mutex_lock(param->lock);  in trace_min_max_write()
7522 	if (param->min && val < *param->min)  in trace_min_max_write()
7523 		err = -EINVAL;  in trace_min_max_write()
7525 	if (param->max && val > *param->max)  in trace_min_max_write()
7526 		err = -EINVAL;  in trace_min_max_write()
7529 		*param->val = val;  in trace_min_max_write()
7531 	if (param->lock)  in trace_min_max_write()
7532 		mutex_unlock(param->lock);  in trace_min_max_write()
7537 	return cnt;  in trace_min_max_write()
7541  * trace_min_max_read - Read a u64 value from a trace_min_max_param struct
7544  * @cnt: The maximum number of bytes to read
7548  * The filp->private_data must point to a trace_min_max_param struct with valid
7552 trace_min_max_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)  in trace_min_max_read()  argument
7554 	struct trace_min_max_param *param = filp->private_data;  in trace_min_max_read()
7560 		return -EFAULT;  in trace_min_max_read()
7562 	val = *param->val;  in trace_min_max_read()
7564 	if (cnt > sizeof(buf))  in trace_min_max_read()
7565 		cnt = sizeof(buf);  in trace_min_max_read()
7569 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, len);  in trace_min_max_read()
7584 	const char	**errs;	/* ptr to loc-specific array of err strings */
7585 	u8		type;	/* index into errs -> specific err string */
7594 	char			*cmd;                     /* what caused err */  member
7605 		return ERR_PTR(-ENOMEM);  in alloc_tracing_log_err()
7607 	err->cmd = kzalloc(len, GFP_KERNEL);  in alloc_tracing_log_err()
7608 	if (!err->cmd) {  in alloc_tracing_log_err()
7610 		return ERR_PTR(-ENOMEM);  in alloc_tracing_log_err()
7618 	kfree(err->cmd);  in free_tracing_log_err()
7626 	char *cmd;  in get_tracing_log_err()  local
7628 	if (tr->n_err_log_entries < TRACING_LOG_ERRS_MAX) {  in get_tracing_log_err()
7630 		if (PTR_ERR(err) != -ENOMEM)  in get_tracing_log_err()
7631 			tr->n_err_log_entries++;  in get_tracing_log_err()
7635 	cmd = kzalloc(len, GFP_KERNEL);  in get_tracing_log_err()
7636 	if (!cmd)  in get_tracing_log_err()
7637 		return ERR_PTR(-ENOMEM);  in get_tracing_log_err()
7638 	err = list_first_entry(&tr->err_log, struct tracing_log_err, list);  in get_tracing_log_err()
7639 	kfree(err->cmd);  in get_tracing_log_err()
7640 	err->cmd = cmd;  in get_tracing_log_err()
7641 	list_del(&err->list);  in get_tracing_log_err()
7647  * err_pos - find the position of a string within a command for error careting
7648  * @cmd: The tracing command that caused the error
7649  * @str: The string to position the caret at within @cmd
7651  * Finds the position of the first occurrence of @str within @cmd.  The
7653  * within @cmd.
7655  * Returns the index within @cmd of the first occurrence of @str or 0
7658 unsigned int err_pos(char *cmd, const char *str)  in err_pos()  argument
7662 	if (WARN_ON(!strlen(cmd)))  in err_pos()
7665 	found = strstr(cmd, str);  in err_pos()
7667 		return found - cmd;  in err_pos()
7673  * tracing_log_err - write an error to the tracing error log
7676  * @cmd: The tracing command that caused the error
7677  * @errs: The array of loc-specific static error strings
7679  * @pos: The position the caret should be placed in the cmd
7684  *   Command: <cmd>
7694  * produce a static error string - this string is not copied and saved
7695  * when the error is logged - only a pointer to it is saved.  See
7700 		     const char *loc, const char *cmd,  in tracing_log_err()  argument
7709 	len += sizeof(CMD_PREFIX) + 2 * sizeof("\n") + strlen(cmd) + 1;  in tracing_log_err()
7714 	if (PTR_ERR(err) == -ENOMEM)  in tracing_log_err()
7717 	snprintf(err->loc, TRACING_LOG_LOC_MAX, "%s: error: ", loc);  in tracing_log_err()
7718 	snprintf(err->cmd, len, "\n" CMD_PREFIX "%s\n", cmd);  in tracing_log_err()
7720 	err->info.errs = errs;  in tracing_log_err()
7721 	err->info.type = type;  in tracing_log_err()
7722 	err->info.pos = pos;  in tracing_log_err()
7723 	err->info.ts = local_clock();  in tracing_log_err()
7725 	list_add_tail(&err->list, &tr->err_log);  in tracing_log_err()
7733 	list_for_each_entry_safe(err, next, &tr->err_log, list) {  in clear_tracing_err_log()
7734 		list_del(&err->list);  in clear_tracing_err_log()
7738 	tr->n_err_log_entries = 0;  in clear_tracing_err_log()
7744 	struct trace_array *tr = m->private;  in tracing_err_log_seq_start()
7748 	return seq_list_start(&tr->err_log, *pos);  in tracing_err_log_seq_start()
7753 	struct trace_array *tr = m->private;  in tracing_err_log_seq_next()
7755 	return seq_list_next(v, &tr->err_log, pos);  in tracing_err_log_seq_next()
7767 	for (i = 0; i < sizeof(CMD_PREFIX) - 1; i++)  in tracing_err_log_show_pos()
7779 		const char *err_text = err->info.errs[err->info.type];  in tracing_err_log_seq_show()
7780 		u64 sec = err->info.ts;  in tracing_err_log_seq_show()
7785 			   err->loc, err_text);  in tracing_err_log_seq_show()
7786 		seq_printf(m, "%s", err->cmd);  in tracing_err_log_seq_show()
7787 		tracing_err_log_show_pos(m, err->info.pos);  in tracing_err_log_seq_show()
7802 	struct trace_array *tr = inode->i_private;  in tracing_err_log_open()
7810 	if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC))  in tracing_err_log_open()
7813 	if (file->f_mode & FMODE_READ) {  in tracing_err_log_open()
7816 			struct seq_file *m = file->private_data;  in tracing_err_log_open()
7817 			m->private = tr;  in tracing_err_log_open()
7834 	struct trace_array *tr = inode->i_private;  in tracing_err_log_release()
7838 	if (file->f_mode & FMODE_READ)  in tracing_err_log_release()
7854 	struct trace_array *tr = inode->i_private;  in tracing_buffers_open()
7865 		return -ENOMEM;  in tracing_buffers_open()
7870 	info->iter.tr		= tr;  in tracing_buffers_open()
7871 	info->iter.cpu_file	= tracing_get_cpu(inode);  in tracing_buffers_open()
7872 	info->iter.trace	= tr->current_trace;  in tracing_buffers_open()
7873 	info->iter.array_buffer = &tr->array_buffer;  in tracing_buffers_open()
7874 	info->spare		= NULL;  in tracing_buffers_open()
7876 	info->read		= (unsigned int)-1;  in tracing_buffers_open()
7878 	filp->private_data = info;  in tracing_buffers_open()
7880 	tr->trace_ref++;  in tracing_buffers_open()
7894 	struct ftrace_buffer_info *info = filp->private_data;  in tracing_buffers_poll()
7895 	struct trace_iterator *iter = &info->iter;  in tracing_buffers_poll()
7904 	struct ftrace_buffer_info *info = filp->private_data;  in tracing_buffers_read()
7905 	struct trace_iterator *iter = &info->iter;  in tracing_buffers_read()
7915 	if (iter->snapshot && iter->tr->current_trace->use_max_tr)  in tracing_buffers_read()
7916 		return -EBUSY;  in tracing_buffers_read()
7919 	page_size = ring_buffer_subbuf_size_get(iter->array_buffer->buffer);  in tracing_buffers_read()
7922 	if (info->spare) {  in tracing_buffers_read()
7923 		if (page_size != info->spare_size) {  in tracing_buffers_read()
7924 			ring_buffer_free_read_page(iter->array_buffer->buffer,  in tracing_buffers_read()
7925 						   info->spare_cpu, info->spare);  in tracing_buffers_read()
7926 			info->spare = NULL;  in tracing_buffers_read()
7930 	if (!info->spare) {  in tracing_buffers_read()
7931 		info->spare = ring_buffer_alloc_read_page(iter->array_buffer->buffer,  in tracing_buffers_read()
7932 							  iter->cpu_file);  in tracing_buffers_read()
7933 		if (IS_ERR(info->spare)) {  in tracing_buffers_read()
7934 			ret = PTR_ERR(info->spare);  in tracing_buffers_read()
7935 			info->spare = NULL;  in tracing_buffers_read()
7937 			info->spare_cpu = iter->cpu_file;  in tracing_buffers_read()
7938 			info->spare_size = page_size;  in tracing_buffers_read()
7941 	if (!info->spare)  in tracing_buffers_read()
7945 	if (info->read < page_size)  in tracing_buffers_read()
7949 	trace_access_lock(iter->cpu_file);  in tracing_buffers_read()
7950 	ret = ring_buffer_read_page(iter->array_buffer->buffer,  in tracing_buffers_read()
7951 				    info->spare,  in tracing_buffers_read()
7953 				    iter->cpu_file, 0);  in tracing_buffers_read()
7954 	trace_access_unlock(iter->cpu_file);  in tracing_buffers_read()
7957 		if (trace_empty(iter) && !iter->closed) {  in tracing_buffers_read()
7958 			if ((filp->f_flags & O_NONBLOCK))  in tracing_buffers_read()
7959 				return -EAGAIN;  in tracing_buffers_read()
7970 	info->read = 0;  in tracing_buffers_read()
7972 	size = page_size - info->read;  in tracing_buffers_read()
7975 	trace_data = ring_buffer_read_page_data(info->spare);  in tracing_buffers_read()
7976 	ret = copy_to_user(ubuf, trace_data + info->read, size);  in tracing_buffers_read()
7978 		return -EFAULT;  in tracing_buffers_read()
7980 	size -= ret;  in tracing_buffers_read()
7983 	info->read += size;  in tracing_buffers_read()
7990 	struct ftrace_buffer_info *info = file->private_data;  in tracing_buffers_flush()
7991 	struct trace_iterator *iter = &info->iter;  in tracing_buffers_flush()
7993 	iter->closed = true;  in tracing_buffers_flush()
7995 	(void)atomic_fetch_inc_release(&iter->wait_index);  in tracing_buffers_flush()
7997 	ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file);  in tracing_buffers_flush()
8004 	struct ftrace_buffer_info *info = file->private_data;  in tracing_buffers_release()
8005 	struct trace_iterator *iter = &info->iter;  in tracing_buffers_release()
8009 	iter->tr->trace_ref--;  in tracing_buffers_release()
8011 	__trace_array_put(iter->tr);  in tracing_buffers_release()
8013 	if (info->spare)  in tracing_buffers_release()
8014 		ring_buffer_free_read_page(iter->array_buffer->buffer,  in tracing_buffers_release()
8015 					   info->spare_cpu, info->spare);  in tracing_buffers_release()
8032 	if (!refcount_dec_and_test(&ref->refcount))  in buffer_ref_release()
8034 	ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);  in buffer_ref_release()
8041 	struct buffer_ref *ref = (struct buffer_ref *)buf->private;  in buffer_pipe_buf_release()
8044 	buf->private = 0;  in buffer_pipe_buf_release()
8050 	struct buffer_ref *ref = (struct buffer_ref *)buf->private;  in buffer_pipe_buf_get()
8052 	if (refcount_read(&ref->refcount) > INT_MAX/2)  in buffer_pipe_buf_get()
8055 	refcount_inc(&ref->refcount);  in buffer_pipe_buf_get()
8072 		(struct buffer_ref *)spd->partial[i].private;  in buffer_spd_release()
8075 	spd->partial[i].private = 0;  in buffer_spd_release()
8083 	struct ftrace_buffer_info *info = file->private_data;  in tracing_buffers_splice_read()
8084 	struct trace_iterator *iter = &info->iter;  in tracing_buffers_splice_read()
8101 	if (iter->snapshot && iter->tr->current_trace->use_max_tr)  in tracing_buffers_splice_read()
8102 		return -EBUSY;  in tracing_buffers_splice_read()
8105 	page_size = ring_buffer_subbuf_size_get(iter->array_buffer->buffer);  in tracing_buffers_splice_read()
8106 	if (*ppos & (page_size - 1))  in tracing_buffers_splice_read()
8107 		return -EINVAL;  in tracing_buffers_splice_read()
8109 	if (len & (page_size - 1)) {  in tracing_buffers_splice_read()
8111 			return -EINVAL;  in tracing_buffers_splice_read()
8112 		len &= (~(page_size - 1));  in tracing_buffers_splice_read()
8116 		return -ENOMEM;  in tracing_buffers_splice_read()
8119 	trace_access_lock(iter->cpu_file);  in tracing_buffers_splice_read()
8120 	entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);  in tracing_buffers_splice_read()
8122 	for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= page_size) {  in tracing_buffers_splice_read()
8128 			ret = -ENOMEM;  in tracing_buffers_splice_read()
8132 		refcount_set(&ref->refcount, 1);  in tracing_buffers_splice_read()
8133 		ref->buffer = iter->array_buffer->buffer;  in tracing_buffers_splice_read()
8134 		ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);  in tracing_buffers_splice_read()
8135 		if (IS_ERR(ref->page)) {  in tracing_buffers_splice_read()
8136 			ret = PTR_ERR(ref->page);  in tracing_buffers_splice_read()
8137 			ref->page = NULL;  in tracing_buffers_splice_read()
8141 		ref->cpu = iter->cpu_file;  in tracing_buffers_splice_read()
8143 		r = ring_buffer_read_page(ref->buffer, ref->page,  in tracing_buffers_splice_read()
8144 					  len, iter->cpu_file, 1);  in tracing_buffers_splice_read()
8146 			ring_buffer_free_read_page(ref->buffer, ref->cpu,  in tracing_buffers_splice_read()
8147 						   ref->page);  in tracing_buffers_splice_read()
8152 		page = virt_to_page(ring_buffer_read_page_data(ref->page));  in tracing_buffers_splice_read()
8161 		entries = ring_buffer_entries_cpu(iter->array_buffer->buffer, iter->cpu_file);  in tracing_buffers_splice_read()
8164 	trace_access_unlock(iter->cpu_file);  in tracing_buffers_splice_read()
8176 		ret = -EAGAIN;  in tracing_buffers_splice_read()
8177 		if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))  in tracing_buffers_splice_read()
8180 		ret = wait_on_pipe(iter, iter->snapshot ? 0 : iter->tr->buffer_percent);  in tracing_buffers_splice_read()
8185 		if (!tracer_tracing_is_on(iter->tr))  in tracing_buffers_splice_read()
8201 static long tracing_buffers_ioctl(struct file *file, unsigned int cmd, unsigned long arg)  in tracing_buffers_ioctl()  argument
8203 	struct ftrace_buffer_info *info = file->private_data;  in tracing_buffers_ioctl()
8204 	struct trace_iterator *iter = &info->iter;  in tracing_buffers_ioctl()
8207 	if (cmd == TRACE_MMAP_IOCTL_GET_READER) {  in tracing_buffers_ioctl()
8208 		if (!(file->f_flags & O_NONBLOCK)) {  in tracing_buffers_ioctl()
8209 			err = ring_buffer_wait(iter->array_buffer->buffer,  in tracing_buffers_ioctl()
8210 					       iter->cpu_file,  in tracing_buffers_ioctl()
8211 					       iter->tr->buffer_percent,  in tracing_buffers_ioctl()
8217 		return ring_buffer_map_get_reader(iter->array_buffer->buffer,  in tracing_buffers_ioctl()
8218 						  iter->cpu_file);  in tracing_buffers_ioctl()
8219 	} else if (cmd) {  in tracing_buffers_ioctl()
8220 		return -ENOTTY;  in tracing_buffers_ioctl()
8224 	 * An ioctl call with cmd 0 to the ring buffer file will wake up all  in tracing_buffers_ioctl()
8230 	(void)atomic_fetch_inc_release(&iter->wait_index);  in tracing_buffers_ioctl()
8232 	ring_buffer_wake_waiters(iter->array_buffer->buffer, iter->cpu_file);  in tracing_buffers_ioctl()
8248 	spin_lock(&tr->snapshot_trigger_lock);  in get_snapshot_map()
8250 	if (tr->snapshot || tr->mapped == UINT_MAX)  in get_snapshot_map()
8251 		err = -EBUSY;  in get_snapshot_map()
8253 		tr->mapped++;  in get_snapshot_map()
8255 	spin_unlock(&tr->snapshot_trigger_lock);  in get_snapshot_map()
8257 	/* Wait for update_max_tr() to observe iter->tr->mapped */  in get_snapshot_map()
8258 	if (tr->mapped == 1)  in get_snapshot_map()
8266 	spin_lock(&tr->snapshot_trigger_lock);  in put_snapshot_map()
8267 	if (!WARN_ON(!tr->mapped))  in put_snapshot_map()
8268 		tr->mapped--;  in put_snapshot_map()
8269 	spin_unlock(&tr->snapshot_trigger_lock);  in put_snapshot_map()
8278 	struct ftrace_buffer_info *info = vma->vm_file->private_data;  in tracing_buffers_mmap_close()
8279 	struct trace_iterator *iter = &info->iter;  in tracing_buffers_mmap_close()
8281 	WARN_ON(ring_buffer_unmap(iter->array_buffer->buffer, iter->cpu_file));  in tracing_buffers_mmap_close()
8282 	put_snapshot_map(iter->tr);  in tracing_buffers_mmap_close()
8291 	struct ftrace_buffer_info *info = filp->private_data;  in tracing_buffers_mmap()
8292 	struct trace_iterator *iter = &info->iter;  in tracing_buffers_mmap()
8296 	if (iter->tr->flags & TRACE_ARRAY_FL_BOOT)  in tracing_buffers_mmap()
8297 		return -ENODEV;  in tracing_buffers_mmap()
8299 	ret = get_snapshot_map(iter->tr);  in tracing_buffers_mmap()
8303 	ret = ring_buffer_map(iter->array_buffer->buffer, iter->cpu_file, vma);  in tracing_buffers_mmap()
8305 		put_snapshot_map(iter->tr);  in tracing_buffers_mmap()
8307 	vma->vm_ops = &tracing_buffers_vmops;  in tracing_buffers_mmap()
8328 	struct trace_array *tr = inode->i_private;  in tracing_stats_read()
8329 	struct array_buffer *trace_buf = &tr->array_buffer;  in tracing_stats_read()
8332 	unsigned long cnt;  in tracing_stats_read()  local
8338 		return -ENOMEM;  in tracing_stats_read()
8342 	cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);  in tracing_stats_read()
8343 	trace_seq_printf(s, "entries: %ld\n", cnt);  in tracing_stats_read()
8345 	cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);  in tracing_stats_read()
8346 	trace_seq_printf(s, "overrun: %ld\n", cnt);  in tracing_stats_read()
8348 	cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);  in tracing_stats_read()
8349 	trace_seq_printf(s, "commit overrun: %ld\n", cnt);  in tracing_stats_read()
8351 	cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);  in tracing_stats_read()
8352 	trace_seq_printf(s, "bytes: %ld\n", cnt);  in tracing_stats_read()
8354 	if (trace_clocks[tr->clock_id].in_ns) {  in tracing_stats_read()
8356 		t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));  in tracing_stats_read()
8361 		t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer));  in tracing_stats_read()
8367 				ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));  in tracing_stats_read()
8370 				ring_buffer_time_stamp(trace_buf->buffer));  in tracing_stats_read()
8373 	cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);  in tracing_stats_read()
8374 	trace_seq_printf(s, "dropped events: %ld\n", cnt);  in tracing_stats_read()
8376 	cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);  in tracing_stats_read()
8377 	trace_seq_printf(s, "read events: %ld\n", cnt);  in tracing_stats_read()
8380 					s->buffer, trace_seq_used(s));  in tracing_stats_read()
8398 		  size_t cnt, loff_t *ppos)  in tracing_read_dyn_info()  argument
8409 		return -ENOMEM;  in tracing_read_dyn_info()
8421 	ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);  in tracing_read_dyn_info()
8458 		(*count)--;  in ftrace_count_snapshot()
8495 			return -ENOMEM;  in ftrace_snapshot_init()
8532 			       char *glob, char *cmd, char *param, int enable)  in ftrace_trace_snapshot_callback()  argument
8535 	void *count = (void *)-1;  in ftrace_trace_snapshot_callback()
8540 		return -ENODEV;  in ftrace_trace_snapshot_callback()
8544 		return -EINVAL;  in ftrace_trace_snapshot_callback()
8585 	.name			= "snapshot",
8599 	if (WARN_ON(!tr->dir))  in tracing_get_dentry()
8600 		return ERR_PTR(-ENODEV);  in tracing_get_dentry()
8603 	if (tr->flags & TRACE_ARRAY_FL_GLOBAL)  in tracing_get_dentry()
8607 	return tr->dir;  in tracing_get_dentry()
8614 	if (tr->percpu_dir)  in tracing_dentry_percpu()
8615 		return tr->percpu_dir;  in tracing_dentry_percpu()
8621 	tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);  in tracing_dentry_percpu()
8623 	MEM_FAIL(!tr->percpu_dir,  in tracing_dentry_percpu()
8626 	return tr->percpu_dir;  in tracing_dentry_percpu()
8630 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,  in trace_create_cpu_file()  argument
8633 	struct dentry *ret = trace_create_file(name, mode, parent, data, fops);  in trace_create_cpu_file()
8636 		d_inode(ret)->i_cdev = (void *)(cpu + 1);  in trace_create_cpu_file()
8674 	if (tr->range_addr_start)  in tracing_init_tracefs_percpu()
8678 	if (!tr->range_addr_start) {  in tracing_init_tracefs_percpu()
8694 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,  in trace_options_read()  argument
8697 	struct trace_option_dentry *topt = filp->private_data;  in trace_options_read()
8700 	if (topt->flags->val & topt->opt->bit)  in trace_options_read()
8705 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);  in trace_options_read()
8709 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,  in trace_options_write()  argument
8712 	struct trace_option_dentry *topt = filp->private_data;  in trace_options_write()
8716 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);  in trace_options_write()
8721 		return -EINVAL;  in trace_options_write()
8723 	if (!!(topt->flags->val & topt->opt->bit) != val) {  in trace_options_write()
8725 		ret = __set_tracer_option(topt->tr, topt->flags,  in trace_options_write()
8726 					  topt->opt, !val);  in trace_options_write()
8732 	*ppos += cnt;  in trace_options_write()
8734 	return cnt;  in trace_options_write()
8739 	struct trace_option_dentry *topt = inode->i_private;  in tracing_open_options()
8742 	ret = tracing_check_open_get_tr(topt->tr);  in tracing_open_options()
8746 	filp->private_data = inode->i_private;  in tracing_open_options()
8752 	struct trace_option_dentry *topt = file->private_data;  in tracing_release_options()
8754 	trace_array_put(topt->tr);  in tracing_release_options()
8785  *   ptr - idx == &index[0]
8795 	*ptr = container_of(data - *pindex, struct trace_array,  in get_tr_index()
8800 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,  in trace_options_core_read()  argument
8803 	void *tr_index = filp->private_data;  in trace_options_core_read()
8810 	if (tr->trace_flags & (1 << index))  in trace_options_core_read()
8815 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);  in trace_options_core_read()
8819 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,  in trace_options_core_write()  argument
8822 	void *tr_index = filp->private_data;  in trace_options_core_write()
8830 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);  in trace_options_core_write()
8835 		return -EINVAL;  in trace_options_core_write()
8846 	*ppos += cnt;  in trace_options_core_write()
8848 	return cnt;  in trace_options_core_write()
8858 struct dentry *trace_create_file(const char *name,  in trace_create_file()  argument
8866 	ret = tracefs_create_file(name, mode, parent, data, fops);  in trace_create_file()
8868 		pr_warn("Could not create tracefs '%s' entry\n", name);  in trace_create_file()
8878 	if (tr->options)  in trace_options_init_dentry()
8879 		return tr->options;  in trace_options_init_dentry()
8885 	tr->options = tracefs_create_dir("options", d_tracer);  in trace_options_init_dentry()
8886 	if (!tr->options) {  in trace_options_init_dentry()
8891 	return tr->options;  in trace_options_init_dentry()
8906 	topt->flags = flags;  in create_trace_option_file()
8907 	topt->opt = opt;  in create_trace_option_file()
8908 	topt->tr = tr;  in create_trace_option_file()
8910 	topt->entry = trace_create_file(opt->name, TRACE_MODE_WRITE,  in create_trace_option_file()
8922 	int cnt;  in create_trace_option_files()  local
8928 	flags = tracer->flags;  in create_trace_option_files()
8930 	if (!flags || !flags->opts)  in create_trace_option_files()
8940 	for (i = 0; i < tr->nr_topts; i++) {  in create_trace_option_files()
8942 		if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))  in create_trace_option_files()
8946 	opts = flags->opts;  in create_trace_option_files()
8948 	for (cnt = 0; opts[cnt].name; cnt++)  in create_trace_option_files()
8951 	topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);  in create_trace_option_files()
8955 	tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),  in create_trace_option_files()
8962 	tr->topts = tr_topts;  in create_trace_option_files()
8963 	tr->topts[tr->nr_topts].tracer = tracer;  in create_trace_option_files()
8964 	tr->topts[tr->nr_topts].topts = topts;  in create_trace_option_files()
8965 	tr->nr_topts++;  in create_trace_option_files()
8967 	for (cnt = 0; opts[cnt].name; cnt++) {  in create_trace_option_files()
8968 		create_trace_option_file(tr, &topts[cnt], flags,  in create_trace_option_files()
8969 					 &opts[cnt]);  in create_trace_option_files()
8970 		MEM_FAIL(topts[cnt].entry == NULL,  in create_trace_option_files()
8972 			  opts[cnt].name);  in create_trace_option_files()
8987 				 (void *)&tr->trace_flags_index[index],  in create_trace_option_core_file()
9010 	       size_t cnt, loff_t *ppos)  in rb_simple_read()  argument
9012 	struct trace_array *tr = filp->private_data;  in rb_simple_read()
9019 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);  in rb_simple_read()
9024 		size_t cnt, loff_t *ppos)  in rb_simple_write()  argument
9026 	struct trace_array *tr = filp->private_data;  in rb_simple_write()
9027 	struct trace_buffer *buffer = tr->array_buffer.buffer;  in rb_simple_write()
9031 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);  in rb_simple_write()
9041 			if (tr->current_trace->start)  in rb_simple_write()
9042 				tr->current_trace->start(tr);  in rb_simple_write()
9045 			if (tr->current_trace->stop)  in rb_simple_write()
9046 				tr->current_trace->stop(tr);  in rb_simple_write()
9055 	return cnt;  in rb_simple_write()
9068 		    size_t cnt, loff_t *ppos)  in buffer_percent_read()  argument
9070 	struct trace_array *tr = filp->private_data;  in buffer_percent_read()
9074 	r = tr->buffer_percent;  in buffer_percent_read()
9077 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);  in buffer_percent_read()
9082 		     size_t cnt, loff_t *ppos)  in buffer_percent_write()  argument
9084 	struct trace_array *tr = filp->private_data;  in buffer_percent_write()
9088 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);  in buffer_percent_write()
9093 		return -EINVAL;  in buffer_percent_write()
9095 	tr->buffer_percent = val;  in buffer_percent_write()
9099 	return cnt;  in buffer_percent_write()
9111 buffer_subbuf_size_read(struct file *filp, char __user *ubuf, size_t cnt, loff_t *ppos)  in buffer_subbuf_size_read()  argument
9113 	struct trace_array *tr = filp->private_data;  in buffer_subbuf_size_read()
9119 	order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer);  in buffer_subbuf_size_read()
9124 	return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);  in buffer_subbuf_size_read()
9129 			 size_t cnt, loff_t *ppos)  in buffer_subbuf_size_write()  argument
9131 	struct trace_array *tr = filp->private_data;  in buffer_subbuf_size_write()
9138 	ret = kstrtoul_from_user(ubuf, cnt, 10, &val);  in buffer_subbuf_size_write()
9145 	order = fls(pages - 1);  in buffer_subbuf_size_write()
9149 		return -EINVAL;  in buffer_subbuf_size_write()
9154 	old_order = ring_buffer_subbuf_order_get(tr->array_buffer.buffer);  in buffer_subbuf_size_write()
9158 	ret = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, order);  in buffer_subbuf_size_write()
9164 	if (!tr->allocated_snapshot)  in buffer_subbuf_size_write()
9167 	ret = ring_buffer_subbuf_order_set(tr->max_buffer.buffer, order);  in buffer_subbuf_size_write()
9170 		cnt = ring_buffer_subbuf_order_set(tr->array_buffer.buffer, old_order);  in buffer_subbuf_size_write()
9171 		if (WARN_ON_ONCE(cnt)) {  in buffer_subbuf_size_write()
9194 		cnt = ret;  in buffer_subbuf_size_write()
9196 	return cnt;  in buffer_subbuf_size_write()
9217 	rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;  in allocate_trace_buffer()
9219 	buf->tr = tr;  in allocate_trace_buffer()
9221 	if (tr->range_addr_start && tr->range_addr_size) {  in allocate_trace_buffer()
9222 		buf->buffer = ring_buffer_alloc_range(size, rb_flags, 0,  in allocate_trace_buffer()
9223 						      tr->range_addr_start,  in allocate_trace_buffer()
9224 						      tr->range_addr_size);  in allocate_trace_buffer()
9226 		ring_buffer_last_boot_delta(buf->buffer,  in allocate_trace_buffer()
9227 					    &tr->text_delta, &tr->data_delta);  in allocate_trace_buffer()
9232 		tr->mapped++;  in allocate_trace_buffer()
9234 		buf->buffer = ring_buffer_alloc(size, rb_flags);  in allocate_trace_buffer()
9236 	if (!buf->buffer)  in allocate_trace_buffer()
9237 		return -ENOMEM;  in allocate_trace_buffer()
9239 	buf->data = alloc_percpu(struct trace_array_cpu);  in allocate_trace_buffer()
9240 	if (!buf->data) {  in allocate_trace_buffer()
9241 		ring_buffer_free(buf->buffer);  in allocate_trace_buffer()
9242 		buf->buffer = NULL;  in allocate_trace_buffer()
9243 		return -ENOMEM;  in allocate_trace_buffer()
9247 	set_buffer_entries(&tr->array_buffer,  in allocate_trace_buffer()
9248 			   ring_buffer_size(tr->array_buffer.buffer, 0));  in allocate_trace_buffer()
9255 	if (buf->buffer) {  in free_trace_buffer()
9256 		ring_buffer_free(buf->buffer);  in free_trace_buffer()
9257 		buf->buffer = NULL;  in free_trace_buffer()
9258 		free_percpu(buf->data);  in free_trace_buffer()
9259 		buf->data = NULL;  in free_trace_buffer()
9267 	ret = allocate_trace_buffer(tr, &tr->array_buffer, size);  in allocate_trace_buffers()
9273 	if (tr->range_addr_start)  in allocate_trace_buffers()
9276 	ret = allocate_trace_buffer(tr, &tr->max_buffer,  in allocate_trace_buffers()
9279 		free_trace_buffer(&tr->array_buffer);  in allocate_trace_buffers()
9280 		return -ENOMEM;  in allocate_trace_buffers()
9282 	tr->allocated_snapshot = allocate_snapshot;  in allocate_trace_buffers()
9295 	free_trace_buffer(&tr->array_buffer);  in free_trace_buffers()
9298 	free_trace_buffer(&tr->max_buffer);  in free_trace_buffers()
9308 		tr->trace_flags_index[i] = i;  in init_trace_flags_index()
9315 	for (t = trace_types; t; t = t->next)  in __update_tracer_options()
9333 		if (tr->name && strcmp(tr->name, instance) == 0) {  in trace_array_find()
9349 		tr->ref++;  in trace_array_find_get()
9359 	tr->dir = tracefs_create_dir(tr->name, trace_instance_dir);  in trace_array_create_dir()
9360 	if (!tr->dir)  in trace_array_create_dir()
9361 		return -EINVAL;  in trace_array_create_dir()
9363 	ret = event_trace_add_tracer(tr->dir, tr);  in trace_array_create_dir()
9365 		tracefs_remove(tr->dir);  in trace_array_create_dir()
9369 	init_tracer_tracefs(tr, tr->dir);  in trace_array_create_dir()
9376 trace_array_create_systems(const char *name, const char *systems,  in trace_array_create_systems()  argument
9383 	ret = -ENOMEM;  in trace_array_create_systems()
9388 	tr->name = kstrdup(name, GFP_KERNEL);  in trace_array_create_systems()
9389 	if (!tr->name)  in trace_array_create_systems()
9392 	if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))  in trace_array_create_systems()
9395 	if (!zalloc_cpumask_var(&tr->pipe_cpumask, GFP_KERNEL))  in trace_array_create_systems()
9399 		tr->system_names = kstrdup_const(systems, GFP_KERNEL);  in trace_array_create_systems()
9400 		if (!tr->system_names)  in trace_array_create_systems()
9405 	tr->range_addr_start = range_addr_start;  in trace_array_create_systems()
9406 	tr->range_addr_size = range_addr_size;  in trace_array_create_systems()
9408 	tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;  in trace_array_create_systems()
9410 	cpumask_copy(tr->tracing_cpumask, cpu_all_mask);  in trace_array_create_systems()
9412 	raw_spin_lock_init(&tr->start_lock);  in trace_array_create_systems()
9414 	tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;  in trace_array_create_systems()
9416 	spin_lock_init(&tr->snapshot_trigger_lock);  in trace_array_create_systems()
9418 	tr->current_trace = &nop_trace;  in trace_array_create_systems()
9420 	INIT_LIST_HEAD(&tr->systems);  in trace_array_create_systems()
9421 	INIT_LIST_HEAD(&tr->events);  in trace_array_create_systems()
9422 	INIT_LIST_HEAD(&tr->hist_vars);  in trace_array_create_systems()
9423 	INIT_LIST_HEAD(&tr->err_log);  in trace_array_create_systems()
9426 	INIT_LIST_HEAD(&tr->mod_events);  in trace_array_create_systems()
9449 	list_add(&tr->list, &ftrace_trace_arrays);  in trace_array_create_systems()
9451 	tr->ref++;  in trace_array_create_systems()
9458 	free_cpumask_var(tr->pipe_cpumask);  in trace_array_create_systems()
9459 	free_cpumask_var(tr->tracing_cpumask);  in trace_array_create_systems()
9460 	kfree_const(tr->system_names);  in trace_array_create_systems()
9461 	kfree(tr->name);  in trace_array_create_systems()
9467 static struct trace_array *trace_array_create(const char *name)  in trace_array_create()  argument
9469 	return trace_array_create_systems(name, NULL, 0, 0);  in trace_array_create()
9472 static int instance_mkdir(const char *name)  in instance_mkdir()  argument
9480 	ret = -EEXIST;  in instance_mkdir()
9481 	if (trace_array_find(name))  in instance_mkdir()
9482 		return -EEXIST;  in instance_mkdir()
9484 	tr = trace_array_create(name);  in instance_mkdir()
9517  * trace_array_get_by_name - Create/Lookup a trace array, given its name.
9518  * @name: The name of the trace array to be looked up/created.
9521  * Returns pointer to trace array with given name.
9533 struct trace_array *trace_array_get_by_name(const char *name, const char *systems)  in trace_array_get_by_name()  argument
9541 		if (tr->name && strcmp(tr->name, name) == 0) {  in trace_array_get_by_name()
9542 			tr->ref++;  in trace_array_get_by_name()
9547 	tr = trace_array_create_systems(name, systems, 0, 0);  in trace_array_get_by_name()
9552 		tr->ref++;  in trace_array_get_by_name()
9563 	if (tr->ref > 1 || (tr->current_trace && tr->trace_ref))  in __remove_instance()
9564 		return -EBUSY;  in __remove_instance()
9566 	list_del(&tr->list);  in __remove_instance()
9582 	tracefs_remove(tr->dir);  in __remove_instance()
9583 	free_percpu(tr->last_func_repeats);  in __remove_instance()
9587 	for (i = 0; i < tr->nr_topts; i++) {  in __remove_instance()
9588 		kfree(tr->topts[i].topts);  in __remove_instance()
9590 	kfree(tr->topts);  in __remove_instance()
9592 	free_cpumask_var(tr->pipe_cpumask);  in __remove_instance()
9593 	free_cpumask_var(tr->tracing_cpumask);  in __remove_instance()
9594 	kfree_const(tr->system_names);  in __remove_instance()
9595 	kfree(tr->name);  in __remove_instance()
9606 		return -EINVAL;  in trace_array_destroy()
9618 	return -ENODEV;  in trace_array_destroy()
9622 static int instance_rmdir(const char *name)  in instance_rmdir()  argument
9629 	tr = trace_array_find(name);  in instance_rmdir()
9631 		return -ENODEV;  in instance_rmdir()
9650 		if (!tr->name)  in create_trace_instances()
9693 	tr->trace_marker_file = __find_event_file(tr, "ftrace", "print");  in init_tracer_tracefs()
9707 	tr->buffer_percent = 50;  in init_tracer_tracefs()
9724 	if (tr->range_addr_start) {  in init_tracer_tracefs()
9766  * tracing_init_dentry - initialize top level trace array
9778 		return -EPERM;  in tracing_init_dentry()
9782 	if (tr->dir)  in tracing_init_dentry()
9786 		return -ENODEV;  in tracing_init_dentry()
9794 	tr->dir = debugfs_create_automount("tracing", NULL,  in tracing_init_dentry()
9811 	len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;  in eval_map_work_func()
9824 		return -ENOMEM;  in trace_eval_init()
9856 	if (n > sizeof(modname) - 1)  in module_exists()
9865 	if (!mod->num_trace_evals)  in trace_module_add_evals()
9875 	trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals);  in trace_module_add_evals()
9884 	if (!mod->num_trace_evals)  in trace_module_remove_evals()
9892 		if (map->head.mod == mod)  in trace_module_remove_evals()
9895 		last = &map->tail.next;  in trace_module_remove_evals()
9896 		map = map->tail.next;  in trace_module_remove_evals()
9901 	*last = trace_eval_jmp_to_tail(map)->tail.next;  in trace_module_remove_evals()
9999 	.priority = INT_MAX - 1,
10004 	.priority = INT_MAX - 1,
10045 	if (s->seq.len >= TRACE_MAX_PRINT)  in trace_printk_seq()
10046 		s->seq.len = TRACE_MAX_PRINT;  in trace_printk_seq()
10053 	if (WARN_ON_ONCE(s->seq.len >= s->seq.size))  in trace_printk_seq()
10054 		s->seq.len = s->seq.size - 1;  in trace_printk_seq()
10057 	s->buffer[s->seq.len] = 0;  in trace_printk_seq()
10059 	printk(KERN_TRACE "%s", s->buffer);  in trace_printk_seq()
10066 	iter->tr = tr;  in trace_init_iter()
10067 	iter->trace = iter->tr->current_trace;  in trace_init_iter()
10068 	iter->cpu_file = RING_BUFFER_ALL_CPUS;  in trace_init_iter()
10069 	iter->array_buffer = &tr->array_buffer;  in trace_init_iter()
10071 	if (iter->trace && iter->trace->open)  in trace_init_iter()
10072 		iter->trace->open(iter);  in trace_init_iter()
10075 	if (ring_buffer_overruns(iter->array_buffer->buffer))  in trace_init_iter()
10076 		iter->iter_flags |= TRACE_FILE_ANNOTATE;  in trace_init_iter()
10079 	if (trace_clocks[iter->tr->clock_id].in_ns)  in trace_init_iter()
10080 		iter->iter_flags |= TRACE_FILE_TIME_IN_NS;  in trace_init_iter()
10083 	iter->temp = static_temp_buf;  in trace_init_iter()
10084 	iter->temp_size = STATIC_TEMP_BUF_SIZE;  in trace_init_iter()
10085 	iter->fmt = static_fmt_buf;  in trace_init_iter()
10086 	iter->fmt_size = STATIC_FMT_BUF_SIZE;  in trace_init_iter()
10100 	int cnt = 0, cpu;  in ftrace_dump_one()  local
10107 	 * If the user does a sysrq-z, then they can re-enable  in ftrace_dump_one()
10118 		atomic_inc(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);  in ftrace_dump_one()
10121 	old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;  in ftrace_dump_one()
10124 	tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;  in ftrace_dump_one()
10134 		printk(KERN_TRACE "Dumping ftrace instance %s buffer:\n", tr->name);  in ftrace_dump_one()
10151 		if (!cnt)  in ftrace_dump_one()
10152 			printk(KERN_TRACE "---------------------------------\n");  in ftrace_dump_one()
10154 		cnt++;  in ftrace_dump_one()
10171 	if (!cnt)  in ftrace_dump_one()
10174 		printk(KERN_TRACE "---------------------------------\n");  in ftrace_dump_one()
10176 	tr->trace_flags |= old_userobj;  in ftrace_dump_one()
10179 		atomic_dec(&per_cpu_ptr(iter.array_buffer->data, cpu)->disabled);  in ftrace_dump_one()
10269 		return -ENOMEM;  in trace_parse_run_command()
10272 		size = count - done;  in trace_parse_run_command()
10275 			size = WRITE_BUFSIZE - 1;  in trace_parse_run_command()
10278 			ret = -EFAULT;  in trace_parse_run_command()
10287 				size = tmp - buf + 1;  in trace_parse_run_command()
10293 					/* This can accept WRITE_BUFSIZE - 2 ('\n' + '\0') */  in trace_parse_run_command()
10295 						WRITE_BUFSIZE - 2);  in trace_parse_run_command()
10296 					ret = -EINVAL;  in trace_parse_run_command()
10324 __init static bool tr_needs_alloc_snapshot(const char *name)  in tr_needs_alloc_snapshot()  argument
10327 	int len = strlen(name);  in tr_needs_alloc_snapshot()
10333 	if (strncmp(name, boot_snapshot_info, len) == 0 &&  in tr_needs_alloc_snapshot()
10337 	test = kmalloc(strlen(name) + 3, GFP_KERNEL);  in tr_needs_alloc_snapshot()
10341 	sprintf(test, "\t%s\t", name);  in tr_needs_alloc_snapshot()
10347 __init static void do_allocate_snapshot(const char *name)  in do_allocate_snapshot()  argument
10349 	if (!tr_needs_alloc_snapshot(name))  in do_allocate_snapshot()
10361 static inline void do_allocate_snapshot(const char *name) { }  in do_allocate_snapshot()  argument
10368 	char *name;  in enable_instances()  local
10373 	boot_instance_info[boot_instance_index - 1] = '\0';  in enable_instances()
10396 		name = tok;  in enable_instances()
10410 						flag, name);  in enable_instances()
10420 					name);  in enable_instances()
10424 				pr_warn("Tracing: No size specified for instance %s\n", name);  in enable_instances()
10431 					name);  in enable_instances()
10437 				pr_warn("Failed to map boot instance %s to %s\n", name, tok);  in enable_instances()
10446 					name, &start, (unsigned long)size);  in enable_instances()
10448 				pr_warn("Tracing: Failed to map boot instance %s\n", name);  in enable_instances()
10454 				do_allocate_snapshot(name);  in enable_instances()
10457 		tr = trace_array_create_systems(name, NULL, addr, size);  in enable_instances()
10475 			tr->flags |= TRACE_ARRAY_FL_BOOT;  in enable_instances()
10476 			tr->ref++;  in enable_instances()
10488 	int ret = -ENOMEM;  in tracer_alloc_buffers()
10493 		return -EPERM;  in tracer_alloc_buffers()
10536 	ret = -ENOMEM;  in tracer_alloc_buffers()
10641 		if (!tr->allocated_snapshot)  in ftrace_boot_snapshot()