69 static void trace_note(struct blk_trace *bt, pid_t pid, int action,  in trace_note()  argument
92 	if (!bt->rchan)  in trace_note()
95 	t = relay_reserve(bt->rchan, sizeof(*t) + len + cgid_len);  in trace_note()
100 		t->device = bt->dev;  in trace_note()
121 	struct blk_trace *bt;  in trace_note_tsk()  local
125 	list_for_each_entry(bt, &running_trace_list, running_list) {  in trace_note_tsk()
126 		trace_note(bt, tsk->pid, BLK_TN_PROCESS, tsk->comm,  in trace_note_tsk()
132 static void trace_note_time(struct blk_trace *bt)  in trace_note_time()  argument
144 	trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words), 0);  in trace_note_time()
148 void __blk_trace_note_message(struct blk_trace *bt,  in __blk_trace_note_message()  argument
157 	if (unlikely(bt->trace_state != Blktrace_running &&  in __blk_trace_note_message()
165 	if (!(bt->act_mask & BLK_TC_NOTIFY))  in __blk_trace_note_message()
169 	buf = this_cpu_ptr(bt->msg_data);  in __blk_trace_note_message()
180 	trace_note(bt, current->pid, BLK_TN_MESSAGE, buf, n, cgid);  in __blk_trace_note_message()
185 static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector,  in act_log_check()  argument
188 	if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0)  in act_log_check()
190 	if (sector && (sector < bt->start_lba || sector > bt->end_lba))  in act_log_check()
192 	if (bt->pid && pid != bt->pid)  in act_log_check()
215 static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,  in __blk_add_trace()  argument
232 	if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer))  in __blk_add_trace()
249 	if (act_log_check(bt, what, sector, pid))  in __blk_add_trace()
276 	t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len + cgid_len);  in __blk_add_trace()
278 		sequence = per_cpu_ptr(bt->sequence, cpu);  in __blk_add_trace()
296 		t->device = bt->dev;  in __blk_add_trace()
314 static void blk_trace_free(struct request_queue *q, struct blk_trace *bt)  in blk_trace_free()  argument
316 	relay_close(bt->rchan);  in blk_trace_free()
319 	 * If 'bt->dir' is not set, then both 'dropped' and 'msg' are created  in blk_trace_free()
322 	if (!bt->dir) {  in blk_trace_free()
326 		debugfs_remove(bt->dir);  in blk_trace_free()
328 	free_percpu(bt->sequence);  in blk_trace_free()
329 	free_percpu(bt->msg_data);  in blk_trace_free()
330 	kfree(bt);  in blk_trace_free()
349 static int blk_trace_start(struct blk_trace *bt)  in blk_trace_start()  argument
351 	if (bt->trace_state != Blktrace_setup &&  in blk_trace_start()
352 	    bt->trace_state != Blktrace_stopped)  in blk_trace_start()
357 	bt->trace_state = Blktrace_running;  in blk_trace_start()
359 	list_add(&bt->running_list, &running_trace_list);  in blk_trace_start()
361 	trace_note_time(bt);  in blk_trace_start()
366 static int blk_trace_stop(struct blk_trace *bt)  in blk_trace_stop()  argument
368 	if (bt->trace_state != Blktrace_running)  in blk_trace_stop()
371 	bt->trace_state = Blktrace_stopped;  in blk_trace_stop()
373 	list_del_init(&bt->running_list);  in blk_trace_stop()
375 	relay_flush(bt->rchan);  in blk_trace_stop()
380 static void blk_trace_cleanup(struct request_queue *q, struct blk_trace *bt)  in blk_trace_cleanup()  argument
382 	blk_trace_stop(bt);  in blk_trace_cleanup()
384 	blk_trace_free(q, bt);  in blk_trace_cleanup()
390 	struct blk_trace *bt;  in __blk_trace_remove()  local
392 	bt = rcu_replace_pointer(q->blk_trace, NULL,  in __blk_trace_remove()
394 	if (!bt)  in __blk_trace_remove()
397 	blk_trace_cleanup(q, bt);  in __blk_trace_remove()
417 	struct blk_trace *bt = filp->private_data;  in blk_dropped_read()  local
418 	size_t dropped = relay_stats(bt->rchan, RELAY_STATS_BUF_FULL);  in blk_dropped_read()
437 	struct blk_trace *bt;  in blk_msg_write()  local
446 	bt = filp->private_data;  in blk_msg_write()
447 	__blk_trace_note_message(bt, NULL, "%s", msg);  in blk_msg_write()
482 static void blk_trace_setup_lba(struct blk_trace *bt,  in blk_trace_setup_lba()  argument
486 		bt->start_lba = bdev->bd_start_sect;  in blk_trace_setup_lba()
487 		bt->end_lba = bdev->bd_start_sect + bdev_nr_sectors(bdev);  in blk_trace_setup_lba()
489 		bt->start_lba = 0;  in blk_trace_setup_lba()
490 		bt->end_lba = -1ULL;  in blk_trace_setup_lba()
501 	struct blk_trace *bt = NULL;  in do_blk_trace_setup()  local
529 	bt = kzalloc(sizeof(*bt), GFP_KERNEL);  in do_blk_trace_setup()
530 	if (!bt)  in do_blk_trace_setup()
534 	bt->sequence = alloc_percpu(unsigned long);  in do_blk_trace_setup()
535 	if (!bt->sequence)  in do_blk_trace_setup()
538 	bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));  in do_blk_trace_setup()
539 	if (!bt->msg_data)  in do_blk_trace_setup()
551 		bt->dir = dir = debugfs_create_dir(buts->name, blk_debugfs_root);  in do_blk_trace_setup()
565 	bt->dev = dev;  in do_blk_trace_setup()
566 	INIT_LIST_HEAD(&bt->running_list);  in do_blk_trace_setup()
569 	debugfs_create_file("dropped", 0444, dir, bt, &blk_dropped_fops);  in do_blk_trace_setup()
570 	debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops);  in do_blk_trace_setup()
572 	bt->rchan = relay_open("trace", dir, buts->buf_size,  in do_blk_trace_setup()
573 				buts->buf_nr, &blk_relay_callbacks, bt);  in do_blk_trace_setup()
574 	if (!bt->rchan)  in do_blk_trace_setup()
577 	bt->act_mask = buts->act_mask;  in do_blk_trace_setup()
578 	if (!bt->act_mask)  in do_blk_trace_setup()
579 		bt->act_mask = (u16) -1;  in do_blk_trace_setup()
581 	blk_trace_setup_lba(bt, bdev);  in do_blk_trace_setup()
585 		bt->start_lba = buts->start_lba;  in do_blk_trace_setup()
587 		bt->end_lba = buts->end_lba;  in do_blk_trace_setup()
589 	bt->pid = buts->pid;  in do_blk_trace_setup()
590 	bt->trace_state = Blktrace_setup;  in do_blk_trace_setup()
592 	rcu_assign_pointer(q->blk_trace, bt);  in do_blk_trace_setup()
598 		blk_trace_free(q, bt);  in do_blk_trace_setup()
665 	struct blk_trace *bt;  in __blk_trace_startstop()  local
667 	bt = rcu_dereference_protected(q->blk_trace,  in __blk_trace_startstop()
669 	if (bt == NULL)  in __blk_trace_startstop()
673 		return blk_trace_start(bt);  in __blk_trace_startstop()
675 		return blk_trace_stop(bt);  in __blk_trace_startstop()
752 	struct blk_trace *bt;  in blk_trace_bio_get_cgid()  local
754 	/* We don't use the 'bt' value here except as an optimization... */  in blk_trace_bio_get_cgid()
755 	bt = rcu_dereference_protected(q->blk_trace, 1);  in blk_trace_bio_get_cgid()
756 	if (!bt || !(blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP))  in blk_trace_bio_get_cgid()
799 	struct blk_trace *bt;  in blk_add_trace_rq()  local
802 	bt = rcu_dereference(rq->q->blk_trace);  in blk_add_trace_rq()
803 	if (likely(!bt)) {  in blk_add_trace_rq()
813 	__blk_add_trace(bt, blk_rq_trace_sector(rq), nr_bytes, rq->cmd_flags,  in blk_add_trace_rq()
863 	struct blk_trace *bt;  in blk_add_trace_bio()  local
866 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_bio()
867 	if (likely(!bt)) {  in blk_add_trace_bio()
872 	__blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,  in blk_add_trace_bio()
909 	struct blk_trace *bt;  in blk_add_trace_plug()  local
912 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_plug()
913 	if (bt)  in blk_add_trace_plug()
914 		__blk_add_trace(bt, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL, 0);  in blk_add_trace_plug()
921 	struct blk_trace *bt;  in blk_add_trace_unplug()  local
924 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_unplug()
925 	if (bt) {  in blk_add_trace_unplug()
934 		__blk_add_trace(bt, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu, 0);  in blk_add_trace_unplug()
942 	struct blk_trace *bt;  in blk_add_trace_split()  local
945 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_split()
946 	if (bt) {  in blk_add_trace_split()
949 		__blk_add_trace(bt, bio->bi_iter.bi_sector,  in blk_add_trace_split()
971 	struct blk_trace *bt;  in blk_add_trace_bio_remap()  local
975 	bt = rcu_dereference(q->blk_trace);  in blk_add_trace_bio_remap()
976 	if (likely(!bt)) {  in blk_add_trace_bio_remap()
985 	__blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,  in blk_add_trace_bio_remap()
1007 	struct blk_trace *bt;  in blk_add_trace_rq_remap()  local
1011 	bt = rcu_dereference(rq->q->blk_trace);  in blk_add_trace_rq_remap()
1012 	if (likely(!bt)) {  in blk_add_trace_rq_remap()
1021 	__blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq),  in blk_add_trace_rq_remap()
1039 	struct blk_trace *bt;  in blk_add_driver_data()  local
1042 	bt = rcu_dereference(rq->q->blk_trace);  in blk_add_driver_data()
1043 	if (likely(!bt)) {  in blk_add_driver_data()
1048 	__blk_add_trace(bt, blk_rq_trace_sector(rq), blk_rq_bytes(rq), 0,  in blk_add_driver_data()
1569 	struct blk_trace *bt;  in blk_trace_remove_queue()  local
1571 	bt = rcu_replace_pointer(q->blk_trace, NULL,  in blk_trace_remove_queue()
1573 	if (bt == NULL)  in blk_trace_remove_queue()
1576 	blk_trace_stop(bt);  in blk_trace_remove_queue()
1580 	blk_trace_free(q, bt);  in blk_trace_remove_queue()
1590 	struct blk_trace *bt = NULL;  in blk_trace_setup_queue()  local
1593 	bt = kzalloc(sizeof(*bt), GFP_KERNEL);  in blk_trace_setup_queue()
1594 	if (!bt)  in blk_trace_setup_queue()
1597 	bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char));  in blk_trace_setup_queue()
1598 	if (!bt->msg_data)  in blk_trace_setup_queue()
1601 	bt->dev = bdev->bd_dev;  in blk_trace_setup_queue()
1602 	bt->act_mask = (u16)-1;  in blk_trace_setup_queue()
1604 	blk_trace_setup_lba(bt, bdev);  in blk_trace_setup_queue()
1606 	rcu_assign_pointer(q->blk_trace, bt);  in blk_trace_setup_queue()
1611 	blk_trace_free(q, bt);  in blk_trace_setup_queue()
1729 	struct blk_trace *bt;  in sysfs_blk_trace_attr_show()  local
1734 	bt = rcu_dereference_protected(q->blk_trace,  in sysfs_blk_trace_attr_show()
1737 		ret = sprintf(buf, "%u\n", !!bt);  in sysfs_blk_trace_attr_show()
1741 	if (bt == NULL)  in sysfs_blk_trace_attr_show()
1744 		ret = blk_trace_mask2str(buf, bt->act_mask);  in sysfs_blk_trace_attr_show()
1746 		ret = sprintf(buf, "%u\n", bt->pid);  in sysfs_blk_trace_attr_show()
1748 		ret = sprintf(buf, "%llu\n", bt->start_lba);  in sysfs_blk_trace_attr_show()
1750 		ret = sprintf(buf, "%llu\n", bt->end_lba);  in sysfs_blk_trace_attr_show()
1763 	struct blk_trace *bt;  in sysfs_blk_trace_attr_store()  local
1785 	bt = rcu_dereference_protected(q->blk_trace,  in sysfs_blk_trace_attr_store()
1788 		if (!!value == !!bt) {  in sysfs_blk_trace_attr_store()
1800 	if (bt == NULL) {  in sysfs_blk_trace_attr_store()
1802 		bt = rcu_dereference_protected(q->blk_trace,  in sysfs_blk_trace_attr_store()
1808 			bt->act_mask = value;  in sysfs_blk_trace_attr_store()
1810 			bt->pid = value;  in sysfs_blk_trace_attr_store()
1812 			bt->start_lba = value;  in sysfs_blk_trace_attr_store()
1814 			bt->end_lba = value;  in sysfs_blk_trace_attr_store()