188 static int dm_stat_in_flight(struct dm_stat_shared *shared)  in dm_stat_in_flight()  argument
190 	return atomic_read(&shared->in_flight[READ]) +  in dm_stat_in_flight()
191 	       atomic_read(&shared->in_flight[WRITE]);  in dm_stat_in_flight()
219 	struct dm_stat_shared *shared;  in dm_stats_cleanup()  local
225 			shared = &s->stat_shared[ni];  in dm_stats_cleanup()
226 			if (WARN_ON(dm_stat_in_flight(shared))) {  in dm_stats_cleanup()
233 				       atomic_read(&shared->in_flight[READ]),  in dm_stats_cleanup()
234 				       atomic_read(&shared->in_flight[WRITE]));  in dm_stats_cleanup()
531 static void dm_stat_round(struct dm_stat *s, struct dm_stat_shared *shared,  in dm_stat_round()  argument
545 	difference = now - shared->stamp;  in dm_stat_round()
549 	in_flight_read = (unsigned int)atomic_read(&shared->in_flight[READ]);  in dm_stat_round()
550 	in_flight_write = (unsigned int)atomic_read(&shared->in_flight[WRITE]);  in dm_stat_round()
559 	shared->stamp = now;  in dm_stat_round()
567 	struct dm_stat_shared *shared = &s->stat_shared[entry];  in dm_stat_for_entry()  local
595 		dm_stat_round(s, shared, p);  in dm_stat_for_entry()
596 		atomic_inc(&shared->in_flight[idx]);  in dm_stat_for_entry()
600 		dm_stat_round(s, shared, p);  in dm_stat_for_entry()
601 		atomic_dec(&shared->in_flight[idx]);  in dm_stat_for_entry()
720 static void __dm_stat_init_temporary_percpu_totals(struct dm_stat_shared *shared,  in __dm_stat_init_temporary_percpu_totals()  argument
728 	dm_stat_round(s, shared, p);  in __dm_stat_init_temporary_percpu_totals()
731 	shared->tmp.sectors[READ] = 0;  in __dm_stat_init_temporary_percpu_totals()
732 	shared->tmp.sectors[WRITE] = 0;  in __dm_stat_init_temporary_percpu_totals()
733 	shared->tmp.ios[READ] = 0;  in __dm_stat_init_temporary_percpu_totals()
734 	shared->tmp.ios[WRITE] = 0;  in __dm_stat_init_temporary_percpu_totals()
735 	shared->tmp.merges[READ] = 0;  in __dm_stat_init_temporary_percpu_totals()
736 	shared->tmp.merges[WRITE] = 0;  in __dm_stat_init_temporary_percpu_totals()
737 	shared->tmp.ticks[READ] = 0;  in __dm_stat_init_temporary_percpu_totals()
738 	shared->tmp.ticks[WRITE] = 0;  in __dm_stat_init_temporary_percpu_totals()
739 	shared->tmp.io_ticks[READ] = 0;  in __dm_stat_init_temporary_percpu_totals()
740 	shared->tmp.io_ticks[WRITE] = 0;  in __dm_stat_init_temporary_percpu_totals()
741 	shared->tmp.io_ticks_total = 0;  in __dm_stat_init_temporary_percpu_totals()
742 	shared->tmp.time_in_queue = 0;  in __dm_stat_init_temporary_percpu_totals()
745 		memset(shared->tmp.histogram, 0, (s->n_histogram_entries + 1) * sizeof(unsigned long long));  in __dm_stat_init_temporary_percpu_totals()
749 		shared->tmp.sectors[READ] += READ_ONCE(p->sectors[READ]);  in __dm_stat_init_temporary_percpu_totals()
750 		shared->tmp.sectors[WRITE] += READ_ONCE(p->sectors[WRITE]);  in __dm_stat_init_temporary_percpu_totals()
751 		shared->tmp.ios[READ] += READ_ONCE(p->ios[READ]);  in __dm_stat_init_temporary_percpu_totals()
752 		shared->tmp.ios[WRITE] += READ_ONCE(p->ios[WRITE]);  in __dm_stat_init_temporary_percpu_totals()
753 		shared->tmp.merges[READ] += READ_ONCE(p->merges[READ]);  in __dm_stat_init_temporary_percpu_totals()
754 		shared->tmp.merges[WRITE] += READ_ONCE(p->merges[WRITE]);  in __dm_stat_init_temporary_percpu_totals()
755 		shared->tmp.ticks[READ] += READ_ONCE(p->ticks[READ]);  in __dm_stat_init_temporary_percpu_totals()
756 		shared->tmp.ticks[WRITE] += READ_ONCE(p->ticks[WRITE]);  in __dm_stat_init_temporary_percpu_totals()
757 		shared->tmp.io_ticks[READ] += READ_ONCE(p->io_ticks[READ]);  in __dm_stat_init_temporary_percpu_totals()
758 		shared->tmp.io_ticks[WRITE] += READ_ONCE(p->io_ticks[WRITE]);  in __dm_stat_init_temporary_percpu_totals()
759 		shared->tmp.io_ticks_total += READ_ONCE(p->io_ticks_total);  in __dm_stat_init_temporary_percpu_totals()
760 		shared->tmp.time_in_queue += READ_ONCE(p->time_in_queue);  in __dm_stat_init_temporary_percpu_totals()
765 				shared->tmp.histogram[i] += READ_ONCE(p->histogram[i]);  in __dm_stat_init_temporary_percpu_totals()
774 	struct dm_stat_shared *shared;  in __dm_stat_clear()  local
778 		shared = &s->stat_shared[x];  in __dm_stat_clear()
780 			__dm_stat_init_temporary_percpu_totals(shared, s, x);  in __dm_stat_clear()
783 		p->sectors[READ] -= shared->tmp.sectors[READ];  in __dm_stat_clear()
784 		p->sectors[WRITE] -= shared->tmp.sectors[WRITE];  in __dm_stat_clear()
785 		p->ios[READ] -= shared->tmp.ios[READ];  in __dm_stat_clear()
786 		p->ios[WRITE] -= shared->tmp.ios[WRITE];  in __dm_stat_clear()
787 		p->merges[READ] -= shared->tmp.merges[READ];  in __dm_stat_clear()
788 		p->merges[WRITE] -= shared->tmp.merges[WRITE];  in __dm_stat_clear()
789 		p->ticks[READ] -= shared->tmp.ticks[READ];  in __dm_stat_clear()
790 		p->ticks[WRITE] -= shared->tmp.ticks[WRITE];  in __dm_stat_clear()
791 		p->io_ticks[READ] -= shared->tmp.io_ticks[READ];  in __dm_stat_clear()
792 		p->io_ticks[WRITE] -= shared->tmp.io_ticks[WRITE];  in __dm_stat_clear()
793 		p->io_ticks_total -= shared->tmp.io_ticks_total;  in __dm_stat_clear()
794 		p->time_in_queue -= shared->tmp.time_in_queue;  in __dm_stat_clear()
802 				p->histogram[i] -= shared->tmp.histogram[i];  in __dm_stat_clear()
862 	struct dm_stat_shared *shared;  in dm_stats_print()  local
889 		shared = &s->stat_shared[x];  in dm_stats_print()
894 		__dm_stat_init_temporary_percpu_totals(shared, s, x);  in dm_stats_print()
899 		       shared->tmp.ios[READ],  in dm_stats_print()
900 		       shared->tmp.merges[READ],  in dm_stats_print()
901 		       shared->tmp.sectors[READ],  in dm_stats_print()
902 		       dm_jiffies_to_msec64(s, shared->tmp.ticks[READ]),  in dm_stats_print()
903 		       shared->tmp.ios[WRITE],  in dm_stats_print()
904 		       shared->tmp.merges[WRITE],  in dm_stats_print()
905 		       shared->tmp.sectors[WRITE],  in dm_stats_print()
906 		       dm_jiffies_to_msec64(s, shared->tmp.ticks[WRITE]),  in dm_stats_print()
907 		       dm_stat_in_flight(shared),  in dm_stats_print()
908 		       dm_jiffies_to_msec64(s, shared->tmp.io_ticks_total),  in dm_stats_print()
909 		       dm_jiffies_to_msec64(s, shared->tmp.time_in_queue),  in dm_stats_print()
910 		       dm_jiffies_to_msec64(s, shared->tmp.io_ticks[READ]),  in dm_stats_print()
911 		       dm_jiffies_to_msec64(s, shared->tmp.io_ticks[WRITE]));  in dm_stats_print()
916 				DMEMIT("%s%llu", !i ? " " : ":", shared->tmp.histogram[i]);  in dm_stats_print()