1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <linux/errno.h> 3 #include <linux/numa.h> 4 #include <linux/slab.h> 5 #include <linux/rculist.h> 6 #include <linux/threads.h> 7 #include <linux/preempt.h> 8 #include <linux/irqflags.h> 9 #include <linux/vmalloc.h> 10 #include <linux/mm.h> 11 #include <linux/module.h> 12 #include <linux/device-mapper.h> 13 14 #include "dm-core.h" 15 #include "dm-stats.h" 16 17 #define DM_MSG_PREFIX "stats" 18 19 static int dm_stat_need_rcu_barrier; 20 21 /* 22 * Using 64-bit values to avoid overflow (which is a 23 * problem that block/genhd.c's IO accounting has). 24 */ 25 struct dm_stat_percpu { 26 unsigned long long sectors[2]; 27 unsigned long long ios[2]; 28 unsigned long long merges[2]; 29 unsigned long long ticks[2]; 30 unsigned long long io_ticks[2]; 31 unsigned long long io_ticks_total; 32 unsigned long long time_in_queue; 33 unsigned long long *histogram; 34 }; 35 36 struct dm_stat_shared { 37 atomic_t in_flight[2]; 38 unsigned long long stamp; 39 struct dm_stat_percpu tmp; 40 }; 41 42 struct dm_stat { 43 struct list_head list_entry; 44 int id; 45 unsigned int stat_flags; 46 size_t n_entries; 47 sector_t start; 48 sector_t end; 49 sector_t step; 50 unsigned int n_histogram_entries; 51 unsigned long long *histogram_boundaries; 52 const char *program_id; 53 const char *aux_data; 54 struct rcu_head rcu_head; 55 size_t shared_alloc_size; 56 size_t percpu_alloc_size; 57 size_t histogram_alloc_size; 58 struct dm_stat_percpu *stat_percpu[NR_CPUS]; 59 struct dm_stat_shared stat_shared[] __counted_by(n_entries); 60 }; 61 62 #define STAT_PRECISE_TIMESTAMPS 1 63 64 struct dm_stats_last_position { 65 sector_t last_sector; 66 unsigned int last_rw; 67 }; 68 69 #define DM_STAT_MAX_ENTRIES 8388608 70 #define DM_STAT_MAX_HISTOGRAM_ENTRIES 134217728 71 72 /* 73 * A typo on the command line could possibly make the kernel run out of memory 74 * and crash. To prevent the crash we account all used memory. We fail if we 75 * exhaust 1/4 of all memory or 1/2 of vmalloc space. 76 */ 77 #define DM_STATS_MEMORY_FACTOR 4 78 #define DM_STATS_VMALLOC_FACTOR 2 79 80 static DEFINE_SPINLOCK(shared_memory_lock); 81 82 static unsigned long shared_memory_amount; 83 84 static bool __check_shared_memory(size_t alloc_size) 85 { 86 size_t a; 87 88 a = shared_memory_amount + alloc_size; 89 if (a < shared_memory_amount) 90 return false; 91 if (a >> PAGE_SHIFT > totalram_pages() / DM_STATS_MEMORY_FACTOR) 92 return false; 93 #ifdef CONFIG_MMU 94 if (a > (VMALLOC_END - VMALLOC_START) / DM_STATS_VMALLOC_FACTOR) 95 return false; 96 #endif 97 return true; 98 } 99 100 static bool check_shared_memory(size_t alloc_size) 101 { 102 bool ret; 103 104 spin_lock_irq(&shared_memory_lock); 105 106 ret = __check_shared_memory(alloc_size); 107 108 spin_unlock_irq(&shared_memory_lock); 109 110 return ret; 111 } 112 113 static bool claim_shared_memory(size_t alloc_size) 114 { 115 spin_lock_irq(&shared_memory_lock); 116 117 if (!__check_shared_memory(alloc_size)) { 118 spin_unlock_irq(&shared_memory_lock); 119 return false; 120 } 121 122 shared_memory_amount += alloc_size; 123 124 spin_unlock_irq(&shared_memory_lock); 125 126 return true; 127 } 128 129 static void free_shared_memory(size_t alloc_size) 130 { 131 unsigned long flags; 132 133 spin_lock_irqsave(&shared_memory_lock, flags); 134 135 if (WARN_ON_ONCE(shared_memory_amount < alloc_size)) { 136 spin_unlock_irqrestore(&shared_memory_lock, flags); 137 DMCRIT("Memory usage accounting bug."); 138 return; 139 } 140 141 shared_memory_amount -= alloc_size; 142 143 spin_unlock_irqrestore(&shared_memory_lock, flags); 144 } 145 146 static void *dm_kvzalloc(size_t alloc_size, int node) 147 { 148 void *p; 149 150 if (!claim_shared_memory(alloc_size)) 151 return NULL; 152 153 p = kvzalloc_node(alloc_size, GFP_KERNEL | __GFP_NOMEMALLOC, node); 154 if (p) 155 return p; 156 157 free_shared_memory(alloc_size); 158 159 return NULL; 160 } 161 162 static void dm_kvfree(void *ptr, size_t alloc_size) 163 { 164 if (!ptr) 165 return; 166 167 free_shared_memory(alloc_size); 168 169 kvfree(ptr); 170 } 171 172 static void dm_stat_free(struct rcu_head *head) 173 { 174 int cpu; 175 struct dm_stat *s = container_of(head, struct dm_stat, rcu_head); 176 177 kfree(s->histogram_boundaries); 178 kfree(s->program_id); 179 kfree(s->aux_data); 180 for_each_possible_cpu(cpu) { 181 if (s->stat_percpu[cpu]) { 182 dm_kvfree(s->stat_percpu[cpu][0].histogram, s->histogram_alloc_size); 183 dm_kvfree(s->stat_percpu[cpu], s->percpu_alloc_size); 184 } 185 } 186 dm_kvfree(s->stat_shared[0].tmp.histogram, s->histogram_alloc_size); 187 dm_kvfree(s, s->shared_alloc_size); 188 } 189 190 static int dm_stat_in_flight(struct dm_stat_shared *shared) 191 { 192 return atomic_read(&shared->in_flight[READ]) + 193 atomic_read(&shared->in_flight[WRITE]); 194 } 195 196 int dm_stats_init(struct dm_stats *stats) 197 { 198 int cpu; 199 struct dm_stats_last_position *last; 200 201 mutex_init(&stats->mutex); 202 INIT_LIST_HEAD(&stats->list); 203 stats->precise_timestamps = false; 204 stats->last = alloc_percpu(struct dm_stats_last_position); 205 if (!stats->last) 206 return -ENOMEM; 207 208 for_each_possible_cpu(cpu) { 209 last = per_cpu_ptr(stats->last, cpu); 210 last->last_sector = (sector_t)ULLONG_MAX; 211 last->last_rw = UINT_MAX; 212 } 213 214 return 0; 215 } 216 217 void dm_stats_cleanup(struct dm_stats *stats) 218 { 219 size_t ni; 220 struct dm_stat *s; 221 struct dm_stat_shared *shared; 222 223 while (!list_empty(&stats->list)) { 224 s = container_of(stats->list.next, struct dm_stat, list_entry); 225 list_del(&s->list_entry); 226 for (ni = 0; ni < s->n_entries; ni++) { 227 shared = &s->stat_shared[ni]; 228 if (WARN_ON(dm_stat_in_flight(shared))) { 229 DMCRIT("leaked in-flight counter at index %lu " 230 "(start %llu, end %llu, step %llu): reads %d, writes %d", 231 (unsigned long)ni, 232 (unsigned long long)s->start, 233 (unsigned long long)s->end, 234 (unsigned long long)s->step, 235 atomic_read(&shared->in_flight[READ]), 236 atomic_read(&shared->in_flight[WRITE])); 237 } 238 cond_resched(); 239 } 240 dm_stat_free(&s->rcu_head); 241 } 242 free_percpu(stats->last); 243 mutex_destroy(&stats->mutex); 244 } 245 246 static void dm_stats_recalc_precise_timestamps(struct dm_stats *stats) 247 { 248 struct list_head *l; 249 struct dm_stat *tmp_s; 250 bool precise_timestamps = false; 251 252 list_for_each(l, &stats->list) { 253 tmp_s = container_of(l, struct dm_stat, list_entry); 254 if (tmp_s->stat_flags & STAT_PRECISE_TIMESTAMPS) { 255 precise_timestamps = true; 256 break; 257 } 258 } 259 stats->precise_timestamps = precise_timestamps; 260 } 261 262 static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end, 263 sector_t step, unsigned int stat_flags, 264 unsigned int n_histogram_entries, 265 unsigned long long *histogram_boundaries, 266 const char *program_id, const char *aux_data, 267 void (*suspend_callback)(struct mapped_device *), 268 void (*resume_callback)(struct mapped_device *), 269 struct mapped_device *md) 270 { 271 struct list_head *l; 272 struct dm_stat *s, *tmp_s; 273 sector_t n_entries; 274 size_t ni; 275 size_t shared_alloc_size; 276 size_t percpu_alloc_size; 277 size_t histogram_alloc_size; 278 struct dm_stat_percpu *p; 279 int cpu; 280 int ret_id; 281 int r; 282 283 if (end < start || !step) 284 return -EINVAL; 285 286 n_entries = end - start; 287 if (dm_sector_div64(n_entries, step)) 288 n_entries++; 289 290 if (n_entries != (size_t)n_entries || !(size_t)(n_entries + 1)) 291 return -EOVERFLOW; 292 293 if (n_entries > DM_STAT_MAX_ENTRIES) 294 return -EOVERFLOW; 295 296 shared_alloc_size = struct_size(s, stat_shared, n_entries); 297 if ((shared_alloc_size - sizeof(struct dm_stat)) / sizeof(struct dm_stat_shared) != n_entries) 298 return -EOVERFLOW; 299 300 percpu_alloc_size = (size_t)n_entries * sizeof(struct dm_stat_percpu); 301 if (percpu_alloc_size / sizeof(struct dm_stat_percpu) != n_entries) 302 return -EOVERFLOW; 303 304 histogram_alloc_size = (n_histogram_entries + 1) * (size_t)n_entries * sizeof(unsigned long long); 305 if (histogram_alloc_size / (n_histogram_entries + 1) != (size_t)n_entries * sizeof(unsigned long long)) 306 return -EOVERFLOW; 307 308 if ((n_histogram_entries + 1) * (size_t)n_entries > DM_STAT_MAX_HISTOGRAM_ENTRIES) 309 return -EOVERFLOW; 310 311 if (!check_shared_memory(shared_alloc_size + histogram_alloc_size + 312 num_possible_cpus() * (percpu_alloc_size + histogram_alloc_size))) 313 return -ENOMEM; 314 315 s = dm_kvzalloc(shared_alloc_size, NUMA_NO_NODE); 316 if (!s) 317 return -ENOMEM; 318 319 s->stat_flags = stat_flags; 320 s->n_entries = n_entries; 321 s->start = start; 322 s->end = end; 323 s->step = step; 324 s->shared_alloc_size = shared_alloc_size; 325 s->percpu_alloc_size = percpu_alloc_size; 326 s->histogram_alloc_size = histogram_alloc_size; 327 328 s->n_histogram_entries = n_histogram_entries; 329 s->histogram_boundaries = kmemdup(histogram_boundaries, 330 s->n_histogram_entries * sizeof(unsigned long long), GFP_KERNEL); 331 if (!s->histogram_boundaries) { 332 r = -ENOMEM; 333 goto out; 334 } 335 336 s->program_id = kstrdup(program_id, GFP_KERNEL); 337 if (!s->program_id) { 338 r = -ENOMEM; 339 goto out; 340 } 341 s->aux_data = kstrdup(aux_data, GFP_KERNEL); 342 if (!s->aux_data) { 343 r = -ENOMEM; 344 goto out; 345 } 346 347 for (ni = 0; ni < n_entries; ni++) { 348 atomic_set(&s->stat_shared[ni].in_flight[READ], 0); 349 atomic_set(&s->stat_shared[ni].in_flight[WRITE], 0); 350 cond_resched(); 351 } 352 353 if (s->n_histogram_entries) { 354 unsigned long long *hi; 355 356 hi = dm_kvzalloc(s->histogram_alloc_size, NUMA_NO_NODE); 357 if (!hi) { 358 r = -ENOMEM; 359 goto out; 360 } 361 for (ni = 0; ni < n_entries; ni++) { 362 s->stat_shared[ni].tmp.histogram = hi; 363 hi += s->n_histogram_entries + 1; 364 cond_resched(); 365 } 366 } 367 368 for_each_possible_cpu(cpu) { 369 p = dm_kvzalloc(percpu_alloc_size, cpu_to_node(cpu)); 370 if (!p) { 371 r = -ENOMEM; 372 goto out; 373 } 374 s->stat_percpu[cpu] = p; 375 if (s->n_histogram_entries) { 376 unsigned long long *hi; 377 378 hi = dm_kvzalloc(s->histogram_alloc_size, cpu_to_node(cpu)); 379 if (!hi) { 380 r = -ENOMEM; 381 goto out; 382 } 383 for (ni = 0; ni < n_entries; ni++) { 384 p[ni].histogram = hi; 385 hi += s->n_histogram_entries + 1; 386 cond_resched(); 387 } 388 } 389 } 390 391 /* 392 * Suspend/resume to make sure there is no i/o in flight, 393 * so that newly created statistics will be exact. 394 * 395 * (note: we couldn't suspend earlier because we must not 396 * allocate memory while suspended) 397 */ 398 suspend_callback(md); 399 400 mutex_lock(&stats->mutex); 401 s->id = 0; 402 list_for_each(l, &stats->list) { 403 tmp_s = container_of(l, struct dm_stat, list_entry); 404 if (WARN_ON(tmp_s->id < s->id)) { 405 r = -EINVAL; 406 goto out_unlock_resume; 407 } 408 if (tmp_s->id > s->id) 409 break; 410 if (unlikely(s->id == INT_MAX)) { 411 r = -ENFILE; 412 goto out_unlock_resume; 413 } 414 s->id++; 415 } 416 ret_id = s->id; 417 list_add_tail_rcu(&s->list_entry, l); 418 419 dm_stats_recalc_precise_timestamps(stats); 420 421 if (!static_key_enabled(&stats_enabled.key)) 422 static_branch_enable(&stats_enabled); 423 424 mutex_unlock(&stats->mutex); 425 426 resume_callback(md); 427 428 return ret_id; 429 430 out_unlock_resume: 431 mutex_unlock(&stats->mutex); 432 resume_callback(md); 433 out: 434 dm_stat_free(&s->rcu_head); 435 return r; 436 } 437 438 static struct dm_stat *__dm_stats_find(struct dm_stats *stats, int id) 439 { 440 struct dm_stat *s; 441 442 list_for_each_entry(s, &stats->list, list_entry) { 443 if (s->id > id) 444 break; 445 if (s->id == id) 446 return s; 447 } 448 449 return NULL; 450 } 451 452 static int dm_stats_delete(struct dm_stats *stats, int id) 453 { 454 struct dm_stat *s; 455 int cpu; 456 457 mutex_lock(&stats->mutex); 458 459 s = __dm_stats_find(stats, id); 460 if (!s) { 461 mutex_unlock(&stats->mutex); 462 return -ENOENT; 463 } 464 465 list_del_rcu(&s->list_entry); 466 467 dm_stats_recalc_precise_timestamps(stats); 468 469 mutex_unlock(&stats->mutex); 470 471 /* 472 * vfree can't be called from RCU callback 473 */ 474 for_each_possible_cpu(cpu) 475 if (is_vmalloc_addr(s->stat_percpu) || 476 is_vmalloc_addr(s->stat_percpu[cpu][0].histogram)) 477 goto do_sync_free; 478 if (is_vmalloc_addr(s) || 479 is_vmalloc_addr(s->stat_shared[0].tmp.histogram)) { 480 do_sync_free: 481 synchronize_rcu_expedited(); 482 dm_stat_free(&s->rcu_head); 483 } else { 484 WRITE_ONCE(dm_stat_need_rcu_barrier, 1); 485 call_rcu(&s->rcu_head, dm_stat_free); 486 } 487 return 0; 488 } 489 490 static int dm_stats_list(struct dm_stats *stats, const char *program, 491 char *result, unsigned int maxlen) 492 { 493 struct dm_stat *s; 494 sector_t len; 495 unsigned int sz = 0; 496 497 /* 498 * Output format: 499 * <region_id>: <start_sector>+<length> <step> <program_id> <aux_data> 500 */ 501 502 mutex_lock(&stats->mutex); 503 list_for_each_entry(s, &stats->list, list_entry) { 504 if (!program || !strcmp(program, s->program_id)) { 505 len = s->end - s->start; 506 DMEMIT("%d: %llu+%llu %llu %s %s", s->id, 507 (unsigned long long)s->start, 508 (unsigned long long)len, 509 (unsigned long long)s->step, 510 s->program_id, 511 s->aux_data); 512 if (s->stat_flags & STAT_PRECISE_TIMESTAMPS) 513 DMEMIT(" precise_timestamps"); 514 if (s->n_histogram_entries) { 515 unsigned int i; 516 517 DMEMIT(" histogram:"); 518 for (i = 0; i < s->n_histogram_entries; i++) { 519 if (i) 520 DMEMIT(","); 521 DMEMIT("%llu", s->histogram_boundaries[i]); 522 } 523 } 524 DMEMIT("\n"); 525 } 526 cond_resched(); 527 } 528 mutex_unlock(&stats->mutex); 529 530 return 1; 531 } 532 533 static void dm_stat_round(struct dm_stat *s, struct dm_stat_shared *shared, 534 struct dm_stat_percpu *p) 535 { 536 /* 537 * This is racy, but so is part_round_stats_single. 538 */ 539 unsigned long long now, difference; 540 unsigned int in_flight_read, in_flight_write; 541 542 if (likely(!(s->stat_flags & STAT_PRECISE_TIMESTAMPS))) 543 now = jiffies; 544 else 545 now = ktime_to_ns(ktime_get()); 546 547 difference = now - shared->stamp; 548 if (!difference) 549 return; 550 551 in_flight_read = (unsigned int)atomic_read(&shared->in_flight[READ]); 552 in_flight_write = (unsigned int)atomic_read(&shared->in_flight[WRITE]); 553 if (in_flight_read) 554 p->io_ticks[READ] += difference; 555 if (in_flight_write) 556 p->io_ticks[WRITE] += difference; 557 if (in_flight_read + in_flight_write) { 558 p->io_ticks_total += difference; 559 p->time_in_queue += (in_flight_read + in_flight_write) * difference; 560 } 561 shared->stamp = now; 562 } 563 564 static void dm_stat_for_entry(struct dm_stat *s, size_t entry, 565 int idx, sector_t len, 566 struct dm_stats_aux *stats_aux, bool end, 567 unsigned long duration_jiffies) 568 { 569 struct dm_stat_shared *shared = &s->stat_shared[entry]; 570 struct dm_stat_percpu *p; 571 572 /* 573 * For strict correctness we should use local_irq_save/restore 574 * instead of preempt_disable/enable. 575 * 576 * preempt_disable/enable is racy if the driver finishes bios 577 * from non-interrupt context as well as from interrupt context 578 * or from more different interrupts. 579 * 580 * On 64-bit architectures the race only results in not counting some 581 * events, so it is acceptable. On 32-bit architectures the race could 582 * cause the counter going off by 2^32, so we need to do proper locking 583 * there. 584 * 585 * part_stat_lock()/part_stat_unlock() have this race too. 586 */ 587 #if BITS_PER_LONG == 32 588 unsigned long flags; 589 590 local_irq_save(flags); 591 #else 592 preempt_disable(); 593 #endif 594 p = &s->stat_percpu[smp_processor_id()][entry]; 595 596 if (!end) { 597 dm_stat_round(s, shared, p); 598 atomic_inc(&shared->in_flight[idx]); 599 } else { 600 unsigned long long duration; 601 602 dm_stat_round(s, shared, p); 603 atomic_dec(&shared->in_flight[idx]); 604 p->sectors[idx] += len; 605 p->ios[idx] += 1; 606 p->merges[idx] += stats_aux->merged; 607 if (!(s->stat_flags & STAT_PRECISE_TIMESTAMPS)) { 608 p->ticks[idx] += duration_jiffies; 609 duration = jiffies_to_msecs(duration_jiffies); 610 } else { 611 p->ticks[idx] += stats_aux->duration_ns; 612 duration = stats_aux->duration_ns; 613 } 614 if (s->n_histogram_entries) { 615 unsigned int lo = 0, hi = s->n_histogram_entries + 1; 616 617 while (lo + 1 < hi) { 618 unsigned int mid = (lo + hi) / 2; 619 620 if (s->histogram_boundaries[mid - 1] > duration) 621 hi = mid; 622 else 623 lo = mid; 624 } 625 p->histogram[lo]++; 626 } 627 } 628 629 #if BITS_PER_LONG == 32 630 local_irq_restore(flags); 631 #else 632 preempt_enable(); 633 #endif 634 } 635 636 static void __dm_stat_bio(struct dm_stat *s, int bi_rw, 637 sector_t bi_sector, sector_t end_sector, 638 bool end, unsigned long duration_jiffies, 639 struct dm_stats_aux *stats_aux) 640 { 641 sector_t rel_sector, offset, todo, fragment_len; 642 size_t entry; 643 644 if (end_sector <= s->start || bi_sector >= s->end) 645 return; 646 if (unlikely(bi_sector < s->start)) { 647 rel_sector = 0; 648 todo = end_sector - s->start; 649 } else { 650 rel_sector = bi_sector - s->start; 651 todo = end_sector - bi_sector; 652 } 653 if (unlikely(end_sector > s->end)) 654 todo -= (end_sector - s->end); 655 656 offset = dm_sector_div64(rel_sector, s->step); 657 entry = rel_sector; 658 do { 659 if (WARN_ON_ONCE(entry >= s->n_entries)) { 660 DMCRIT("Invalid area access in region id %d", s->id); 661 return; 662 } 663 fragment_len = todo; 664 if (fragment_len > s->step - offset) 665 fragment_len = s->step - offset; 666 dm_stat_for_entry(s, entry, bi_rw, fragment_len, 667 stats_aux, end, duration_jiffies); 668 todo -= fragment_len; 669 entry++; 670 offset = 0; 671 } while (unlikely(todo != 0)); 672 } 673 674 void dm_stats_account_io(struct dm_stats *stats, unsigned long bi_rw, 675 sector_t bi_sector, unsigned int bi_sectors, bool end, 676 unsigned long start_time, 677 struct dm_stats_aux *stats_aux) 678 { 679 struct dm_stat *s; 680 sector_t end_sector; 681 struct dm_stats_last_position *last; 682 bool got_precise_time; 683 unsigned long duration_jiffies = 0; 684 685 if (unlikely(!bi_sectors)) 686 return; 687 688 end_sector = bi_sector + bi_sectors; 689 690 if (!end) { 691 /* 692 * A race condition can at worst result in the merged flag being 693 * misrepresented, so we don't have to disable preemption here. 694 */ 695 last = raw_cpu_ptr(stats->last); 696 stats_aux->merged = 697 bi_sector == READ_ONCE(last->last_sector) && 698 (bi_rw == WRITE) == (READ_ONCE(last->last_rw) == WRITE); 699 WRITE_ONCE(last->last_sector, end_sector); 700 WRITE_ONCE(last->last_rw, bi_rw); 701 } else 702 duration_jiffies = jiffies - start_time; 703 704 rcu_read_lock(); 705 706 got_precise_time = false; 707 list_for_each_entry_rcu(s, &stats->list, list_entry) { 708 if (s->stat_flags & STAT_PRECISE_TIMESTAMPS && !got_precise_time) { 709 /* start (!end) duration_ns is set by DM core's alloc_io() */ 710 if (end) 711 stats_aux->duration_ns = ktime_to_ns(ktime_get()) - stats_aux->duration_ns; 712 got_precise_time = true; 713 } 714 __dm_stat_bio(s, bi_rw, bi_sector, end_sector, end, duration_jiffies, stats_aux); 715 } 716 717 rcu_read_unlock(); 718 } 719 720 static void __dm_stat_init_temporary_percpu_totals(struct dm_stat_shared *shared, 721 struct dm_stat *s, size_t x) 722 { 723 int cpu; 724 struct dm_stat_percpu *p; 725 726 local_irq_disable(); 727 p = &s->stat_percpu[smp_processor_id()][x]; 728 dm_stat_round(s, shared, p); 729 local_irq_enable(); 730 731 shared->tmp.sectors[READ] = 0; 732 shared->tmp.sectors[WRITE] = 0; 733 shared->tmp.ios[READ] = 0; 734 shared->tmp.ios[WRITE] = 0; 735 shared->tmp.merges[READ] = 0; 736 shared->tmp.merges[WRITE] = 0; 737 shared->tmp.ticks[READ] = 0; 738 shared->tmp.ticks[WRITE] = 0; 739 shared->tmp.io_ticks[READ] = 0; 740 shared->tmp.io_ticks[WRITE] = 0; 741 shared->tmp.io_ticks_total = 0; 742 shared->tmp.time_in_queue = 0; 743 744 if (s->n_histogram_entries) 745 memset(shared->tmp.histogram, 0, (s->n_histogram_entries + 1) * sizeof(unsigned long long)); 746 747 for_each_possible_cpu(cpu) { 748 p = &s->stat_percpu[cpu][x]; 749 shared->tmp.sectors[READ] += READ_ONCE(p->sectors[READ]); 750 shared->tmp.sectors[WRITE] += READ_ONCE(p->sectors[WRITE]); 751 shared->tmp.ios[READ] += READ_ONCE(p->ios[READ]); 752 shared->tmp.ios[WRITE] += READ_ONCE(p->ios[WRITE]); 753 shared->tmp.merges[READ] += READ_ONCE(p->merges[READ]); 754 shared->tmp.merges[WRITE] += READ_ONCE(p->merges[WRITE]); 755 shared->tmp.ticks[READ] += READ_ONCE(p->ticks[READ]); 756 shared->tmp.ticks[WRITE] += READ_ONCE(p->ticks[WRITE]); 757 shared->tmp.io_ticks[READ] += READ_ONCE(p->io_ticks[READ]); 758 shared->tmp.io_ticks[WRITE] += READ_ONCE(p->io_ticks[WRITE]); 759 shared->tmp.io_ticks_total += READ_ONCE(p->io_ticks_total); 760 shared->tmp.time_in_queue += READ_ONCE(p->time_in_queue); 761 if (s->n_histogram_entries) { 762 unsigned int i; 763 764 for (i = 0; i < s->n_histogram_entries + 1; i++) 765 shared->tmp.histogram[i] += READ_ONCE(p->histogram[i]); 766 } 767 } 768 } 769 770 static void __dm_stat_clear(struct dm_stat *s, size_t idx_start, size_t idx_end, 771 bool init_tmp_percpu_totals) 772 { 773 size_t x; 774 struct dm_stat_shared *shared; 775 struct dm_stat_percpu *p; 776 777 for (x = idx_start; x < idx_end; x++) { 778 shared = &s->stat_shared[x]; 779 if (init_tmp_percpu_totals) 780 __dm_stat_init_temporary_percpu_totals(shared, s, x); 781 local_irq_disable(); 782 p = &s->stat_percpu[smp_processor_id()][x]; 783 p->sectors[READ] -= shared->tmp.sectors[READ]; 784 p->sectors[WRITE] -= shared->tmp.sectors[WRITE]; 785 p->ios[READ] -= shared->tmp.ios[READ]; 786 p->ios[WRITE] -= shared->tmp.ios[WRITE]; 787 p->merges[READ] -= shared->tmp.merges[READ]; 788 p->merges[WRITE] -= shared->tmp.merges[WRITE]; 789 p->ticks[READ] -= shared->tmp.ticks[READ]; 790 p->ticks[WRITE] -= shared->tmp.ticks[WRITE]; 791 p->io_ticks[READ] -= shared->tmp.io_ticks[READ]; 792 p->io_ticks[WRITE] -= shared->tmp.io_ticks[WRITE]; 793 p->io_ticks_total -= shared->tmp.io_ticks_total; 794 p->time_in_queue -= shared->tmp.time_in_queue; 795 local_irq_enable(); 796 if (s->n_histogram_entries) { 797 unsigned int i; 798 799 for (i = 0; i < s->n_histogram_entries + 1; i++) { 800 local_irq_disable(); 801 p = &s->stat_percpu[smp_processor_id()][x]; 802 p->histogram[i] -= shared->tmp.histogram[i]; 803 local_irq_enable(); 804 } 805 } 806 cond_resched(); 807 } 808 } 809 810 static int dm_stats_clear(struct dm_stats *stats, int id) 811 { 812 struct dm_stat *s; 813 814 mutex_lock(&stats->mutex); 815 816 s = __dm_stats_find(stats, id); 817 if (!s) { 818 mutex_unlock(&stats->mutex); 819 return -ENOENT; 820 } 821 822 __dm_stat_clear(s, 0, s->n_entries, true); 823 824 mutex_unlock(&stats->mutex); 825 826 return 1; 827 } 828 829 /* 830 * This is like jiffies_to_msec, but works for 64-bit values. 831 */ 832 static unsigned long long dm_jiffies_to_msec64(struct dm_stat *s, unsigned long long j) 833 { 834 unsigned long long result; 835 unsigned int mult; 836 837 if (s->stat_flags & STAT_PRECISE_TIMESTAMPS) 838 return j; 839 840 result = 0; 841 if (j) 842 result = jiffies_to_msecs(j & 0x3fffff); 843 if (j >= 1 << 22) { 844 mult = jiffies_to_msecs(1 << 22); 845 result += (unsigned long long)mult * ((j >> 22) & 0x3fffff); 846 } 847 if (j >= 1ULL << 44) 848 result += (unsigned long long)mult * (unsigned long long)(1 << 22) * (j >> 44); 849 850 return result; 851 } 852 853 static int dm_stats_print(struct dm_stats *stats, int id, 854 size_t idx_start, size_t idx_len, 855 bool clear, char *result, unsigned int maxlen) 856 { 857 unsigned int sz = 0; 858 struct dm_stat *s; 859 size_t x; 860 sector_t start, end, step; 861 size_t idx_end; 862 struct dm_stat_shared *shared; 863 864 /* 865 * Output format: 866 * <start_sector>+<length> counters 867 */ 868 869 mutex_lock(&stats->mutex); 870 871 s = __dm_stats_find(stats, id); 872 if (!s) { 873 mutex_unlock(&stats->mutex); 874 return -ENOENT; 875 } 876 877 idx_end = idx_start + idx_len; 878 if (idx_end < idx_start || 879 idx_end > s->n_entries) 880 idx_end = s->n_entries; 881 882 if (idx_start > idx_end) 883 idx_start = idx_end; 884 885 step = s->step; 886 start = s->start + (step * idx_start); 887 888 for (x = idx_start; x < idx_end; x++, start = end) { 889 shared = &s->stat_shared[x]; 890 end = start + step; 891 if (unlikely(end > s->end)) 892 end = s->end; 893 894 __dm_stat_init_temporary_percpu_totals(shared, s, x); 895 896 DMEMIT("%llu+%llu %llu %llu %llu %llu %llu %llu %llu %llu %d %llu %llu %llu %llu", 897 (unsigned long long)start, 898 (unsigned long long)step, 899 shared->tmp.ios[READ], 900 shared->tmp.merges[READ], 901 shared->tmp.sectors[READ], 902 dm_jiffies_to_msec64(s, shared->tmp.ticks[READ]), 903 shared->tmp.ios[WRITE], 904 shared->tmp.merges[WRITE], 905 shared->tmp.sectors[WRITE], 906 dm_jiffies_to_msec64(s, shared->tmp.ticks[WRITE]), 907 dm_stat_in_flight(shared), 908 dm_jiffies_to_msec64(s, shared->tmp.io_ticks_total), 909 dm_jiffies_to_msec64(s, shared->tmp.time_in_queue), 910 dm_jiffies_to_msec64(s, shared->tmp.io_ticks[READ]), 911 dm_jiffies_to_msec64(s, shared->tmp.io_ticks[WRITE])); 912 if (s->n_histogram_entries) { 913 unsigned int i; 914 915 for (i = 0; i < s->n_histogram_entries + 1; i++) 916 DMEMIT("%s%llu", !i ? " " : ":", shared->tmp.histogram[i]); 917 } 918 DMEMIT("\n"); 919 920 if (unlikely(sz + 1 >= maxlen)) 921 goto buffer_overflow; 922 923 cond_resched(); 924 } 925 926 if (clear) 927 __dm_stat_clear(s, idx_start, idx_end, false); 928 929 buffer_overflow: 930 mutex_unlock(&stats->mutex); 931 932 return 1; 933 } 934 935 static int dm_stats_set_aux(struct dm_stats *stats, int id, const char *aux_data) 936 { 937 struct dm_stat *s; 938 const char *new_aux_data; 939 940 mutex_lock(&stats->mutex); 941 942 s = __dm_stats_find(stats, id); 943 if (!s) { 944 mutex_unlock(&stats->mutex); 945 return -ENOENT; 946 } 947 948 new_aux_data = kstrdup(aux_data, GFP_KERNEL); 949 if (!new_aux_data) { 950 mutex_unlock(&stats->mutex); 951 return -ENOMEM; 952 } 953 954 kfree(s->aux_data); 955 s->aux_data = new_aux_data; 956 957 mutex_unlock(&stats->mutex); 958 959 return 0; 960 } 961 962 static int parse_histogram(const char *h, unsigned int *n_histogram_entries, 963 unsigned long long **histogram_boundaries) 964 { 965 const char *q; 966 unsigned int n; 967 unsigned long long last; 968 969 *n_histogram_entries = 1; 970 for (q = h; *q; q++) 971 if (*q == ',') 972 (*n_histogram_entries)++; 973 974 *histogram_boundaries = kmalloc_objs(unsigned long long, 975 *n_histogram_entries); 976 if (!*histogram_boundaries) 977 return -ENOMEM; 978 979 n = 0; 980 last = 0; 981 while (1) { 982 unsigned long long hi; 983 int s; 984 char ch; 985 986 s = sscanf(h, "%llu%c", &hi, &ch); 987 if (!s || (s == 2 && ch != ',')) 988 return -EINVAL; 989 if (hi <= last) 990 return -EINVAL; 991 last = hi; 992 (*histogram_boundaries)[n] = hi; 993 if (s == 1) 994 return 0; 995 h = strchr(h, ',') + 1; 996 n++; 997 } 998 } 999 1000 static int message_stats_create(struct mapped_device *md, 1001 unsigned int argc, char **argv, 1002 char *result, unsigned int maxlen) 1003 { 1004 int r; 1005 int id; 1006 char dummy; 1007 unsigned long long start, end, len, step; 1008 unsigned int divisor; 1009 const char *program_id, *aux_data; 1010 unsigned int stat_flags = 0; 1011 unsigned int n_histogram_entries = 0; 1012 unsigned long long *histogram_boundaries = NULL; 1013 struct dm_arg_set as, as_backup; 1014 const char *a; 1015 unsigned int feature_args; 1016 1017 /* 1018 * Input format: 1019 * <range> <step> [<extra_parameters> <parameters>] [<program_id> [<aux_data>]] 1020 */ 1021 1022 if (argc < 3) 1023 goto ret_einval; 1024 1025 as.argc = argc; 1026 as.argv = argv; 1027 dm_consume_args(&as, 1); 1028 1029 a = dm_shift_arg(&as); 1030 if (!strcmp(a, "-")) { 1031 start = 0; 1032 len = dm_get_size(md); 1033 if (!len) 1034 len = 1; 1035 } else if (sscanf(a, "%llu+%llu%c", &start, &len, &dummy) != 2 || 1036 start != (sector_t)start || len != (sector_t)len) 1037 goto ret_einval; 1038 1039 end = start + len; 1040 if (start >= end) 1041 goto ret_einval; 1042 1043 a = dm_shift_arg(&as); 1044 if (sscanf(a, "/%u%c", &divisor, &dummy) == 1) { 1045 if (!divisor) 1046 return -EINVAL; 1047 step = end - start; 1048 if (do_div(step, divisor)) 1049 step++; 1050 if (!step) 1051 step = 1; 1052 } else if (sscanf(a, "%llu%c", &step, &dummy) != 1 || 1053 step != (sector_t)step || !step) 1054 goto ret_einval; 1055 1056 as_backup = as; 1057 a = dm_shift_arg(&as); 1058 if (a && sscanf(a, "%u%c", &feature_args, &dummy) == 1) { 1059 while (feature_args--) { 1060 a = dm_shift_arg(&as); 1061 if (!a) 1062 goto ret_einval; 1063 if (!strcasecmp(a, "precise_timestamps")) 1064 stat_flags |= STAT_PRECISE_TIMESTAMPS; 1065 else if (!strncasecmp(a, "histogram:", 10)) { 1066 if (n_histogram_entries) 1067 goto ret_einval; 1068 r = parse_histogram(a + 10, &n_histogram_entries, &histogram_boundaries); 1069 if (r) 1070 goto ret; 1071 } else 1072 goto ret_einval; 1073 } 1074 } else { 1075 as = as_backup; 1076 } 1077 1078 program_id = "-"; 1079 aux_data = "-"; 1080 1081 a = dm_shift_arg(&as); 1082 if (a) 1083 program_id = a; 1084 1085 a = dm_shift_arg(&as); 1086 if (a) 1087 aux_data = a; 1088 1089 if (as.argc) 1090 goto ret_einval; 1091 1092 /* 1093 * If a buffer overflow happens after we created the region, 1094 * it's too late (the userspace would retry with a larger 1095 * buffer, but the region id that caused the overflow is already 1096 * leaked). So we must detect buffer overflow in advance. 1097 */ 1098 snprintf(result, maxlen, "%d", INT_MAX); 1099 if (dm_message_test_buffer_overflow(result, maxlen)) { 1100 r = 1; 1101 goto ret; 1102 } 1103 1104 id = dm_stats_create(dm_get_stats(md), start, end, step, stat_flags, 1105 n_histogram_entries, histogram_boundaries, program_id, aux_data, 1106 dm_internal_suspend_fast, dm_internal_resume_fast, md); 1107 if (id < 0) { 1108 r = id; 1109 goto ret; 1110 } 1111 1112 snprintf(result, maxlen, "%d", id); 1113 1114 r = 1; 1115 goto ret; 1116 1117 ret_einval: 1118 r = -EINVAL; 1119 ret: 1120 kfree(histogram_boundaries); 1121 return r; 1122 } 1123 1124 static int message_stats_delete(struct mapped_device *md, 1125 unsigned int argc, char **argv) 1126 { 1127 int id; 1128 char dummy; 1129 1130 if (argc != 2) 1131 return -EINVAL; 1132 1133 if (sscanf(argv[1], "%d%c", &id, &dummy) != 1 || id < 0) 1134 return -EINVAL; 1135 1136 return dm_stats_delete(dm_get_stats(md), id); 1137 } 1138 1139 static int message_stats_clear(struct mapped_device *md, 1140 unsigned int argc, char **argv) 1141 { 1142 int id; 1143 char dummy; 1144 1145 if (argc != 2) 1146 return -EINVAL; 1147 1148 if (sscanf(argv[1], "%d%c", &id, &dummy) != 1 || id < 0) 1149 return -EINVAL; 1150 1151 return dm_stats_clear(dm_get_stats(md), id); 1152 } 1153 1154 static int message_stats_list(struct mapped_device *md, 1155 unsigned int argc, char **argv, 1156 char *result, unsigned int maxlen) 1157 { 1158 int r; 1159 const char *program = NULL; 1160 1161 if (argc < 1 || argc > 2) 1162 return -EINVAL; 1163 1164 if (argc > 1) { 1165 program = kstrdup(argv[1], GFP_KERNEL); 1166 if (!program) 1167 return -ENOMEM; 1168 } 1169 1170 r = dm_stats_list(dm_get_stats(md), program, result, maxlen); 1171 1172 kfree(program); 1173 1174 return r; 1175 } 1176 1177 static int message_stats_print(struct mapped_device *md, 1178 unsigned int argc, char **argv, bool clear, 1179 char *result, unsigned int maxlen) 1180 { 1181 int id; 1182 char dummy; 1183 unsigned long idx_start = 0, idx_len = ULONG_MAX; 1184 1185 if (argc != 2 && argc != 4) 1186 return -EINVAL; 1187 1188 if (sscanf(argv[1], "%d%c", &id, &dummy) != 1 || id < 0) 1189 return -EINVAL; 1190 1191 if (argc > 3) { 1192 if (strcmp(argv[2], "-") && 1193 sscanf(argv[2], "%lu%c", &idx_start, &dummy) != 1) 1194 return -EINVAL; 1195 if (strcmp(argv[3], "-") && 1196 sscanf(argv[3], "%lu%c", &idx_len, &dummy) != 1) 1197 return -EINVAL; 1198 } 1199 1200 return dm_stats_print(dm_get_stats(md), id, idx_start, idx_len, clear, 1201 result, maxlen); 1202 } 1203 1204 static int message_stats_set_aux(struct mapped_device *md, 1205 unsigned int argc, char **argv) 1206 { 1207 int id; 1208 char dummy; 1209 1210 if (argc != 3) 1211 return -EINVAL; 1212 1213 if (sscanf(argv[1], "%d%c", &id, &dummy) != 1 || id < 0) 1214 return -EINVAL; 1215 1216 return dm_stats_set_aux(dm_get_stats(md), id, argv[2]); 1217 } 1218 1219 int dm_stats_message(struct mapped_device *md, unsigned int argc, char **argv, 1220 char *result, unsigned int maxlen) 1221 { 1222 int r; 1223 1224 /* All messages here must start with '@' */ 1225 if (!strcasecmp(argv[0], "@stats_create")) 1226 r = message_stats_create(md, argc, argv, result, maxlen); 1227 else if (!strcasecmp(argv[0], "@stats_delete")) 1228 r = message_stats_delete(md, argc, argv); 1229 else if (!strcasecmp(argv[0], "@stats_clear")) 1230 r = message_stats_clear(md, argc, argv); 1231 else if (!strcasecmp(argv[0], "@stats_list")) 1232 r = message_stats_list(md, argc, argv, result, maxlen); 1233 else if (!strcasecmp(argv[0], "@stats_print")) 1234 r = message_stats_print(md, argc, argv, false, result, maxlen); 1235 else if (!strcasecmp(argv[0], "@stats_print_clear")) 1236 r = message_stats_print(md, argc, argv, true, result, maxlen); 1237 else if (!strcasecmp(argv[0], "@stats_set_aux")) 1238 r = message_stats_set_aux(md, argc, argv); 1239 else 1240 return 2; /* this wasn't a stats message */ 1241 1242 if (r == -EINVAL) 1243 DMCRIT("Invalid parameters for message %s", argv[0]); 1244 1245 return r; 1246 } 1247 1248 int __init dm_statistics_init(void) 1249 { 1250 shared_memory_amount = 0; 1251 dm_stat_need_rcu_barrier = 0; 1252 return 0; 1253 } 1254 1255 void dm_statistics_exit(void) 1256 { 1257 if (dm_stat_need_rcu_barrier) 1258 rcu_barrier(); 1259 if (WARN_ON(shared_memory_amount)) 1260 DMCRIT("shared_memory_amount leaked: %lu", shared_memory_amount); 1261 } 1262 1263 module_param_named(stats_current_allocated_bytes, shared_memory_amount, ulong, 0444); 1264 MODULE_PARM_DESC(stats_current_allocated_bytes, "Memory currently used by statistics"); 1265