1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 HiSilicon Limited. 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/cleanup.h> 9 #include <linux/debugfs.h> 10 #include <linux/delay.h> 11 #include <linux/device.h> 12 #include <linux/dma-mapping.h> 13 #include <linux/kernel.h> 14 #include <linux/kthread.h> 15 #include <linux/math64.h> 16 #include <linux/module.h> 17 #include <linux/pci.h> 18 #include <linux/platform_device.h> 19 #include <linux/scatterlist.h> 20 #include <linux/slab.h> 21 #include <linux/timekeeping.h> 22 #include <uapi/linux/map_benchmark.h> 23 24 struct map_benchmark_data { 25 struct map_benchmark bparam; 26 struct device *dev; 27 struct dentry *debugfs; 28 enum dma_data_direction dir; 29 atomic64_t sum_map_100ns; 30 atomic64_t sum_unmap_100ns; 31 atomic64_t sum_sq_map; 32 atomic64_t sum_sq_unmap; 33 atomic64_t loops; 34 }; 35 36 struct map_benchmark_ops { 37 void *(*prepare)(struct map_benchmark_data *map); 38 void (*unprepare)(void *mparam); 39 void (*initialize_data)(void *mparam); 40 int (*do_map)(void *mparam); 41 void (*do_unmap)(void *mparam); 42 }; 43 44 struct dma_single_map_param { 45 struct device *dev; 46 dma_addr_t addr; 47 void *xbuf; 48 u32 npages; 49 u32 dma_dir; 50 }; 51 52 static void *dma_single_map_benchmark_prepare(struct map_benchmark_data *map) 53 { 54 struct dma_single_map_param *params __free(kfree) = kzalloc(sizeof(*params), 55 GFP_KERNEL); 56 if (!params) 57 return NULL; 58 59 params->npages = map->bparam.granule; 60 params->dma_dir = map->bparam.dma_dir; 61 params->dev = map->dev; 62 params->xbuf = alloc_pages_exact(params->npages * PAGE_SIZE, GFP_KERNEL); 63 if (!params->xbuf) 64 return NULL; 65 66 return_ptr(params); 67 } 68 69 static void dma_single_map_benchmark_unprepare(void *mparam) 70 { 71 struct dma_single_map_param *params = mparam; 72 73 free_pages_exact(params->xbuf, params->npages * PAGE_SIZE); 74 kfree(params); 75 } 76 77 static void dma_single_map_benchmark_initialize_data(void *mparam) 78 { 79 struct dma_single_map_param *params = mparam; 80 81 /* 82 * for a non-coherent device, if we don't stain them in the 83 * cache, this will give an underestimate of the real-world 84 * overhead of BIDIRECTIONAL or TO_DEVICE mappings; 85 * 66 means everything goes well! 66 is lucky. 86 */ 87 if (params->dma_dir != DMA_FROM_DEVICE) 88 memset(params->xbuf, 0x66, params->npages * PAGE_SIZE); 89 } 90 91 static int dma_single_map_benchmark_do_map(void *mparam) 92 { 93 struct dma_single_map_param *params = mparam; 94 95 params->addr = dma_map_single(params->dev, params->xbuf, 96 params->npages * PAGE_SIZE, params->dma_dir); 97 if (unlikely(dma_mapping_error(params->dev, params->addr))) { 98 pr_err("dma_map_single failed on %s\n", dev_name(params->dev)); 99 return -ENOMEM; 100 } 101 102 return 0; 103 } 104 105 static void dma_single_map_benchmark_do_unmap(void *mparam) 106 { 107 struct dma_single_map_param *params = mparam; 108 109 dma_unmap_single(params->dev, params->addr, 110 params->npages * PAGE_SIZE, params->dma_dir); 111 } 112 113 static struct map_benchmark_ops dma_single_map_benchmark_ops = { 114 .prepare = dma_single_map_benchmark_prepare, 115 .unprepare = dma_single_map_benchmark_unprepare, 116 .initialize_data = dma_single_map_benchmark_initialize_data, 117 .do_map = dma_single_map_benchmark_do_map, 118 .do_unmap = dma_single_map_benchmark_do_unmap, 119 }; 120 121 struct dma_sg_map_param { 122 struct sg_table sgt; 123 struct device *dev; 124 void **buf; 125 u32 npages; 126 u32 dma_dir; 127 }; 128 129 static void *dma_sg_map_benchmark_prepare(struct map_benchmark_data *map) 130 { 131 struct scatterlist *sg; 132 int i; 133 134 struct dma_sg_map_param *params = kzalloc(sizeof(*params), GFP_KERNEL); 135 136 if (!params) 137 return NULL; 138 /* 139 * Set the number of scatterlist entries based on the granule. 140 * In SG mode, 'granule' represents the number of scatterlist entries. 141 * Each scatterlist entry corresponds to a single page. 142 */ 143 params->npages = map->bparam.granule; 144 params->dma_dir = map->bparam.dma_dir; 145 params->dev = map->dev; 146 params->buf = kmalloc_array(params->npages, sizeof(*params->buf), 147 GFP_KERNEL); 148 if (!params->buf) 149 goto out; 150 151 if (sg_alloc_table(¶ms->sgt, params->npages, GFP_KERNEL)) 152 goto free_buf; 153 154 for_each_sgtable_sg(¶ms->sgt, sg, i) { 155 params->buf[i] = (void *)__get_free_page(GFP_KERNEL); 156 if (!params->buf[i]) 157 goto free_page; 158 159 sg_set_buf(sg, params->buf[i], PAGE_SIZE); 160 } 161 162 return params; 163 164 free_page: 165 while (i-- > 0) 166 free_page((unsigned long)params->buf[i]); 167 168 sg_free_table(¶ms->sgt); 169 free_buf: 170 kfree(params->buf); 171 out: 172 kfree(params); 173 return NULL; 174 } 175 176 static void dma_sg_map_benchmark_unprepare(void *mparam) 177 { 178 struct dma_sg_map_param *params = mparam; 179 int i; 180 181 for (i = 0; i < params->npages; i++) 182 free_page((unsigned long)params->buf[i]); 183 184 sg_free_table(¶ms->sgt); 185 186 kfree(params->buf); 187 kfree(params); 188 } 189 190 static void dma_sg_map_benchmark_initialize_data(void *mparam) 191 { 192 struct dma_sg_map_param *params = mparam; 193 struct scatterlist *sg; 194 int i = 0; 195 196 if (params->dma_dir == DMA_FROM_DEVICE) 197 return; 198 199 for_each_sgtable_sg(¶ms->sgt, sg, i) 200 memset(params->buf[i], 0x66, PAGE_SIZE); 201 } 202 203 static int dma_sg_map_benchmark_do_map(void *mparam) 204 { 205 struct dma_sg_map_param *params = mparam; 206 int ret = 0; 207 208 int sg_mapped = dma_map_sg(params->dev, params->sgt.sgl, 209 params->npages, params->dma_dir); 210 if (!sg_mapped) { 211 pr_err("dma_map_sg failed on %s\n", dev_name(params->dev)); 212 ret = -ENOMEM; 213 } 214 215 return ret; 216 } 217 218 static void dma_sg_map_benchmark_do_unmap(void *mparam) 219 { 220 struct dma_sg_map_param *params = mparam; 221 222 dma_unmap_sg(params->dev, params->sgt.sgl, params->npages, 223 params->dma_dir); 224 } 225 226 static struct map_benchmark_ops dma_sg_map_benchmark_ops = { 227 .prepare = dma_sg_map_benchmark_prepare, 228 .unprepare = dma_sg_map_benchmark_unprepare, 229 .initialize_data = dma_sg_map_benchmark_initialize_data, 230 .do_map = dma_sg_map_benchmark_do_map, 231 .do_unmap = dma_sg_map_benchmark_do_unmap, 232 }; 233 234 static struct map_benchmark_ops *dma_map_benchmark_ops[DMA_MAP_BENCH_MODE_MAX] = { 235 [DMA_MAP_BENCH_SINGLE_MODE] = &dma_single_map_benchmark_ops, 236 [DMA_MAP_BENCH_SG_MODE] = &dma_sg_map_benchmark_ops, 237 }; 238 239 static int map_benchmark_thread(void *data) 240 { 241 struct map_benchmark_data *map = data; 242 __u8 map_mode = map->bparam.map_mode; 243 int ret = 0; 244 245 struct map_benchmark_ops *mb_ops = dma_map_benchmark_ops[map_mode]; 246 void *mparam = mb_ops->prepare(map); 247 248 if (!mparam) 249 return -ENOMEM; 250 251 while (!kthread_should_stop()) { 252 u64 map_100ns, unmap_100ns, map_sq, unmap_sq; 253 ktime_t map_stime, map_etime, unmap_stime, unmap_etime; 254 ktime_t map_delta, unmap_delta; 255 256 mb_ops->initialize_data(mparam); 257 map_stime = ktime_get(); 258 ret = mb_ops->do_map(mparam); 259 if (ret) 260 goto out; 261 262 map_etime = ktime_get(); 263 map_delta = ktime_sub(map_etime, map_stime); 264 265 /* Pretend DMA is transmitting */ 266 ndelay(map->bparam.dma_trans_ns); 267 268 unmap_stime = ktime_get(); 269 mb_ops->do_unmap(mparam); 270 271 unmap_etime = ktime_get(); 272 unmap_delta = ktime_sub(unmap_etime, unmap_stime); 273 274 /* calculate sum and sum of squares */ 275 276 map_100ns = div64_ul(map_delta, 100); 277 unmap_100ns = div64_ul(unmap_delta, 100); 278 map_sq = map_100ns * map_100ns; 279 unmap_sq = unmap_100ns * unmap_100ns; 280 281 atomic64_add(map_100ns, &map->sum_map_100ns); 282 atomic64_add(unmap_100ns, &map->sum_unmap_100ns); 283 atomic64_add(map_sq, &map->sum_sq_map); 284 atomic64_add(unmap_sq, &map->sum_sq_unmap); 285 atomic64_inc(&map->loops); 286 287 /* 288 * We may test for a long time so periodically check whether 289 * we need to schedule to avoid starving the others. Otherwise 290 * we may hangup the kernel in a non-preemptible kernel when 291 * the test kthreads number >= CPU number, the test kthreads 292 * will run endless on every CPU since the thread resposible 293 * for notifying the kthread stop (in do_map_benchmark()) 294 * could not be scheduled. 295 * 296 * Note this may degrade the test concurrency since the test 297 * threads may need to share the CPU time with other load 298 * in the system. So it's recommended to run this benchmark 299 * on an idle system. 300 */ 301 cond_resched(); 302 } 303 304 out: 305 mb_ops->unprepare(mparam); 306 return ret; 307 } 308 309 static int do_map_benchmark(struct map_benchmark_data *map) 310 { 311 struct task_struct **tsk; 312 int threads = map->bparam.threads; 313 int node = map->bparam.node; 314 u64 loops; 315 int ret = 0; 316 int i; 317 318 tsk = kmalloc_objs(*tsk, threads); 319 if (!tsk) 320 return -ENOMEM; 321 322 get_device(map->dev); 323 324 for (i = 0; i < threads; i++) { 325 tsk[i] = kthread_create_on_node(map_benchmark_thread, map, 326 map->bparam.node, "dma-map-benchmark/%d", i); 327 if (IS_ERR(tsk[i])) { 328 pr_err("create dma_map thread failed\n"); 329 ret = PTR_ERR(tsk[i]); 330 while (--i >= 0) 331 kthread_stop(tsk[i]); 332 goto out; 333 } 334 335 if (node != NUMA_NO_NODE) 336 kthread_bind_mask(tsk[i], cpumask_of_node(node)); 337 } 338 339 /* clear the old value in the previous benchmark */ 340 atomic64_set(&map->sum_map_100ns, 0); 341 atomic64_set(&map->sum_unmap_100ns, 0); 342 atomic64_set(&map->sum_sq_map, 0); 343 atomic64_set(&map->sum_sq_unmap, 0); 344 atomic64_set(&map->loops, 0); 345 346 for (i = 0; i < threads; i++) { 347 get_task_struct(tsk[i]); 348 wake_up_process(tsk[i]); 349 } 350 351 msleep_interruptible(map->bparam.seconds * 1000); 352 353 /* wait for the completion of all started benchmark threads */ 354 for (i = 0; i < threads; i++) { 355 int kthread_ret = kthread_stop_put(tsk[i]); 356 357 if (kthread_ret) 358 ret = kthread_ret; 359 } 360 361 if (ret) 362 goto out; 363 364 loops = atomic64_read(&map->loops); 365 if (likely(loops > 0)) { 366 u64 map_variance, unmap_variance; 367 u64 sum_map = atomic64_read(&map->sum_map_100ns); 368 u64 sum_unmap = atomic64_read(&map->sum_unmap_100ns); 369 u64 sum_sq_map = atomic64_read(&map->sum_sq_map); 370 u64 sum_sq_unmap = atomic64_read(&map->sum_sq_unmap); 371 372 /* average latency */ 373 map->bparam.avg_map_100ns = div64_u64(sum_map, loops); 374 map->bparam.avg_unmap_100ns = div64_u64(sum_unmap, loops); 375 376 /* standard deviation of latency */ 377 map_variance = div64_u64(sum_sq_map, loops) - 378 map->bparam.avg_map_100ns * 379 map->bparam.avg_map_100ns; 380 unmap_variance = div64_u64(sum_sq_unmap, loops) - 381 map->bparam.avg_unmap_100ns * 382 map->bparam.avg_unmap_100ns; 383 map->bparam.map_stddev = int_sqrt64(map_variance); 384 map->bparam.unmap_stddev = int_sqrt64(unmap_variance); 385 } 386 387 out: 388 put_device(map->dev); 389 kfree(tsk); 390 return ret; 391 } 392 393 static long map_benchmark_ioctl(struct file *file, unsigned int cmd, 394 unsigned long arg) 395 { 396 struct map_benchmark_data *map = file->private_data; 397 void __user *argp = (void __user *)arg; 398 u64 old_dma_mask; 399 int ret; 400 401 if (copy_from_user(&map->bparam, argp, sizeof(map->bparam))) 402 return -EFAULT; 403 404 switch (cmd) { 405 case DMA_MAP_BENCHMARK: 406 if (map->bparam.map_mode < 0 || 407 map->bparam.map_mode >= DMA_MAP_BENCH_MODE_MAX) { 408 pr_err("invalid map mode\n"); 409 return -EINVAL; 410 } 411 412 if (map->bparam.threads == 0 || 413 map->bparam.threads > DMA_MAP_MAX_THREADS) { 414 pr_err("invalid thread number\n"); 415 return -EINVAL; 416 } 417 418 if (map->bparam.seconds == 0 || 419 map->bparam.seconds > DMA_MAP_MAX_SECONDS) { 420 pr_err("invalid duration seconds\n"); 421 return -EINVAL; 422 } 423 424 if (map->bparam.dma_trans_ns > DMA_MAP_MAX_TRANS_DELAY) { 425 pr_err("invalid transmission delay\n"); 426 return -EINVAL; 427 } 428 429 if (map->bparam.node != NUMA_NO_NODE && 430 (map->bparam.node < 0 || map->bparam.node >= MAX_NUMNODES || 431 !node_possible(map->bparam.node))) { 432 pr_err("invalid numa node\n"); 433 return -EINVAL; 434 } 435 436 if (map->bparam.granule < 1 || map->bparam.granule > 1024) { 437 pr_err("invalid granule size\n"); 438 return -EINVAL; 439 } 440 441 switch (map->bparam.dma_dir) { 442 case DMA_MAP_BIDIRECTIONAL: 443 map->dir = DMA_BIDIRECTIONAL; 444 break; 445 case DMA_MAP_FROM_DEVICE: 446 map->dir = DMA_FROM_DEVICE; 447 break; 448 case DMA_MAP_TO_DEVICE: 449 map->dir = DMA_TO_DEVICE; 450 break; 451 default: 452 pr_err("invalid DMA direction\n"); 453 return -EINVAL; 454 } 455 456 old_dma_mask = dma_get_mask(map->dev); 457 458 ret = dma_set_mask(map->dev, 459 DMA_BIT_MASK(map->bparam.dma_bits)); 460 if (ret) { 461 pr_err("failed to set dma_mask on device %s\n", 462 dev_name(map->dev)); 463 return -EINVAL; 464 } 465 466 ret = do_map_benchmark(map); 467 468 /* 469 * restore the original dma_mask as many devices' dma_mask are 470 * set by architectures, acpi, busses. When we bind them back 471 * to their original drivers, those drivers shouldn't see 472 * dma_mask changed by benchmark 473 */ 474 dma_set_mask(map->dev, old_dma_mask); 475 476 if (ret) 477 return ret; 478 break; 479 default: 480 return -EINVAL; 481 } 482 483 if (copy_to_user(argp, &map->bparam, sizeof(map->bparam))) 484 return -EFAULT; 485 486 return ret; 487 } 488 489 static const struct file_operations map_benchmark_fops = { 490 .open = simple_open, 491 .unlocked_ioctl = map_benchmark_ioctl, 492 }; 493 494 static void map_benchmark_remove_debugfs(void *data) 495 { 496 struct map_benchmark_data *map = (struct map_benchmark_data *)data; 497 498 debugfs_remove(map->debugfs); 499 } 500 501 static int __map_benchmark_probe(struct device *dev) 502 { 503 struct dentry *entry; 504 struct map_benchmark_data *map; 505 int ret; 506 507 map = devm_kzalloc(dev, sizeof(*map), GFP_KERNEL); 508 if (!map) 509 return -ENOMEM; 510 map->dev = dev; 511 512 ret = devm_add_action(dev, map_benchmark_remove_debugfs, map); 513 if (ret) { 514 pr_err("Can't add debugfs remove action\n"); 515 return ret; 516 } 517 518 /* 519 * we only permit a device bound with this driver, 2nd probe 520 * will fail 521 */ 522 entry = debugfs_create_file("dma_map_benchmark", 0600, NULL, map, 523 &map_benchmark_fops); 524 if (IS_ERR(entry)) 525 return PTR_ERR(entry); 526 map->debugfs = entry; 527 528 return 0; 529 } 530 531 static int map_benchmark_platform_probe(struct platform_device *pdev) 532 { 533 return __map_benchmark_probe(&pdev->dev); 534 } 535 536 static struct platform_driver map_benchmark_platform_driver = { 537 .driver = { 538 .name = "dma_map_benchmark", 539 }, 540 .probe = map_benchmark_platform_probe, 541 }; 542 543 static int 544 map_benchmark_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 545 { 546 return __map_benchmark_probe(&pdev->dev); 547 } 548 549 static struct pci_driver map_benchmark_pci_driver = { 550 .name = "dma_map_benchmark", 551 .probe = map_benchmark_pci_probe, 552 }; 553 554 static int __init map_benchmark_init(void) 555 { 556 int ret; 557 558 ret = pci_register_driver(&map_benchmark_pci_driver); 559 if (ret) 560 return ret; 561 562 ret = platform_driver_register(&map_benchmark_platform_driver); 563 if (ret) { 564 pci_unregister_driver(&map_benchmark_pci_driver); 565 return ret; 566 } 567 568 return 0; 569 } 570 571 static void __exit map_benchmark_cleanup(void) 572 { 573 platform_driver_unregister(&map_benchmark_platform_driver); 574 pci_unregister_driver(&map_benchmark_pci_driver); 575 } 576 577 module_init(map_benchmark_init); 578 module_exit(map_benchmark_cleanup); 579 580 MODULE_AUTHOR("Barry Song <song.bao.hua@hisilicon.com>"); 581 MODULE_DESCRIPTION("dma_map benchmark driver"); 582