1 /* 2 * DMA Engine test module 3 * 4 * Copyright (C) 2007 Atmel Corporation 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 #include <linux/delay.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/dmaengine.h> 13 #include <linux/freezer.h> 14 #include <linux/init.h> 15 #include <linux/kthread.h> 16 #include <linux/module.h> 17 #include <linux/moduleparam.h> 18 #include <linux/random.h> 19 #include <linux/slab.h> 20 #include <linux/wait.h> 21 22 static unsigned int test_buf_size = 16384; 23 module_param(test_buf_size, uint, S_IRUGO); 24 MODULE_PARM_DESC(test_buf_size, "Size of the memcpy test buffer"); 25 26 static char test_channel[20]; 27 module_param_string(channel, test_channel, sizeof(test_channel), S_IRUGO); 28 MODULE_PARM_DESC(channel, "Bus ID of the channel to test (default: any)"); 29 30 static char test_device[20]; 31 module_param_string(device, test_device, sizeof(test_device), S_IRUGO); 32 MODULE_PARM_DESC(device, "Bus ID of the DMA Engine to test (default: any)"); 33 34 static unsigned int threads_per_chan = 1; 35 module_param(threads_per_chan, uint, S_IRUGO); 36 MODULE_PARM_DESC(threads_per_chan, 37 "Number of threads to start per channel (default: 1)"); 38 39 static unsigned int max_channels; 40 module_param(max_channels, uint, S_IRUGO); 41 MODULE_PARM_DESC(max_channels, 42 "Maximum number of channels to use (default: all)"); 43 44 static unsigned int iterations; 45 module_param(iterations, uint, S_IRUGO); 46 MODULE_PARM_DESC(iterations, 47 "Iterations before stopping test (default: infinite)"); 48 49 static unsigned int xor_sources = 3; 50 module_param(xor_sources, uint, S_IRUGO); 51 MODULE_PARM_DESC(xor_sources, 52 "Number of xor source buffers (default: 3)"); 53 54 static unsigned int pq_sources = 3; 55 module_param(pq_sources, uint, S_IRUGO); 56 MODULE_PARM_DESC(pq_sources, 57 "Number of p+q source buffers (default: 3)"); 58 59 static int timeout = 3000; 60 module_param(timeout, uint, S_IRUGO); 61 MODULE_PARM_DESC(timeout, "Transfer Timeout in msec (default: 3000), " 62 "Pass -1 for infinite timeout"); 63 64 /* 65 * Initialization patterns. All bytes in the source buffer has bit 7 66 * set, all bytes in the destination buffer has bit 7 cleared. 67 * 68 * Bit 6 is set for all bytes which are to be copied by the DMA 69 * engine. Bit 5 is set for all bytes which are to be overwritten by 70 * the DMA engine. 71 * 72 * The remaining bits are the inverse of a counter which increments by 73 * one for each byte address. 74 */ 75 #define PATTERN_SRC 0x80 76 #define PATTERN_DST 0x00 77 #define PATTERN_COPY 0x40 78 #define PATTERN_OVERWRITE 0x20 79 #define PATTERN_COUNT_MASK 0x1f 80 81 struct dmatest_thread { 82 struct list_head node; 83 struct task_struct *task; 84 struct dma_chan *chan; 85 u8 **srcs; 86 u8 **dsts; 87 enum dma_transaction_type type; 88 }; 89 90 struct dmatest_chan { 91 struct list_head node; 92 struct dma_chan *chan; 93 struct list_head threads; 94 }; 95 96 /* 97 * These are protected by dma_list_mutex since they're only used by 98 * the DMA filter function callback 99 */ 100 static LIST_HEAD(dmatest_channels); 101 static unsigned int nr_channels; 102 103 static bool dmatest_match_channel(struct dma_chan *chan) 104 { 105 if (test_channel[0] == '\0') 106 return true; 107 return strcmp(dma_chan_name(chan), test_channel) == 0; 108 } 109 110 static bool dmatest_match_device(struct dma_device *device) 111 { 112 if (test_device[0] == '\0') 113 return true; 114 return strcmp(dev_name(device->dev), test_device) == 0; 115 } 116 117 static unsigned long dmatest_random(void) 118 { 119 unsigned long buf; 120 121 get_random_bytes(&buf, sizeof(buf)); 122 return buf; 123 } 124 125 static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len) 126 { 127 unsigned int i; 128 u8 *buf; 129 130 for (; (buf = *bufs); bufs++) { 131 for (i = 0; i < start; i++) 132 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 133 for ( ; i < start + len; i++) 134 buf[i] = PATTERN_SRC | PATTERN_COPY 135 | (~i & PATTERN_COUNT_MASK); 136 for ( ; i < test_buf_size; i++) 137 buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 138 buf++; 139 } 140 } 141 142 static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len) 143 { 144 unsigned int i; 145 u8 *buf; 146 147 for (; (buf = *bufs); bufs++) { 148 for (i = 0; i < start; i++) 149 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 150 for ( ; i < start + len; i++) 151 buf[i] = PATTERN_DST | PATTERN_OVERWRITE 152 | (~i & PATTERN_COUNT_MASK); 153 for ( ; i < test_buf_size; i++) 154 buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 155 } 156 } 157 158 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index, 159 unsigned int counter, bool is_srcbuf) 160 { 161 u8 diff = actual ^ pattern; 162 u8 expected = pattern | (~counter & PATTERN_COUNT_MASK); 163 const char *thread_name = current->comm; 164 165 if (is_srcbuf) 166 pr_warning("%s: srcbuf[0x%x] overwritten!" 167 " Expected %02x, got %02x\n", 168 thread_name, index, expected, actual); 169 else if ((pattern & PATTERN_COPY) 170 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE))) 171 pr_warning("%s: dstbuf[0x%x] not copied!" 172 " Expected %02x, got %02x\n", 173 thread_name, index, expected, actual); 174 else if (diff & PATTERN_SRC) 175 pr_warning("%s: dstbuf[0x%x] was copied!" 176 " Expected %02x, got %02x\n", 177 thread_name, index, expected, actual); 178 else 179 pr_warning("%s: dstbuf[0x%x] mismatch!" 180 " Expected %02x, got %02x\n", 181 thread_name, index, expected, actual); 182 } 183 184 static unsigned int dmatest_verify(u8 **bufs, unsigned int start, 185 unsigned int end, unsigned int counter, u8 pattern, 186 bool is_srcbuf) 187 { 188 unsigned int i; 189 unsigned int error_count = 0; 190 u8 actual; 191 u8 expected; 192 u8 *buf; 193 unsigned int counter_orig = counter; 194 195 for (; (buf = *bufs); bufs++) { 196 counter = counter_orig; 197 for (i = start; i < end; i++) { 198 actual = buf[i]; 199 expected = pattern | (~counter & PATTERN_COUNT_MASK); 200 if (actual != expected) { 201 if (error_count < 32) 202 dmatest_mismatch(actual, pattern, i, 203 counter, is_srcbuf); 204 error_count++; 205 } 206 counter++; 207 } 208 } 209 210 if (error_count > 32) 211 pr_warning("%s: %u errors suppressed\n", 212 current->comm, error_count - 32); 213 214 return error_count; 215 } 216 217 static void dmatest_callback(void *completion) 218 { 219 complete(completion); 220 } 221 222 /* 223 * This function repeatedly tests DMA transfers of various lengths and 224 * offsets for a given operation type until it is told to exit by 225 * kthread_stop(). There may be multiple threads running this function 226 * in parallel for a single channel, and there may be multiple channels 227 * being tested in parallel. 228 * 229 * Before each test, the source and destination buffer is initialized 230 * with a known pattern. This pattern is different depending on 231 * whether it's in an area which is supposed to be copied or 232 * overwritten, and different in the source and destination buffers. 233 * So if the DMA engine doesn't copy exactly what we tell it to copy, 234 * we'll notice. 235 */ 236 static int dmatest_func(void *data) 237 { 238 struct dmatest_thread *thread = data; 239 struct dma_chan *chan; 240 const char *thread_name; 241 unsigned int src_off, dst_off, len; 242 unsigned int error_count; 243 unsigned int failed_tests = 0; 244 unsigned int total_tests = 0; 245 dma_cookie_t cookie; 246 enum dma_status status; 247 enum dma_ctrl_flags flags; 248 u8 pq_coefs[pq_sources + 1]; 249 int ret; 250 int src_cnt; 251 int dst_cnt; 252 int i; 253 254 thread_name = current->comm; 255 set_freezable_with_signal(); 256 257 ret = -ENOMEM; 258 259 smp_rmb(); 260 chan = thread->chan; 261 if (thread->type == DMA_MEMCPY) 262 src_cnt = dst_cnt = 1; 263 else if (thread->type == DMA_XOR) { 264 src_cnt = xor_sources | 1; /* force odd to ensure dst = src */ 265 dst_cnt = 1; 266 } else if (thread->type == DMA_PQ) { 267 src_cnt = pq_sources | 1; /* force odd to ensure dst = src */ 268 dst_cnt = 2; 269 for (i = 0; i < src_cnt; i++) 270 pq_coefs[i] = 1; 271 } else 272 goto err_srcs; 273 274 thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL); 275 if (!thread->srcs) 276 goto err_srcs; 277 for (i = 0; i < src_cnt; i++) { 278 thread->srcs[i] = kmalloc(test_buf_size, GFP_KERNEL); 279 if (!thread->srcs[i]) 280 goto err_srcbuf; 281 } 282 thread->srcs[i] = NULL; 283 284 thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL); 285 if (!thread->dsts) 286 goto err_dsts; 287 for (i = 0; i < dst_cnt; i++) { 288 thread->dsts[i] = kmalloc(test_buf_size, GFP_KERNEL); 289 if (!thread->dsts[i]) 290 goto err_dstbuf; 291 } 292 thread->dsts[i] = NULL; 293 294 set_user_nice(current, 10); 295 296 /* 297 * src buffers are freed by the DMAEngine code with dma_unmap_single() 298 * dst buffers are freed by ourselves below 299 */ 300 flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT 301 | DMA_COMPL_SKIP_DEST_UNMAP | DMA_COMPL_SRC_UNMAP_SINGLE; 302 303 while (!kthread_should_stop() 304 && !(iterations && total_tests >= iterations)) { 305 struct dma_device *dev = chan->device; 306 struct dma_async_tx_descriptor *tx = NULL; 307 dma_addr_t dma_srcs[src_cnt]; 308 dma_addr_t dma_dsts[dst_cnt]; 309 struct completion cmp; 310 unsigned long start, tmo, end = 0 /* compiler... */; 311 bool reload = true; 312 u8 align = 0; 313 314 total_tests++; 315 316 /* honor alignment restrictions */ 317 if (thread->type == DMA_MEMCPY) 318 align = dev->copy_align; 319 else if (thread->type == DMA_XOR) 320 align = dev->xor_align; 321 else if (thread->type == DMA_PQ) 322 align = dev->pq_align; 323 324 if (1 << align > test_buf_size) { 325 pr_err("%u-byte buffer too small for %d-byte alignment\n", 326 test_buf_size, 1 << align); 327 break; 328 } 329 330 len = dmatest_random() % test_buf_size + 1; 331 len = (len >> align) << align; 332 if (!len) 333 len = 1 << align; 334 src_off = dmatest_random() % (test_buf_size - len + 1); 335 dst_off = dmatest_random() % (test_buf_size - len + 1); 336 337 src_off = (src_off >> align) << align; 338 dst_off = (dst_off >> align) << align; 339 340 dmatest_init_srcs(thread->srcs, src_off, len); 341 dmatest_init_dsts(thread->dsts, dst_off, len); 342 343 for (i = 0; i < src_cnt; i++) { 344 u8 *buf = thread->srcs[i] + src_off; 345 346 dma_srcs[i] = dma_map_single(dev->dev, buf, len, 347 DMA_TO_DEVICE); 348 } 349 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */ 350 for (i = 0; i < dst_cnt; i++) { 351 dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i], 352 test_buf_size, 353 DMA_BIDIRECTIONAL); 354 } 355 356 357 if (thread->type == DMA_MEMCPY) 358 tx = dev->device_prep_dma_memcpy(chan, 359 dma_dsts[0] + dst_off, 360 dma_srcs[0], len, 361 flags); 362 else if (thread->type == DMA_XOR) 363 tx = dev->device_prep_dma_xor(chan, 364 dma_dsts[0] + dst_off, 365 dma_srcs, src_cnt, 366 len, flags); 367 else if (thread->type == DMA_PQ) { 368 dma_addr_t dma_pq[dst_cnt]; 369 370 for (i = 0; i < dst_cnt; i++) 371 dma_pq[i] = dma_dsts[i] + dst_off; 372 tx = dev->device_prep_dma_pq(chan, dma_pq, dma_srcs, 373 src_cnt, pq_coefs, 374 len, flags); 375 } 376 377 if (!tx) { 378 for (i = 0; i < src_cnt; i++) 379 dma_unmap_single(dev->dev, dma_srcs[i], len, 380 DMA_TO_DEVICE); 381 for (i = 0; i < dst_cnt; i++) 382 dma_unmap_single(dev->dev, dma_dsts[i], 383 test_buf_size, 384 DMA_BIDIRECTIONAL); 385 pr_warning("%s: #%u: prep error with src_off=0x%x " 386 "dst_off=0x%x len=0x%x\n", 387 thread_name, total_tests - 1, 388 src_off, dst_off, len); 389 msleep(100); 390 failed_tests++; 391 continue; 392 } 393 394 init_completion(&cmp); 395 tx->callback = dmatest_callback; 396 tx->callback_param = &cmp; 397 cookie = tx->tx_submit(tx); 398 399 if (dma_submit_error(cookie)) { 400 pr_warning("%s: #%u: submit error %d with src_off=0x%x " 401 "dst_off=0x%x len=0x%x\n", 402 thread_name, total_tests - 1, cookie, 403 src_off, dst_off, len); 404 msleep(100); 405 failed_tests++; 406 continue; 407 } 408 dma_async_issue_pending(chan); 409 410 do { 411 start = jiffies; 412 if (reload) 413 end = start + msecs_to_jiffies(timeout); 414 else if (end <= start) 415 end = start + 1; 416 tmo = wait_for_completion_interruptible_timeout(&cmp, 417 end - start); 418 reload = try_to_freeze(); 419 } while (tmo == -ERESTARTSYS); 420 421 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); 422 423 if (tmo == 0) { 424 pr_warning("%s: #%u: test timed out\n", 425 thread_name, total_tests - 1); 426 failed_tests++; 427 continue; 428 } else if (status != DMA_SUCCESS) { 429 pr_warning("%s: #%u: got completion callback," 430 " but status is \'%s\'\n", 431 thread_name, total_tests - 1, 432 status == DMA_ERROR ? "error" : "in progress"); 433 failed_tests++; 434 continue; 435 } 436 437 /* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */ 438 for (i = 0; i < dst_cnt; i++) 439 dma_unmap_single(dev->dev, dma_dsts[i], test_buf_size, 440 DMA_BIDIRECTIONAL); 441 442 error_count = 0; 443 444 pr_debug("%s: verifying source buffer...\n", thread_name); 445 error_count += dmatest_verify(thread->srcs, 0, src_off, 446 0, PATTERN_SRC, true); 447 error_count += dmatest_verify(thread->srcs, src_off, 448 src_off + len, src_off, 449 PATTERN_SRC | PATTERN_COPY, true); 450 error_count += dmatest_verify(thread->srcs, src_off + len, 451 test_buf_size, src_off + len, 452 PATTERN_SRC, true); 453 454 pr_debug("%s: verifying dest buffer...\n", 455 thread->task->comm); 456 error_count += dmatest_verify(thread->dsts, 0, dst_off, 457 0, PATTERN_DST, false); 458 error_count += dmatest_verify(thread->dsts, dst_off, 459 dst_off + len, src_off, 460 PATTERN_SRC | PATTERN_COPY, false); 461 error_count += dmatest_verify(thread->dsts, dst_off + len, 462 test_buf_size, dst_off + len, 463 PATTERN_DST, false); 464 465 if (error_count) { 466 pr_warning("%s: #%u: %u errors with " 467 "src_off=0x%x dst_off=0x%x len=0x%x\n", 468 thread_name, total_tests - 1, error_count, 469 src_off, dst_off, len); 470 failed_tests++; 471 } else { 472 pr_debug("%s: #%u: No errors with " 473 "src_off=0x%x dst_off=0x%x len=0x%x\n", 474 thread_name, total_tests - 1, 475 src_off, dst_off, len); 476 } 477 } 478 479 ret = 0; 480 for (i = 0; thread->dsts[i]; i++) 481 kfree(thread->dsts[i]); 482 err_dstbuf: 483 kfree(thread->dsts); 484 err_dsts: 485 for (i = 0; thread->srcs[i]; i++) 486 kfree(thread->srcs[i]); 487 err_srcbuf: 488 kfree(thread->srcs); 489 err_srcs: 490 pr_notice("%s: terminating after %u tests, %u failures (status %d)\n", 491 thread_name, total_tests, failed_tests, ret); 492 493 /* terminate all transfers on specified channels */ 494 chan->device->device_control(chan, DMA_TERMINATE_ALL, 0); 495 if (iterations > 0) 496 while (!kthread_should_stop()) { 497 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wait_dmatest_exit); 498 interruptible_sleep_on(&wait_dmatest_exit); 499 } 500 501 return ret; 502 } 503 504 static void dmatest_cleanup_channel(struct dmatest_chan *dtc) 505 { 506 struct dmatest_thread *thread; 507 struct dmatest_thread *_thread; 508 int ret; 509 510 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { 511 ret = kthread_stop(thread->task); 512 pr_debug("dmatest: thread %s exited with status %d\n", 513 thread->task->comm, ret); 514 list_del(&thread->node); 515 kfree(thread); 516 } 517 518 /* terminate all transfers on specified channels */ 519 dtc->chan->device->device_control(dtc->chan, DMA_TERMINATE_ALL, 0); 520 521 kfree(dtc); 522 } 523 524 static int dmatest_add_threads(struct dmatest_chan *dtc, enum dma_transaction_type type) 525 { 526 struct dmatest_thread *thread; 527 struct dma_chan *chan = dtc->chan; 528 char *op; 529 unsigned int i; 530 531 if (type == DMA_MEMCPY) 532 op = "copy"; 533 else if (type == DMA_XOR) 534 op = "xor"; 535 else if (type == DMA_PQ) 536 op = "pq"; 537 else 538 return -EINVAL; 539 540 for (i = 0; i < threads_per_chan; i++) { 541 thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL); 542 if (!thread) { 543 pr_warning("dmatest: No memory for %s-%s%u\n", 544 dma_chan_name(chan), op, i); 545 546 break; 547 } 548 thread->chan = dtc->chan; 549 thread->type = type; 550 smp_wmb(); 551 thread->task = kthread_run(dmatest_func, thread, "%s-%s%u", 552 dma_chan_name(chan), op, i); 553 if (IS_ERR(thread->task)) { 554 pr_warning("dmatest: Failed to run thread %s-%s%u\n", 555 dma_chan_name(chan), op, i); 556 kfree(thread); 557 break; 558 } 559 560 /* srcbuf and dstbuf are allocated by the thread itself */ 561 562 list_add_tail(&thread->node, &dtc->threads); 563 } 564 565 return i; 566 } 567 568 static int dmatest_add_channel(struct dma_chan *chan) 569 { 570 struct dmatest_chan *dtc; 571 struct dma_device *dma_dev = chan->device; 572 unsigned int thread_count = 0; 573 int cnt; 574 575 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL); 576 if (!dtc) { 577 pr_warning("dmatest: No memory for %s\n", dma_chan_name(chan)); 578 return -ENOMEM; 579 } 580 581 dtc->chan = chan; 582 INIT_LIST_HEAD(&dtc->threads); 583 584 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 585 cnt = dmatest_add_threads(dtc, DMA_MEMCPY); 586 thread_count += cnt > 0 ? cnt : 0; 587 } 588 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 589 cnt = dmatest_add_threads(dtc, DMA_XOR); 590 thread_count += cnt > 0 ? cnt : 0; 591 } 592 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { 593 cnt = dmatest_add_threads(dtc, DMA_PQ); 594 thread_count += cnt > 0 ?: 0; 595 } 596 597 pr_info("dmatest: Started %u threads using %s\n", 598 thread_count, dma_chan_name(chan)); 599 600 list_add_tail(&dtc->node, &dmatest_channels); 601 nr_channels++; 602 603 return 0; 604 } 605 606 static bool filter(struct dma_chan *chan, void *param) 607 { 608 if (!dmatest_match_channel(chan) || !dmatest_match_device(chan->device)) 609 return false; 610 else 611 return true; 612 } 613 614 static int __init dmatest_init(void) 615 { 616 dma_cap_mask_t mask; 617 struct dma_chan *chan; 618 int err = 0; 619 620 dma_cap_zero(mask); 621 dma_cap_set(DMA_MEMCPY, mask); 622 for (;;) { 623 chan = dma_request_channel(mask, filter, NULL); 624 if (chan) { 625 err = dmatest_add_channel(chan); 626 if (err) { 627 dma_release_channel(chan); 628 break; /* add_channel failed, punt */ 629 } 630 } else 631 break; /* no more channels available */ 632 if (max_channels && nr_channels >= max_channels) 633 break; /* we have all we need */ 634 } 635 636 return err; 637 } 638 /* when compiled-in wait for drivers to load first */ 639 late_initcall(dmatest_init); 640 641 static void __exit dmatest_exit(void) 642 { 643 struct dmatest_chan *dtc, *_dtc; 644 struct dma_chan *chan; 645 646 list_for_each_entry_safe(dtc, _dtc, &dmatest_channels, node) { 647 list_del(&dtc->node); 648 chan = dtc->chan; 649 dmatest_cleanup_channel(dtc); 650 pr_debug("dmatest: dropped channel %s\n", 651 dma_chan_name(chan)); 652 dma_release_channel(chan); 653 } 654 } 655 module_exit(dmatest_exit); 656 657 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)"); 658 MODULE_LICENSE("GPL v2"); 659