1 /*- 2 * Copyright (C) 2012 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/systm.h> 32 #include <sys/bus.h> 33 #include <sys/conf.h> 34 #include <sys/ioccom.h> 35 #include <sys/kernel.h> 36 #include <sys/lock.h> 37 #include <sys/malloc.h> 38 #include <sys/module.h> 39 #include <sys/mutex.h> 40 #include <sys/rman.h> 41 #include <sys/sysctl.h> 42 #include <dev/pci/pcireg.h> 43 #include <dev/pci/pcivar.h> 44 #include <machine/bus.h> 45 #include <machine/resource.h> 46 #include <machine/stdarg.h> 47 #include <vm/vm.h> 48 #include <vm/vm_param.h> 49 #include <vm/pmap.h> 50 51 #include "ioat.h" 52 #include "ioat_hw.h" 53 #include "ioat_internal.h" 54 #include "ioat_test.h" 55 56 #ifndef time_after 57 #define time_after(a,b) ((long)(b) - (long)(a) < 0) 58 #endif 59 60 MALLOC_DEFINE(M_IOAT_TEST, "ioat_test", "ioat test allocations"); 61 62 #define IOAT_MAX_BUFS 256 63 64 struct test_transaction { 65 void *buf[IOAT_MAX_BUFS]; 66 uint32_t length; 67 uint32_t depth; 68 struct ioat_test *test; 69 TAILQ_ENTRY(test_transaction) entry; 70 }; 71 72 #define IT_LOCK() mtx_lock(&ioat_test_lk) 73 #define IT_UNLOCK() mtx_unlock(&ioat_test_lk) 74 #define IT_ASSERT() mtx_assert(&ioat_test_lk, MA_OWNED) 75 static struct mtx ioat_test_lk; 76 MTX_SYSINIT(ioat_test_lk, &ioat_test_lk, "test coordination mtx", MTX_DEF); 77 78 static int g_thread_index = 1; 79 static struct cdev *g_ioat_cdev = NULL; 80 81 #define ioat_test_log(v, ...) _ioat_test_log((v), "ioat_test: " __VA_ARGS__) 82 static inline void _ioat_test_log(int verbosity, const char *fmt, ...); 83 84 static void 85 ioat_test_transaction_destroy(struct test_transaction *tx) 86 { 87 int i; 88 89 for (i = 0; i < IOAT_MAX_BUFS; i++) { 90 if (tx->buf[i] != NULL) { 91 contigfree(tx->buf[i], tx->length, M_IOAT_TEST); 92 tx->buf[i] = NULL; 93 } 94 } 95 96 free(tx, M_IOAT_TEST); 97 } 98 99 static struct 100 test_transaction *ioat_test_transaction_create(unsigned num_buffers, 101 uint32_t buffer_size) 102 { 103 struct test_transaction *tx; 104 unsigned i; 105 106 tx = malloc(sizeof(*tx), M_IOAT_TEST, M_NOWAIT | M_ZERO); 107 if (tx == NULL) 108 return (NULL); 109 110 tx->length = buffer_size; 111 112 for (i = 0; i < num_buffers; i++) { 113 tx->buf[i] = contigmalloc(buffer_size, M_IOAT_TEST, M_NOWAIT, 114 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0); 115 116 if (tx->buf[i] == NULL) { 117 ioat_test_transaction_destroy(tx); 118 return (NULL); 119 } 120 } 121 return (tx); 122 } 123 124 static void 125 dump_hex(void *p, size_t chunks) 126 { 127 size_t i, j; 128 129 for (i = 0; i < chunks; i++) { 130 for (j = 0; j < 8; j++) 131 printf("%08x ", ((uint32_t *)p)[i * 8 + j]); 132 printf("\n"); 133 } 134 } 135 136 static bool 137 ioat_compare_ok(struct test_transaction *tx) 138 { 139 struct ioat_test *test; 140 char *dst, *src; 141 uint32_t i, j; 142 143 test = tx->test; 144 145 for (i = 0; i < tx->depth; i++) { 146 dst = tx->buf[2 * i + 1]; 147 src = tx->buf[2 * i]; 148 149 if (test->testkind == IOAT_TEST_FILL) { 150 for (j = 0; j < tx->length; j += sizeof(uint64_t)) { 151 if (memcmp(src, &dst[j], 152 MIN(sizeof(uint64_t), tx->length - j)) 153 != 0) 154 return (false); 155 } 156 } else if (test->testkind == IOAT_TEST_DMA) { 157 if (memcmp(src, dst, tx->length) != 0) 158 return (false); 159 } else if (test->testkind == IOAT_TEST_RAW_DMA) { 160 if (test->raw_write) 161 dst = test->raw_vtarget; 162 dump_hex(dst, tx->length / 32); 163 } 164 } 165 return (true); 166 } 167 168 static void 169 ioat_dma_test_callback(void *arg) 170 { 171 struct test_transaction *tx; 172 struct ioat_test *test; 173 174 tx = arg; 175 test = tx->test; 176 177 if (test->verify && !ioat_compare_ok(tx)) { 178 ioat_test_log(0, "miscompare found\n"); 179 atomic_add_32(&test->status[IOAT_TEST_MISCOMPARE], tx->depth); 180 } else if (!test->too_late) 181 atomic_add_32(&test->status[IOAT_TEST_OK], tx->depth); 182 183 IT_LOCK(); 184 TAILQ_REMOVE(&test->pend_q, tx, entry); 185 TAILQ_INSERT_TAIL(&test->free_q, tx, entry); 186 wakeup(&test->free_q); 187 IT_UNLOCK(); 188 } 189 190 static int 191 ioat_test_prealloc_memory(struct ioat_test *test, int index) 192 { 193 uint32_t i, j, k; 194 struct test_transaction *tx; 195 196 for (i = 0; i < test->transactions; i++) { 197 tx = ioat_test_transaction_create(test->chain_depth * 2, 198 test->buffer_size); 199 if (tx == NULL) { 200 ioat_test_log(0, "tx == NULL - memory exhausted\n"); 201 test->status[IOAT_TEST_NO_MEMORY]++; 202 return (ENOMEM); 203 } 204 205 TAILQ_INSERT_HEAD(&test->free_q, tx, entry); 206 207 tx->test = test; 208 tx->depth = test->chain_depth; 209 210 /* fill in source buffers */ 211 for (j = 0; j < (tx->length / sizeof(uint32_t)); j++) { 212 uint32_t val = j + (index << 28); 213 214 for (k = 0; k < test->chain_depth; k++) { 215 ((uint32_t *)tx->buf[2*k])[j] = ~val; 216 ((uint32_t *)tx->buf[2*k+1])[j] = val; 217 } 218 } 219 } 220 return (0); 221 } 222 223 static void 224 ioat_test_release_memory(struct ioat_test *test) 225 { 226 struct test_transaction *tx, *s; 227 228 TAILQ_FOREACH_SAFE(tx, &test->free_q, entry, s) 229 ioat_test_transaction_destroy(tx); 230 TAILQ_INIT(&test->free_q); 231 232 TAILQ_FOREACH_SAFE(tx, &test->pend_q, entry, s) 233 ioat_test_transaction_destroy(tx); 234 TAILQ_INIT(&test->pend_q); 235 } 236 237 static void 238 ioat_test_submit_1_tx(struct ioat_test *test, bus_dmaengine_t dma) 239 { 240 struct test_transaction *tx; 241 struct bus_dmadesc *desc; 242 bus_dmaengine_callback_t cb; 243 bus_addr_t src, dest; 244 uint64_t fillpattern; 245 uint32_t i, flags; 246 247 desc = NULL; 248 249 IT_LOCK(); 250 while (TAILQ_EMPTY(&test->free_q)) 251 msleep(&test->free_q, &ioat_test_lk, 0, "test_submit", 0); 252 253 tx = TAILQ_FIRST(&test->free_q); 254 TAILQ_REMOVE(&test->free_q, tx, entry); 255 TAILQ_INSERT_HEAD(&test->pend_q, tx, entry); 256 IT_UNLOCK(); 257 258 ioat_acquire(dma); 259 for (i = 0; i < tx->depth; i++) { 260 src = vtophys((vm_offset_t)tx->buf[2*i]); 261 dest = vtophys((vm_offset_t)tx->buf[2*i+1]); 262 263 if (test->testkind == IOAT_TEST_RAW_DMA) { 264 if (test->raw_write) 265 dest = test->raw_target; 266 else 267 src = test->raw_target; 268 } 269 270 if (i == tx->depth - 1) { 271 cb = ioat_dma_test_callback; 272 flags = DMA_INT_EN; 273 } else { 274 cb = NULL; 275 flags = 0; 276 } 277 278 if (test->testkind == IOAT_TEST_DMA || 279 test->testkind == IOAT_TEST_RAW_DMA) 280 desc = ioat_copy(dma, dest, src, tx->length, cb, tx, 281 flags); 282 else if (test->testkind == IOAT_TEST_FILL) { 283 fillpattern = *(uint64_t *)tx->buf[2*i]; 284 desc = ioat_blockfill(dma, dest, fillpattern, 285 tx->length, cb, tx, flags); 286 } 287 if (desc == NULL) 288 break; 289 } 290 ioat_release(dma); 291 292 /* 293 * We couldn't issue an IO -- either the device is being detached or 294 * the HW reset. Essentially spin until the device comes back up or 295 * our timer expires. 296 */ 297 if (desc == NULL && tx->depth > 0) { 298 atomic_add_32(&test->status[IOAT_TEST_NO_DMA_ENGINE], tx->depth); 299 IT_LOCK(); 300 TAILQ_REMOVE(&test->pend_q, tx, entry); 301 TAILQ_INSERT_HEAD(&test->free_q, tx, entry); 302 IT_UNLOCK(); 303 } 304 } 305 306 static void 307 ioat_dma_test(void *arg) 308 { 309 struct ioat_test *test; 310 bus_dmaengine_t dmaengine; 311 uint32_t loops; 312 int index, rc, start, end; 313 314 test = arg; 315 memset(__DEVOLATILE(void *, test->status), 0, sizeof(test->status)); 316 317 if (test->buffer_size > 1024 * 1024) { 318 ioat_test_log(0, "Buffer size too large >1MB\n"); 319 test->status[IOAT_TEST_NO_MEMORY]++; 320 return; 321 } 322 323 if (test->chain_depth * 2 > IOAT_MAX_BUFS) { 324 ioat_test_log(0, "Depth too large (> %u)\n", 325 (unsigned)IOAT_MAX_BUFS / 2); 326 test->status[IOAT_TEST_NO_MEMORY]++; 327 return; 328 } 329 330 if (btoc((uint64_t)test->buffer_size * test->chain_depth * 331 test->transactions) > (physmem / 4)) { 332 ioat_test_log(0, "Sanity check failed -- test would " 333 "use more than 1/4 of phys mem.\n"); 334 test->status[IOAT_TEST_NO_MEMORY]++; 335 return; 336 } 337 338 if ((uint64_t)test->transactions * test->chain_depth > (1<<16)) { 339 ioat_test_log(0, "Sanity check failed -- test would " 340 "use more than available IOAT ring space.\n"); 341 test->status[IOAT_TEST_NO_MEMORY]++; 342 return; 343 } 344 345 if (test->testkind >= IOAT_NUM_TESTKINDS) { 346 ioat_test_log(0, "Invalid kind %u\n", 347 (unsigned)test->testkind); 348 test->status[IOAT_TEST_INVALID_INPUT]++; 349 return; 350 } 351 352 dmaengine = ioat_get_dmaengine(test->channel_index); 353 if (dmaengine == NULL) { 354 ioat_test_log(0, "Couldn't acquire dmaengine\n"); 355 test->status[IOAT_TEST_NO_DMA_ENGINE]++; 356 return; 357 } 358 359 if (test->testkind == IOAT_TEST_FILL && 360 (to_ioat_softc(dmaengine)->capabilities & IOAT_DMACAP_BFILL) == 0) 361 { 362 ioat_test_log(0, 363 "Hardware doesn't support block fill, aborting test\n"); 364 test->status[IOAT_TEST_INVALID_INPUT]++; 365 goto out; 366 } 367 368 if (test->testkind == IOAT_TEST_RAW_DMA) { 369 if (test->raw_is_virtual) { 370 test->raw_vtarget = (void *)test->raw_target; 371 test->raw_target = vtophys(test->raw_vtarget); 372 } else { 373 test->raw_vtarget = pmap_mapdev(test->raw_target, 374 test->buffer_size); 375 } 376 } 377 378 index = g_thread_index++; 379 TAILQ_INIT(&test->free_q); 380 TAILQ_INIT(&test->pend_q); 381 382 if (test->duration == 0) 383 ioat_test_log(1, "Thread %d: num_loops remaining: 0x%08x\n", 384 index, test->transactions); 385 else 386 ioat_test_log(1, "Thread %d: starting\n", index); 387 388 rc = ioat_test_prealloc_memory(test, index); 389 if (rc != 0) { 390 ioat_test_log(0, "prealloc_memory: %d\n", rc); 391 goto out; 392 } 393 wmb(); 394 395 test->too_late = false; 396 start = ticks; 397 end = start + (((sbintime_t)test->duration * hz) / 1000); 398 399 for (loops = 0;; loops++) { 400 if (test->duration == 0 && loops >= test->transactions) 401 break; 402 else if (test->duration != 0 && time_after(ticks, end)) { 403 test->too_late = true; 404 break; 405 } 406 407 ioat_test_submit_1_tx(test, dmaengine); 408 } 409 410 ioat_test_log(1, "Test Elapsed: %d ticks (overrun %d), %d sec.\n", 411 ticks - start, ticks - end, (ticks - start) / hz); 412 413 IT_LOCK(); 414 while (!TAILQ_EMPTY(&test->pend_q)) 415 msleep(&test->free_q, &ioat_test_lk, 0, "ioattestcompl", hz); 416 IT_UNLOCK(); 417 418 ioat_test_log(1, "Test Elapsed2: %d ticks (overrun %d), %d sec.\n", 419 ticks - start, ticks - end, (ticks - start) / hz); 420 421 ioat_test_release_memory(test); 422 out: 423 if (test->testkind == IOAT_TEST_RAW_DMA && !test->raw_is_virtual) 424 pmap_unmapdev((vm_offset_t)test->raw_vtarget, 425 test->buffer_size); 426 ioat_put_dmaengine(dmaengine); 427 } 428 429 static int 430 ioat_test_open(struct cdev *dev, int flags, int fmt, struct thread *td) 431 { 432 433 return (0); 434 } 435 436 static int 437 ioat_test_close(struct cdev *dev, int flags, int fmt, struct thread *td) 438 { 439 440 return (0); 441 } 442 443 static int 444 ioat_test_ioctl(struct cdev *dev, unsigned long cmd, caddr_t arg, int flag, 445 struct thread *td) 446 { 447 448 switch (cmd) { 449 case IOAT_DMATEST: 450 ioat_dma_test(arg); 451 break; 452 default: 453 return (EINVAL); 454 } 455 return (0); 456 } 457 458 static struct cdevsw ioat_cdevsw = { 459 .d_version = D_VERSION, 460 .d_flags = 0, 461 .d_open = ioat_test_open, 462 .d_close = ioat_test_close, 463 .d_ioctl = ioat_test_ioctl, 464 .d_name = "ioat_test", 465 }; 466 467 static int 468 enable_ioat_test(bool enable) 469 { 470 471 mtx_assert(&Giant, MA_OWNED); 472 473 if (enable && g_ioat_cdev == NULL) { 474 g_ioat_cdev = make_dev(&ioat_cdevsw, 0, UID_ROOT, GID_WHEEL, 475 0600, "ioat_test"); 476 } else if (!enable && g_ioat_cdev != NULL) { 477 destroy_dev(g_ioat_cdev); 478 g_ioat_cdev = NULL; 479 } 480 return (0); 481 } 482 483 static int 484 sysctl_enable_ioat_test(SYSCTL_HANDLER_ARGS) 485 { 486 int error, enabled; 487 488 enabled = (g_ioat_cdev != NULL); 489 error = sysctl_handle_int(oidp, &enabled, 0, req); 490 if (error != 0 || req->newptr == NULL) 491 return (error); 492 493 enable_ioat_test(enabled); 494 return (0); 495 } 496 SYSCTL_PROC(_hw_ioat, OID_AUTO, enable_ioat_test, CTLTYPE_INT | CTLFLAG_RW, 497 0, 0, sysctl_enable_ioat_test, "I", 498 "Non-zero: Enable the /dev/ioat_test device"); 499 500 void 501 ioat_test_attach(void) 502 { 503 char *val; 504 505 val = kern_getenv("hw.ioat.enable_ioat_test"); 506 if (val != NULL && strcmp(val, "0") != 0) { 507 mtx_lock(&Giant); 508 enable_ioat_test(true); 509 mtx_unlock(&Giant); 510 } 511 freeenv(val); 512 } 513 514 void 515 ioat_test_detach(void) 516 { 517 518 mtx_lock(&Giant); 519 enable_ioat_test(false); 520 mtx_unlock(&Giant); 521 } 522 523 static inline void 524 _ioat_test_log(int verbosity, const char *fmt, ...) 525 { 526 va_list argp; 527 528 if (verbosity > g_ioat_debug_level) 529 return; 530 531 va_start(argp, fmt); 532 vprintf(fmt, argp); 533 va_end(argp); 534 } 535