57 MALLOC_DEFINE(M_IOAT_TEST, "ioat_test", "ioat test allocations");
66 	struct ioat_test	*test;  member
74 MTX_SYSINIT(ioat_test_lk, &ioat_test_lk, "test coordination mtx", MTX_DEF);
98 test_transaction *ioat_test_transaction_create(struct ioat_test *test,  in ioat_test_transaction_create()  argument
108 	tx->length = test->buffer_size;  in ioat_test_transaction_create()
111 		if (test->testkind == IOAT_TEST_DMA_8K)  in ioat_test_transaction_create()
112 			tx->buf[i] = malloc(test->buffer_size, M_IOAT_TEST,  in ioat_test_transaction_create()
115 			tx->buf[i] = contigmalloc(test->buffer_size,  in ioat_test_transaction_create()
142 	struct ioat_test *test;  in ioat_compare_ok()  local
146 	test = tx->test;  in ioat_compare_ok()
152 		if (test->testkind == IOAT_TEST_FILL) {  in ioat_compare_ok()
159 		} else if (test->testkind == IOAT_TEST_DMA) {  in ioat_compare_ok()
162 		} else if (test->testkind == IOAT_TEST_RAW_DMA) {  in ioat_compare_ok()
163 			if (test->raw_write)  in ioat_compare_ok()
164 				dst = test->raw_vtarget;  in ioat_compare_ok()
175 	struct ioat_test *test;  in ioat_dma_test_callback()  local
181 	test = tx->test;  in ioat_dma_test_callback()
183 	if (test->verify && !ioat_compare_ok(tx)) {  in ioat_dma_test_callback()
185 		atomic_add_32(&test->status[IOAT_TEST_MISCOMPARE], tx->depth);  in ioat_dma_test_callback()
186 	} else if (!test->too_late)  in ioat_dma_test_callback()
187 		atomic_add_32(&test->status[IOAT_TEST_OK], tx->depth);  in ioat_dma_test_callback()
190 	TAILQ_REMOVE(&test->pend_q, tx, entry);  in ioat_dma_test_callback()
191 	TAILQ_INSERT_TAIL(&test->free_q, tx, entry);  in ioat_dma_test_callback()
192 	wakeup(&test->free_q);  in ioat_dma_test_callback()
197 ioat_test_prealloc_memory(struct ioat_test *test, int index)  in ioat_test_prealloc_memory()  argument
202 	for (i = 0; i < test->transactions; i++) {  in ioat_test_prealloc_memory()
203 		tx = ioat_test_transaction_create(test, test->chain_depth * 2);  in ioat_test_prealloc_memory()
206 			test->status[IOAT_TEST_NO_MEMORY]++;  in ioat_test_prealloc_memory()
210 		TAILQ_INSERT_HEAD(&test->free_q, tx, entry);  in ioat_test_prealloc_memory()
212 		tx->test = test;  in ioat_test_prealloc_memory()
213 		tx->depth = test->chain_depth;  in ioat_test_prealloc_memory()
219 			for (k = 0; k < test->chain_depth; k++) {  in ioat_test_prealloc_memory()
229 ioat_test_release_memory(struct ioat_test *test)  in ioat_test_release_memory()  argument
233 	TAILQ_FOREACH_SAFE(tx, &test->free_q, entry, s)  in ioat_test_release_memory()
235 	TAILQ_INIT(&test->free_q);  in ioat_test_release_memory()
237 	TAILQ_FOREACH_SAFE(tx, &test->pend_q, entry, s)  in ioat_test_release_memory()
239 	TAILQ_INIT(&test->pend_q);  in ioat_test_release_memory()
243 ioat_test_submit_1_tx(struct ioat_test *test, bus_dmaengine_t dma)  in ioat_test_submit_1_tx()  argument
255 	while (TAILQ_EMPTY(&test->free_q))  in ioat_test_submit_1_tx()
256 		msleep(&test->free_q, &ioat_test_lk, 0, "test_submit", 0);  in ioat_test_submit_1_tx()
258 	tx = TAILQ_FIRST(&test->free_q);  in ioat_test_submit_1_tx()
259 	TAILQ_REMOVE(&test->free_q, tx, entry);  in ioat_test_submit_1_tx()
260 	TAILQ_INSERT_HEAD(&test->pend_q, tx, entry);  in ioat_test_submit_1_tx()
263 	if (test->testkind != IOAT_TEST_MEMCPY)  in ioat_test_submit_1_tx()
266 		if (test->testkind == IOAT_TEST_MEMCPY) {  in ioat_test_submit_1_tx()
276 		if (test->testkind == IOAT_TEST_RAW_DMA) {  in ioat_test_submit_1_tx()
277 			if (test->raw_write)  in ioat_test_submit_1_tx()
278 				dest = test->raw_target;  in ioat_test_submit_1_tx()
280 				src = test->raw_target;  in ioat_test_submit_1_tx()
291 		if (test->testkind == IOAT_TEST_DMA ||  in ioat_test_submit_1_tx()
292 		    test->testkind == IOAT_TEST_RAW_DMA)  in ioat_test_submit_1_tx()
295 		else if (test->testkind == IOAT_TEST_FILL) {  in ioat_test_submit_1_tx()
299 		} else if (test->testkind == IOAT_TEST_DMA_8K) {  in ioat_test_submit_1_tx()
307 		} else if (test->testkind == IOAT_TEST_DMA_8K_PB) {  in ioat_test_submit_1_tx()
315 		} else if (test->testkind == IOAT_TEST_DMA_CRC) {  in ioat_test_submit_1_tx()
322 		} else if (test->testkind == IOAT_TEST_DMA_CRC_COPY) {  in ioat_test_submit_1_tx()
333 	if (test->testkind == IOAT_TEST_MEMCPY)  in ioat_test_submit_1_tx()
343 		atomic_add_32(&test->status[IOAT_TEST_NO_DMA_ENGINE], tx->depth);  in ioat_test_submit_1_tx()
345 		TAILQ_REMOVE(&test->pend_q, tx, entry);  in ioat_test_submit_1_tx()
346 		TAILQ_INSERT_HEAD(&test->free_q, tx, entry);  in ioat_test_submit_1_tx()
355 	struct ioat_test *test;  in ioat_dma_test()  local
360 	test = arg;  in ioat_dma_test()
361 	memset(__DEVOLATILE(void *, test->status), 0, sizeof(test->status));  in ioat_dma_test()
363 	if ((test->testkind == IOAT_TEST_DMA_8K ||  in ioat_dma_test()
364 	    test->testkind == IOAT_TEST_DMA_8K_PB) &&  in ioat_dma_test()
365 	    test->buffer_size != 2 * PAGE_SIZE) {  in ioat_dma_test()
366 		ioat_test_log(0, "Asked for 8k test and buffer size isn't 8k\n");  in ioat_dma_test()
367 		test->status[IOAT_TEST_INVALID_INPUT]++;  in ioat_dma_test()
371 	if (test->buffer_size > 1024 * 1024) {  in ioat_dma_test()
373 		test->status[IOAT_TEST_NO_MEMORY]++;  in ioat_dma_test()
377 	if (test->chain_depth * 2 > IOAT_MAX_BUFS) {  in ioat_dma_test()
380 		test->status[IOAT_TEST_NO_MEMORY]++;  in ioat_dma_test()
384 	if (btoc((uint64_t)test->buffer_size * test->chain_depth *  in ioat_dma_test()
385 	    test->transactions) > (physmem / 4)) {  in ioat_dma_test()
386 		ioat_test_log(0, "Sanity check failed -- test would "  in ioat_dma_test()
388 		test->status[IOAT_TEST_NO_MEMORY]++;  in ioat_dma_test()
392 	if ((uint64_t)test->transactions * test->chain_depth > (1<<16)) {  in ioat_dma_test()
393 		ioat_test_log(0, "Sanity check failed -- test would "  in ioat_dma_test()
395 		test->status[IOAT_TEST_NO_MEMORY]++;  in ioat_dma_test()
399 	if (test->testkind >= IOAT_NUM_TESTKINDS) {  in ioat_dma_test()
401 		    (unsigned)test->testkind);  in ioat_dma_test()
402 		test->status[IOAT_TEST_INVALID_INPUT]++;  in ioat_dma_test()
406 	dmaengine = ioat_get_dmaengine(test->channel_index, M_NOWAIT);  in ioat_dma_test()
409 		test->status[IOAT_TEST_NO_DMA_ENGINE]++;  in ioat_dma_test()
414 	if (test->testkind == IOAT_TEST_FILL &&  in ioat_dma_test()
418 		    "Hardware doesn't support block fill, aborting test\n");  in ioat_dma_test()
419 		test->status[IOAT_TEST_INVALID_INPUT]++;  in ioat_dma_test()
423 	if (test->coalesce_period > ioat->intrdelay_max) {  in ioat_dma_test()
426 		    (unsigned)test->coalesce_period);  in ioat_dma_test()
427 		test->status[IOAT_TEST_INVALID_INPUT]++;  in ioat_dma_test()
430 	error = ioat_set_interrupt_coalesce(dmaengine, test->coalesce_period);  in ioat_dma_test()
431 	if (error == ENODEV && test->coalesce_period == 0)  in ioat_dma_test()
435 		test->status[IOAT_TEST_INVALID_INPUT]++;  in ioat_dma_test()
439 	if (test->zero_stats)  in ioat_dma_test()
442 	if (test->testkind == IOAT_TEST_RAW_DMA) {  in ioat_dma_test()
443 		if (test->raw_is_virtual) {  in ioat_dma_test()
444 			test->raw_vtarget = (void *)test->raw_target;  in ioat_dma_test()
445 			test->raw_target = vtophys(test->raw_vtarget);  in ioat_dma_test()
447 			test->raw_vtarget = pmap_mapdev(test->raw_target,  in ioat_dma_test()
448 			    test->buffer_size);  in ioat_dma_test()
453 	TAILQ_INIT(&test->free_q);  in ioat_dma_test()
454 	TAILQ_INIT(&test->pend_q);  in ioat_dma_test()
456 	if (test->duration == 0)  in ioat_dma_test()
458 		    index, test->transactions);  in ioat_dma_test()
462 	rc = ioat_test_prealloc_memory(test, index);  in ioat_dma_test()
469 	test->too_late = false;  in ioat_dma_test()
471 	end = start + (((sbintime_t)test->duration * hz) / 1000);  in ioat_dma_test()
474 		if (test->duration == 0 && loops >= test->transactions)  in ioat_dma_test()
476 		else if (test->duration != 0 && time_after(ticks, end)) {  in ioat_dma_test()
477 			test->too_late = true;  in ioat_dma_test()
481 		ioat_test_submit_1_tx(test, dmaengine);  in ioat_dma_test()
484 	ioat_test_log(1, "Test Elapsed: %d ticks (overrun %d), %d sec.\n",  in ioat_dma_test()
488 	while (!TAILQ_EMPTY(&test->pend_q))  in ioat_dma_test()
489 		msleep(&test->free_q, &ioat_test_lk, 0, "ioattestcompl", hz);  in ioat_dma_test()
492 	ioat_test_log(1, "Test Elapsed2: %d ticks (overrun %d), %d sec.\n",  in ioat_dma_test()
495 	ioat_test_release_memory(test);  in ioat_dma_test()
497 	if (test->testkind == IOAT_TEST_RAW_DMA && !test->raw_is_virtual)  in ioat_dma_test()
498 		pmap_unmapdev(test->raw_vtarget, test->buffer_size);  in ioat_dma_test()