83 ioat_test_transaction_destroy(struct test_transaction *tx)  in ioat_test_transaction_destroy()  argument
88 		if (tx->buf[i] != NULL) {  in ioat_test_transaction_destroy()
89 			free(tx->buf[i], M_IOAT_TEST);  in ioat_test_transaction_destroy()
90 			tx->buf[i] = NULL;  in ioat_test_transaction_destroy()
94 	free(tx, M_IOAT_TEST);  in ioat_test_transaction_destroy()
101 	struct test_transaction *tx;  in ioat_test_transaction_create()  local
104 	tx = malloc(sizeof(*tx), M_IOAT_TEST, M_NOWAIT | M_ZERO);  in ioat_test_transaction_create()
105 	if (tx == NULL)  in ioat_test_transaction_create()
108 	tx->length = test->buffer_size;  in ioat_test_transaction_create()
112 			tx->buf[i] = malloc(test->buffer_size, M_IOAT_TEST,  in ioat_test_transaction_create()
115 			tx->buf[i] = contigmalloc(test->buffer_size,  in ioat_test_transaction_create()
119 		if (tx->buf[i] == NULL) {  in ioat_test_transaction_create()
120 			ioat_test_transaction_destroy(tx);  in ioat_test_transaction_create()
124 	return (tx);  in ioat_test_transaction_create()
140 ioat_compare_ok(struct test_transaction *tx)  in ioat_compare_ok()  argument
146 	test = tx->test;  in ioat_compare_ok()
148 	for (i = 0; i < tx->depth; i++) {  in ioat_compare_ok()
149 		dst = tx->buf[2 * i + 1];  in ioat_compare_ok()
150 		src = tx->buf[2 * i];  in ioat_compare_ok()
153 			for (j = 0; j < tx->length; j += sizeof(uint64_t)) {  in ioat_compare_ok()
155 					MIN(sizeof(uint64_t), tx->length - j))  in ioat_compare_ok()
160 			if (memcmp(src, dst, tx->length) != 0)  in ioat_compare_ok()
165 			dump_hex(dst, tx->length / 32);  in ioat_compare_ok()
174 	struct test_transaction *tx;  in ioat_dma_test_callback()  local
180 	tx = arg;  in ioat_dma_test_callback()
181 	test = tx->test;  in ioat_dma_test_callback()
183 	if (test->verify && !ioat_compare_ok(tx)) {  in ioat_dma_test_callback()
185 		atomic_add_32(&test->status[IOAT_TEST_MISCOMPARE], tx->depth);  in ioat_dma_test_callback()
187 		atomic_add_32(&test->status[IOAT_TEST_OK], tx->depth);  in ioat_dma_test_callback()
190 	TAILQ_REMOVE(&test->pend_q, tx, entry);  in ioat_dma_test_callback()
191 	TAILQ_INSERT_TAIL(&test->free_q, tx, entry);  in ioat_dma_test_callback()
200 	struct test_transaction *tx;  in ioat_test_prealloc_memory()  local
203 		tx = ioat_test_transaction_create(test, test->chain_depth * 2);  in ioat_test_prealloc_memory()
204 		if (tx == NULL) {  in ioat_test_prealloc_memory()
205 			ioat_test_log(0, "tx == NULL - memory exhausted\n");  in ioat_test_prealloc_memory()
210 		TAILQ_INSERT_HEAD(&test->free_q, tx, entry);  in ioat_test_prealloc_memory()
212 		tx->test = test;  in ioat_test_prealloc_memory()
213 		tx->depth = test->chain_depth;  in ioat_test_prealloc_memory()
216 		for (j = 0; j < (tx->length / sizeof(uint32_t)); j++) {  in ioat_test_prealloc_memory()
220 				((uint32_t *)tx->buf[2*k])[j] = ~val;  in ioat_test_prealloc_memory()
221 				((uint32_t *)tx->buf[2*k+1])[j] = val;  in ioat_test_prealloc_memory()
231 	struct test_transaction *tx, *s;  in ioat_test_release_memory()  local
233 	TAILQ_FOREACH_SAFE(tx, &test->free_q, entry, s)  in ioat_test_release_memory()
234 		ioat_test_transaction_destroy(tx);  in ioat_test_release_memory()
237 	TAILQ_FOREACH_SAFE(tx, &test->pend_q, entry, s)  in ioat_test_release_memory()
238 		ioat_test_transaction_destroy(tx);  in ioat_test_release_memory()
245 	struct test_transaction *tx;  in ioat_test_submit_1_tx()  local
258 	tx = TAILQ_FIRST(&test->free_q);  in ioat_test_submit_1_tx()
259 	TAILQ_REMOVE(&test->free_q, tx, entry);  in ioat_test_submit_1_tx()
260 	TAILQ_INSERT_HEAD(&test->pend_q, tx, entry);  in ioat_test_submit_1_tx()
265 	for (i = 0; i < tx->depth; i++) {  in ioat_test_submit_1_tx()
267 			memcpy(tx->buf[2 * i + 1], tx->buf[2 * i], tx->length);  in ioat_test_submit_1_tx()
268 			if (i == tx->depth - 1)  in ioat_test_submit_1_tx()
269 				ioat_dma_test_callback(tx, 0);  in ioat_test_submit_1_tx()
273 		src = vtophys((vm_offset_t)tx->buf[2*i]);  in ioat_test_submit_1_tx()
274 		dest = vtophys((vm_offset_t)tx->buf[2*i+1]);  in ioat_test_submit_1_tx()
283 		if (i == tx->depth - 1) {  in ioat_test_submit_1_tx()
293 			desc = ioat_copy(dma, dest, src, tx->length, cb, tx,  in ioat_test_submit_1_tx()
296 			fillpattern = *(uint64_t *)tx->buf[2*i];  in ioat_test_submit_1_tx()
298 			    tx->length, cb, tx, flags);  in ioat_test_submit_1_tx()
302 			src2 = vtophys((vm_offset_t)tx->buf[2*i] + PAGE_SIZE);  in ioat_test_submit_1_tx()
303 			dst2 = vtophys((vm_offset_t)tx->buf[2*i+1] + PAGE_SIZE);  in ioat_test_submit_1_tx()
306 			    cb, tx, flags);  in ioat_test_submit_1_tx()
310 			src2 = vtophys((vm_offset_t)tx->buf[2*i+1] + PAGE_SIZE);  in ioat_test_submit_1_tx()
311 			dst2 = vtophys((vm_offset_t)tx->buf[2*i] + PAGE_SIZE);  in ioat_test_submit_1_tx()
314 			    cb, tx, flags);  in ioat_test_submit_1_tx()
318 			tx->crc[i] = 0;  in ioat_test_submit_1_tx()
319 			crc = vtophys((vm_offset_t)&tx->crc[i]);  in ioat_test_submit_1_tx()
320 			desc = ioat_crc(dma, src, tx->length,  in ioat_test_submit_1_tx()
321 			    NULL, crc, cb, tx, flags | DMA_CRC_STORE);  in ioat_test_submit_1_tx()
325 			tx->crc[i] = 0;  in ioat_test_submit_1_tx()
326 			crc = vtophys((vm_offset_t)&tx->crc[i]);  in ioat_test_submit_1_tx()
327 			desc = ioat_copy_crc(dma, dest, src, tx->length,  in ioat_test_submit_1_tx()
328 			    NULL, crc, cb, tx, flags | DMA_CRC_STORE);  in ioat_test_submit_1_tx()
342 	if (desc == NULL && tx->depth > 0) {  in ioat_test_submit_1_tx()
343 		atomic_add_32(&test->status[IOAT_TEST_NO_DMA_ENGINE], tx->depth);  in ioat_test_submit_1_tx()
345 		TAILQ_REMOVE(&test->pend_q, tx, entry);  in ioat_test_submit_1_tx()
346 		TAILQ_INSERT_HEAD(&test->free_q, tx, entry);  in ioat_test_submit_1_tx()