1 // SPDX-License-Identifier: GPL-2.0-only 2 /* I/O iterator tests. This can only test kernel-backed iterator types. 3 * 4 * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 10 #include <linux/module.h> 11 #include <linux/vmalloc.h> 12 #include <linux/mm.h> 13 #include <linux/uio.h> 14 #include <linux/bvec.h> 15 #include <linux/folio_queue.h> 16 #include <linux/scatterlist.h> 17 #include <linux/minmax.h> 18 #include <linux/mman.h> 19 #include <kunit/test.h> 20 21 MODULE_DESCRIPTION("iov_iter testing"); 22 MODULE_AUTHOR("David Howells <dhowells@redhat.com>"); 23 MODULE_LICENSE("GPL"); 24 25 struct kvec_test_range { 26 int from, to; 27 }; 28 29 static const struct kvec_test_range kvec_test_ranges[] = { 30 { 0x00002, 0x00002 }, 31 { 0x00027, 0x03000 }, 32 { 0x05193, 0x18794 }, 33 { 0x20000, 0x20000 }, 34 { 0x20000, 0x24000 }, 35 { 0x24000, 0x27001 }, 36 { 0x29000, 0xffffb }, 37 { 0xffffd, 0xffffe }, 38 { -1 } 39 }; 40 41 static inline u8 pattern(unsigned long x) 42 { 43 return (u8)x + (u8)(x >> 8) + (u8)(x >> 16); 44 } 45 46 static void iov_kunit_unmap(void *data) 47 { 48 vfree(data); 49 } 50 51 static void *__init iov_kunit_create_buffer(struct kunit *test, 52 struct page ***ppages, 53 size_t npages) 54 { 55 struct page **pages; 56 unsigned long got, last; 57 void *buffer; 58 unsigned int i; 59 60 pages = kzalloc_objs(struct page *, npages, GFP_KERNEL); 61 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pages); 62 *ppages = pages; 63 64 got = 0; 65 while (true) { 66 last = got; 67 got = alloc_pages_bulk(GFP_KERNEL, npages, pages); 68 69 if (last == got || got == npages) 70 break; 71 } 72 73 if (got != npages) { 74 release_pages(pages, got); 75 kvfree(pages); 76 KUNIT_ASSERT_EQ(test, got, npages); 77 } 78 /* Make sure that we don't get a physically contiguous buffer. */ 79 for (i = 0; i < npages / 4; ++i) 80 swap(pages[i], pages[i + npages / 2]); 81 82 buffer = vmap(pages, npages, VM_MAP | VM_MAP_PUT_PAGES, PAGE_KERNEL); 83 if (buffer == NULL) { 84 release_pages(pages, got); 85 kvfree(pages); 86 } 87 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buffer); 88 89 kunit_add_action_or_reset(test, iov_kunit_unmap, buffer); 90 return buffer; 91 } 92 93 static void __init iov_kunit_load_kvec(struct kunit *test, 94 struct iov_iter *iter, int dir, 95 struct kvec *kvec, unsigned int kvmax, 96 void *buffer, size_t bufsize, 97 const struct kvec_test_range *pr) 98 { 99 size_t size = 0; 100 int i; 101 102 for (i = 0; i < kvmax; i++, pr++) { 103 if (pr->from < 0) 104 break; 105 KUNIT_ASSERT_GE(test, pr->to, pr->from); 106 KUNIT_ASSERT_LE(test, pr->to, bufsize); 107 kvec[i].iov_base = buffer + pr->from; 108 kvec[i].iov_len = pr->to - pr->from; 109 size += pr->to - pr->from; 110 } 111 KUNIT_ASSERT_LE(test, size, bufsize); 112 113 iov_iter_kvec(iter, dir, kvec, i, size); 114 } 115 116 /* 117 * Test copying to a ITER_KVEC-type iterator. 118 */ 119 static void __init iov_kunit_copy_to_kvec(struct kunit *test) 120 { 121 const struct kvec_test_range *pr; 122 struct iov_iter iter; 123 struct page **spages, **bpages; 124 struct kvec kvec[8]; 125 u8 *scratch, *buffer; 126 size_t bufsize, npages, size, copied; 127 int i, patt; 128 129 bufsize = 0x100000; 130 npages = bufsize / PAGE_SIZE; 131 132 scratch = iov_kunit_create_buffer(test, &spages, npages); 133 for (i = 0; i < bufsize; i++) 134 scratch[i] = pattern(i); 135 136 buffer = iov_kunit_create_buffer(test, &bpages, npages); 137 memset(buffer, 0, bufsize); 138 139 iov_kunit_load_kvec(test, &iter, READ, kvec, ARRAY_SIZE(kvec), 140 buffer, bufsize, kvec_test_ranges); 141 size = iter.count; 142 143 copied = copy_to_iter(scratch, size, &iter); 144 145 KUNIT_EXPECT_EQ(test, copied, size); 146 KUNIT_EXPECT_EQ(test, iter.count, 0); 147 KUNIT_EXPECT_EQ(test, iter.nr_segs, 0); 148 149 /* Build the expected image in the scratch buffer. */ 150 patt = 0; 151 memset(scratch, 0, bufsize); 152 for (pr = kvec_test_ranges; pr->from >= 0; pr++) 153 for (i = pr->from; i < pr->to; i++) 154 scratch[i] = pattern(patt++); 155 156 /* Compare the images */ 157 for (i = 0; i < bufsize; i++) { 158 KUNIT_EXPECT_EQ_MSG(test, buffer[i], scratch[i], "at i=%x", i); 159 if (buffer[i] != scratch[i]) 160 return; 161 } 162 163 KUNIT_SUCCEED(test); 164 } 165 166 /* 167 * Test copying from a ITER_KVEC-type iterator. 168 */ 169 static void __init iov_kunit_copy_from_kvec(struct kunit *test) 170 { 171 const struct kvec_test_range *pr; 172 struct iov_iter iter; 173 struct page **spages, **bpages; 174 struct kvec kvec[8]; 175 u8 *scratch, *buffer; 176 size_t bufsize, npages, size, copied; 177 int i, j; 178 179 bufsize = 0x100000; 180 npages = bufsize / PAGE_SIZE; 181 182 buffer = iov_kunit_create_buffer(test, &bpages, npages); 183 for (i = 0; i < bufsize; i++) 184 buffer[i] = pattern(i); 185 186 scratch = iov_kunit_create_buffer(test, &spages, npages); 187 memset(scratch, 0, bufsize); 188 189 iov_kunit_load_kvec(test, &iter, WRITE, kvec, ARRAY_SIZE(kvec), 190 buffer, bufsize, kvec_test_ranges); 191 size = min(iter.count, bufsize); 192 193 copied = copy_from_iter(scratch, size, &iter); 194 195 KUNIT_EXPECT_EQ(test, copied, size); 196 KUNIT_EXPECT_EQ(test, iter.count, 0); 197 KUNIT_EXPECT_EQ(test, iter.nr_segs, 0); 198 199 /* Build the expected image in the main buffer. */ 200 i = 0; 201 memset(buffer, 0, bufsize); 202 for (pr = kvec_test_ranges; pr->from >= 0; pr++) { 203 for (j = pr->from; j < pr->to; j++) { 204 buffer[i++] = pattern(j); 205 if (i >= bufsize) 206 goto stop; 207 } 208 } 209 stop: 210 211 /* Compare the images */ 212 for (i = 0; i < bufsize; i++) { 213 KUNIT_EXPECT_EQ_MSG(test, scratch[i], buffer[i], "at i=%x", i); 214 if (scratch[i] != buffer[i]) 215 return; 216 } 217 218 KUNIT_SUCCEED(test); 219 } 220 221 struct bvec_test_range { 222 int page, from, to; 223 }; 224 225 static const struct bvec_test_range bvec_test_ranges[] = { 226 { 0, 0x0002, 0x0002 }, 227 { 1, 0x0027, 0x0893 }, 228 { 2, 0x0193, 0x0794 }, 229 { 3, 0x0000, 0x1000 }, 230 { 4, 0x0000, 0x1000 }, 231 { 5, 0x0000, 0x1000 }, 232 { 6, 0x0000, 0x0ffb }, 233 { 6, 0x0ffd, 0x0ffe }, 234 { -1, -1, -1 } 235 }; 236 237 static void __init iov_kunit_load_bvec(struct kunit *test, 238 struct iov_iter *iter, int dir, 239 struct bio_vec *bvec, unsigned int bvmax, 240 struct page **pages, size_t npages, 241 size_t bufsize, 242 const struct bvec_test_range *pr) 243 { 244 struct page *can_merge = NULL, *page; 245 size_t size = 0; 246 int i; 247 248 for (i = 0; i < bvmax; i++, pr++) { 249 if (pr->from < 0) 250 break; 251 KUNIT_ASSERT_LT(test, pr->page, npages); 252 KUNIT_ASSERT_LT(test, pr->page * PAGE_SIZE, bufsize); 253 KUNIT_ASSERT_GE(test, pr->from, 0); 254 KUNIT_ASSERT_GE(test, pr->to, pr->from); 255 KUNIT_ASSERT_LE(test, pr->to, PAGE_SIZE); 256 257 page = pages[pr->page]; 258 if (pr->from == 0 && pr->from != pr->to && page == can_merge) { 259 i--; 260 bvec[i].bv_len += pr->to; 261 } else { 262 bvec_set_page(&bvec[i], page, pr->to - pr->from, pr->from); 263 } 264 265 size += pr->to - pr->from; 266 if ((pr->to & ~PAGE_MASK) == 0) 267 can_merge = page + pr->to / PAGE_SIZE; 268 else 269 can_merge = NULL; 270 } 271 272 iov_iter_bvec(iter, dir, bvec, i, size); 273 } 274 275 /* 276 * Test copying to a ITER_BVEC-type iterator. 277 */ 278 static void __init iov_kunit_copy_to_bvec(struct kunit *test) 279 { 280 const struct bvec_test_range *pr; 281 struct iov_iter iter; 282 struct bio_vec bvec[8]; 283 struct page **spages, **bpages; 284 u8 *scratch, *buffer; 285 size_t bufsize, npages, size, copied; 286 int i, b, patt; 287 288 bufsize = 0x100000; 289 npages = bufsize / PAGE_SIZE; 290 291 scratch = iov_kunit_create_buffer(test, &spages, npages); 292 for (i = 0; i < bufsize; i++) 293 scratch[i] = pattern(i); 294 295 buffer = iov_kunit_create_buffer(test, &bpages, npages); 296 memset(buffer, 0, bufsize); 297 298 iov_kunit_load_bvec(test, &iter, READ, bvec, ARRAY_SIZE(bvec), 299 bpages, npages, bufsize, bvec_test_ranges); 300 size = iter.count; 301 302 copied = copy_to_iter(scratch, size, &iter); 303 304 KUNIT_EXPECT_EQ(test, copied, size); 305 KUNIT_EXPECT_EQ(test, iter.count, 0); 306 KUNIT_EXPECT_EQ(test, iter.nr_segs, 0); 307 308 /* Build the expected image in the scratch buffer. */ 309 b = 0; 310 patt = 0; 311 memset(scratch, 0, bufsize); 312 for (pr = bvec_test_ranges; pr->from >= 0; pr++, b++) { 313 u8 *p = scratch + pr->page * PAGE_SIZE; 314 315 for (i = pr->from; i < pr->to; i++) 316 p[i] = pattern(patt++); 317 } 318 319 /* Compare the images */ 320 for (i = 0; i < bufsize; i++) { 321 KUNIT_EXPECT_EQ_MSG(test, buffer[i], scratch[i], "at i=%x", i); 322 if (buffer[i] != scratch[i]) 323 return; 324 } 325 326 KUNIT_SUCCEED(test); 327 } 328 329 /* 330 * Test copying from a ITER_BVEC-type iterator. 331 */ 332 static void __init iov_kunit_copy_from_bvec(struct kunit *test) 333 { 334 const struct bvec_test_range *pr; 335 struct iov_iter iter; 336 struct bio_vec bvec[8]; 337 struct page **spages, **bpages; 338 u8 *scratch, *buffer; 339 size_t bufsize, npages, size, copied; 340 int i, j; 341 342 bufsize = 0x100000; 343 npages = bufsize / PAGE_SIZE; 344 345 buffer = iov_kunit_create_buffer(test, &bpages, npages); 346 for (i = 0; i < bufsize; i++) 347 buffer[i] = pattern(i); 348 349 scratch = iov_kunit_create_buffer(test, &spages, npages); 350 memset(scratch, 0, bufsize); 351 352 iov_kunit_load_bvec(test, &iter, WRITE, bvec, ARRAY_SIZE(bvec), 353 bpages, npages, bufsize, bvec_test_ranges); 354 size = iter.count; 355 356 copied = copy_from_iter(scratch, size, &iter); 357 358 KUNIT_EXPECT_EQ(test, copied, size); 359 KUNIT_EXPECT_EQ(test, iter.count, 0); 360 KUNIT_EXPECT_EQ(test, iter.nr_segs, 0); 361 362 /* Build the expected image in the main buffer. */ 363 i = 0; 364 memset(buffer, 0, bufsize); 365 for (pr = bvec_test_ranges; pr->from >= 0; pr++) { 366 size_t patt = pr->page * PAGE_SIZE; 367 368 for (j = pr->from; j < pr->to; j++) { 369 buffer[i++] = pattern(patt + j); 370 if (i >= bufsize) 371 goto stop; 372 } 373 } 374 stop: 375 376 /* Compare the images */ 377 for (i = 0; i < bufsize; i++) { 378 KUNIT_EXPECT_EQ_MSG(test, scratch[i], buffer[i], "at i=%x", i); 379 if (scratch[i] != buffer[i]) 380 return; 381 } 382 383 KUNIT_SUCCEED(test); 384 } 385 386 static void iov_kunit_destroy_folioq(void *data) 387 { 388 struct folio_queue *folioq, *next; 389 390 for (folioq = data; folioq; folioq = next) { 391 next = folioq->next; 392 kfree(folioq); 393 } 394 } 395 396 static void __init iov_kunit_load_folioq(struct kunit *test, 397 struct iov_iter *iter, int dir, 398 struct folio_queue *folioq, 399 struct page **pages, size_t npages) 400 { 401 struct folio_queue *p = folioq; 402 size_t size = 0; 403 int i; 404 405 for (i = 0; i < npages; i++) { 406 if (folioq_full(p)) { 407 p->next = kzalloc_obj(struct folio_queue); 408 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p->next); 409 folioq_init(p->next, 0); 410 p->next->prev = p; 411 p = p->next; 412 } 413 folioq_append(p, page_folio(pages[i])); 414 size += PAGE_SIZE; 415 } 416 iov_iter_folio_queue(iter, dir, folioq, 0, 0, size); 417 } 418 419 static struct folio_queue *iov_kunit_create_folioq(struct kunit *test) 420 { 421 struct folio_queue *folioq; 422 423 folioq = kzalloc_obj(struct folio_queue); 424 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folioq); 425 kunit_add_action_or_reset(test, iov_kunit_destroy_folioq, folioq); 426 folioq_init(folioq, 0); 427 return folioq; 428 } 429 430 /* 431 * Test copying to a ITER_FOLIOQ-type iterator. 432 */ 433 static void __init iov_kunit_copy_to_folioq(struct kunit *test) 434 { 435 const struct kvec_test_range *pr; 436 struct iov_iter iter; 437 struct folio_queue *folioq; 438 struct page **spages, **bpages; 439 u8 *scratch, *buffer; 440 size_t bufsize, npages, size, copied; 441 int i, patt; 442 443 bufsize = 0x100000; 444 npages = bufsize / PAGE_SIZE; 445 446 folioq = iov_kunit_create_folioq(test); 447 448 scratch = iov_kunit_create_buffer(test, &spages, npages); 449 for (i = 0; i < bufsize; i++) 450 scratch[i] = pattern(i); 451 452 buffer = iov_kunit_create_buffer(test, &bpages, npages); 453 memset(buffer, 0, bufsize); 454 455 iov_kunit_load_folioq(test, &iter, READ, folioq, bpages, npages); 456 457 i = 0; 458 for (pr = kvec_test_ranges; pr->from >= 0; pr++) { 459 size = pr->to - pr->from; 460 KUNIT_ASSERT_LE(test, pr->to, bufsize); 461 462 iov_iter_folio_queue(&iter, READ, folioq, 0, 0, pr->to); 463 iov_iter_advance(&iter, pr->from); 464 copied = copy_to_iter(scratch + i, size, &iter); 465 466 KUNIT_EXPECT_EQ(test, copied, size); 467 KUNIT_EXPECT_EQ(test, iter.count, 0); 468 KUNIT_EXPECT_EQ(test, iter.iov_offset, pr->to % PAGE_SIZE); 469 i += size; 470 if (test->status == KUNIT_FAILURE) 471 goto stop; 472 } 473 474 /* Build the expected image in the scratch buffer. */ 475 patt = 0; 476 memset(scratch, 0, bufsize); 477 for (pr = kvec_test_ranges; pr->from >= 0; pr++) 478 for (i = pr->from; i < pr->to; i++) 479 scratch[i] = pattern(patt++); 480 481 /* Compare the images */ 482 for (i = 0; i < bufsize; i++) { 483 KUNIT_EXPECT_EQ_MSG(test, buffer[i], scratch[i], "at i=%x", i); 484 if (buffer[i] != scratch[i]) 485 return; 486 } 487 488 stop: 489 KUNIT_SUCCEED(test); 490 } 491 492 /* 493 * Test copying from a ITER_FOLIOQ-type iterator. 494 */ 495 static void __init iov_kunit_copy_from_folioq(struct kunit *test) 496 { 497 const struct kvec_test_range *pr; 498 struct iov_iter iter; 499 struct folio_queue *folioq; 500 struct page **spages, **bpages; 501 u8 *scratch, *buffer; 502 size_t bufsize, npages, size, copied; 503 int i, j; 504 505 bufsize = 0x100000; 506 npages = bufsize / PAGE_SIZE; 507 508 folioq = iov_kunit_create_folioq(test); 509 510 buffer = iov_kunit_create_buffer(test, &bpages, npages); 511 for (i = 0; i < bufsize; i++) 512 buffer[i] = pattern(i); 513 514 scratch = iov_kunit_create_buffer(test, &spages, npages); 515 memset(scratch, 0, bufsize); 516 517 iov_kunit_load_folioq(test, &iter, READ, folioq, bpages, npages); 518 519 i = 0; 520 for (pr = kvec_test_ranges; pr->from >= 0; pr++) { 521 size = pr->to - pr->from; 522 KUNIT_ASSERT_LE(test, pr->to, bufsize); 523 524 iov_iter_folio_queue(&iter, WRITE, folioq, 0, 0, pr->to); 525 iov_iter_advance(&iter, pr->from); 526 copied = copy_from_iter(scratch + i, size, &iter); 527 528 KUNIT_EXPECT_EQ(test, copied, size); 529 KUNIT_EXPECT_EQ(test, iter.count, 0); 530 KUNIT_EXPECT_EQ(test, iter.iov_offset, pr->to % PAGE_SIZE); 531 i += size; 532 } 533 534 /* Build the expected image in the main buffer. */ 535 i = 0; 536 memset(buffer, 0, bufsize); 537 for (pr = kvec_test_ranges; pr->from >= 0; pr++) { 538 for (j = pr->from; j < pr->to; j++) { 539 buffer[i++] = pattern(j); 540 if (i >= bufsize) 541 goto stop; 542 } 543 } 544 stop: 545 546 /* Compare the images */ 547 for (i = 0; i < bufsize; i++) { 548 KUNIT_EXPECT_EQ_MSG(test, scratch[i], buffer[i], "at i=%x", i); 549 if (scratch[i] != buffer[i]) 550 return; 551 } 552 553 KUNIT_SUCCEED(test); 554 } 555 556 static void iov_kunit_destroy_xarray(void *data) 557 { 558 struct xarray *xarray = data; 559 560 xa_destroy(xarray); 561 kfree(xarray); 562 } 563 564 static void __init iov_kunit_load_xarray(struct kunit *test, 565 struct iov_iter *iter, int dir, 566 struct xarray *xarray, 567 struct page **pages, size_t npages) 568 { 569 size_t size = 0; 570 int i; 571 572 for (i = 0; i < npages; i++) { 573 void *x = xa_store(xarray, i, pages[i], GFP_KERNEL); 574 575 KUNIT_ASSERT_FALSE(test, xa_is_err(x)); 576 size += PAGE_SIZE; 577 } 578 iov_iter_xarray(iter, dir, xarray, 0, size); 579 } 580 581 static struct xarray *iov_kunit_create_xarray(struct kunit *test) 582 { 583 struct xarray *xarray; 584 585 xarray = kzalloc_obj(struct xarray); 586 xa_init(xarray); 587 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xarray); 588 kunit_add_action_or_reset(test, iov_kunit_destroy_xarray, xarray); 589 return xarray; 590 } 591 592 /* 593 * Test copying to a ITER_XARRAY-type iterator. 594 */ 595 static void __init iov_kunit_copy_to_xarray(struct kunit *test) 596 { 597 const struct kvec_test_range *pr; 598 struct iov_iter iter; 599 struct xarray *xarray; 600 struct page **spages, **bpages; 601 u8 *scratch, *buffer; 602 size_t bufsize, npages, size, copied; 603 int i, patt; 604 605 bufsize = 0x100000; 606 npages = bufsize / PAGE_SIZE; 607 608 xarray = iov_kunit_create_xarray(test); 609 610 scratch = iov_kunit_create_buffer(test, &spages, npages); 611 for (i = 0; i < bufsize; i++) 612 scratch[i] = pattern(i); 613 614 buffer = iov_kunit_create_buffer(test, &bpages, npages); 615 memset(buffer, 0, bufsize); 616 617 iov_kunit_load_xarray(test, &iter, READ, xarray, bpages, npages); 618 619 i = 0; 620 for (pr = kvec_test_ranges; pr->from >= 0; pr++) { 621 size = pr->to - pr->from; 622 KUNIT_ASSERT_LE(test, pr->to, bufsize); 623 624 iov_iter_xarray(&iter, READ, xarray, pr->from, size); 625 copied = copy_to_iter(scratch + i, size, &iter); 626 627 KUNIT_EXPECT_EQ(test, copied, size); 628 KUNIT_EXPECT_EQ(test, iter.count, 0); 629 KUNIT_EXPECT_EQ(test, iter.iov_offset, size); 630 i += size; 631 } 632 633 /* Build the expected image in the scratch buffer. */ 634 patt = 0; 635 memset(scratch, 0, bufsize); 636 for (pr = kvec_test_ranges; pr->from >= 0; pr++) 637 for (i = pr->from; i < pr->to; i++) 638 scratch[i] = pattern(patt++); 639 640 /* Compare the images */ 641 for (i = 0; i < bufsize; i++) { 642 KUNIT_EXPECT_EQ_MSG(test, buffer[i], scratch[i], "at i=%x", i); 643 if (buffer[i] != scratch[i]) 644 return; 645 } 646 647 KUNIT_SUCCEED(test); 648 } 649 650 /* 651 * Test copying from a ITER_XARRAY-type iterator. 652 */ 653 static void __init iov_kunit_copy_from_xarray(struct kunit *test) 654 { 655 const struct kvec_test_range *pr; 656 struct iov_iter iter; 657 struct xarray *xarray; 658 struct page **spages, **bpages; 659 u8 *scratch, *buffer; 660 size_t bufsize, npages, size, copied; 661 int i, j; 662 663 bufsize = 0x100000; 664 npages = bufsize / PAGE_SIZE; 665 666 xarray = iov_kunit_create_xarray(test); 667 668 buffer = iov_kunit_create_buffer(test, &bpages, npages); 669 for (i = 0; i < bufsize; i++) 670 buffer[i] = pattern(i); 671 672 scratch = iov_kunit_create_buffer(test, &spages, npages); 673 memset(scratch, 0, bufsize); 674 675 iov_kunit_load_xarray(test, &iter, READ, xarray, bpages, npages); 676 677 i = 0; 678 for (pr = kvec_test_ranges; pr->from >= 0; pr++) { 679 size = pr->to - pr->from; 680 KUNIT_ASSERT_LE(test, pr->to, bufsize); 681 682 iov_iter_xarray(&iter, WRITE, xarray, pr->from, size); 683 copied = copy_from_iter(scratch + i, size, &iter); 684 685 KUNIT_EXPECT_EQ(test, copied, size); 686 KUNIT_EXPECT_EQ(test, iter.count, 0); 687 KUNIT_EXPECT_EQ(test, iter.iov_offset, size); 688 i += size; 689 } 690 691 /* Build the expected image in the main buffer. */ 692 i = 0; 693 memset(buffer, 0, bufsize); 694 for (pr = kvec_test_ranges; pr->from >= 0; pr++) { 695 for (j = pr->from; j < pr->to; j++) { 696 buffer[i++] = pattern(j); 697 if (i >= bufsize) 698 goto stop; 699 } 700 } 701 stop: 702 703 /* Compare the images */ 704 for (i = 0; i < bufsize; i++) { 705 KUNIT_EXPECT_EQ_MSG(test, scratch[i], buffer[i], "at i=%x", i); 706 if (scratch[i] != buffer[i]) 707 return; 708 } 709 710 KUNIT_SUCCEED(test); 711 } 712 713 /* 714 * Test the extraction of ITER_KVEC-type iterators. 715 */ 716 static void __init iov_kunit_extract_pages_kvec(struct kunit *test) 717 { 718 const struct kvec_test_range *pr; 719 struct iov_iter iter; 720 struct page **bpages, *pagelist[8], **pages = pagelist; 721 struct kvec kvec[8]; 722 u8 *buffer; 723 ssize_t len; 724 size_t bufsize, size = 0, npages; 725 int i, from; 726 727 bufsize = 0x100000; 728 npages = bufsize / PAGE_SIZE; 729 730 buffer = iov_kunit_create_buffer(test, &bpages, npages); 731 732 iov_kunit_load_kvec(test, &iter, READ, kvec, ARRAY_SIZE(kvec), 733 buffer, bufsize, kvec_test_ranges); 734 size = iter.count; 735 736 pr = kvec_test_ranges; 737 from = pr->from; 738 do { 739 size_t offset0 = LONG_MAX; 740 741 for (i = 0; i < ARRAY_SIZE(pagelist); i++) 742 pagelist[i] = (void *)(unsigned long)0xaa55aa55aa55aa55ULL; 743 744 len = iov_iter_extract_pages(&iter, &pages, 100 * 1024, 745 ARRAY_SIZE(pagelist), 0, &offset0); 746 KUNIT_EXPECT_GE(test, len, 0); 747 if (len < 0) 748 break; 749 KUNIT_EXPECT_GE(test, (ssize_t)offset0, 0); 750 KUNIT_EXPECT_LT(test, offset0, PAGE_SIZE); 751 KUNIT_EXPECT_LE(test, len, size); 752 KUNIT_EXPECT_EQ(test, iter.count, size - len); 753 size -= len; 754 755 if (len == 0) 756 break; 757 758 for (i = 0; i < ARRAY_SIZE(pagelist); i++) { 759 struct page *p; 760 ssize_t part = min_t(ssize_t, len, PAGE_SIZE - offset0); 761 int ix; 762 763 KUNIT_ASSERT_GE(test, part, 0); 764 while (from == pr->to) { 765 pr++; 766 from = pr->from; 767 if (from < 0) 768 goto stop; 769 } 770 ix = from / PAGE_SIZE; 771 KUNIT_ASSERT_LT(test, ix, npages); 772 p = bpages[ix]; 773 KUNIT_EXPECT_PTR_EQ(test, pagelist[i], p); 774 KUNIT_EXPECT_EQ(test, offset0, from % PAGE_SIZE); 775 from += part; 776 len -= part; 777 KUNIT_ASSERT_GE(test, len, 0); 778 if (len == 0) 779 break; 780 offset0 = 0; 781 } 782 783 if (test->status == KUNIT_FAILURE) 784 break; 785 } while (iov_iter_count(&iter) > 0); 786 787 stop: 788 KUNIT_EXPECT_EQ(test, size, 0); 789 KUNIT_EXPECT_EQ(test, iter.count, 0); 790 KUNIT_SUCCEED(test); 791 } 792 793 /* 794 * Test the extraction of ITER_BVEC-type iterators. 795 */ 796 static void __init iov_kunit_extract_pages_bvec(struct kunit *test) 797 { 798 const struct bvec_test_range *pr; 799 struct iov_iter iter; 800 struct page **bpages, *pagelist[8], **pages = pagelist; 801 struct bio_vec bvec[8]; 802 ssize_t len; 803 size_t bufsize, size = 0, npages; 804 int i, from; 805 806 bufsize = 0x100000; 807 npages = bufsize / PAGE_SIZE; 808 809 iov_kunit_create_buffer(test, &bpages, npages); 810 iov_kunit_load_bvec(test, &iter, READ, bvec, ARRAY_SIZE(bvec), 811 bpages, npages, bufsize, bvec_test_ranges); 812 size = iter.count; 813 814 pr = bvec_test_ranges; 815 from = pr->from; 816 do { 817 size_t offset0 = LONG_MAX; 818 819 for (i = 0; i < ARRAY_SIZE(pagelist); i++) 820 pagelist[i] = (void *)(unsigned long)0xaa55aa55aa55aa55ULL; 821 822 len = iov_iter_extract_pages(&iter, &pages, 100 * 1024, 823 ARRAY_SIZE(pagelist), 0, &offset0); 824 KUNIT_EXPECT_GE(test, len, 0); 825 if (len < 0) 826 break; 827 KUNIT_EXPECT_GE(test, (ssize_t)offset0, 0); 828 KUNIT_EXPECT_LT(test, offset0, PAGE_SIZE); 829 KUNIT_EXPECT_LE(test, len, size); 830 KUNIT_EXPECT_EQ(test, iter.count, size - len); 831 size -= len; 832 833 if (len == 0) 834 break; 835 836 for (i = 0; i < ARRAY_SIZE(pagelist); i++) { 837 struct page *p; 838 ssize_t part = min_t(ssize_t, len, PAGE_SIZE - offset0); 839 int ix; 840 841 KUNIT_ASSERT_GE(test, part, 0); 842 while (from == pr->to) { 843 pr++; 844 from = pr->from; 845 if (from < 0) 846 goto stop; 847 } 848 ix = pr->page + from / PAGE_SIZE; 849 KUNIT_ASSERT_LT(test, ix, npages); 850 p = bpages[ix]; 851 KUNIT_EXPECT_PTR_EQ(test, pagelist[i], p); 852 KUNIT_EXPECT_EQ(test, offset0, from % PAGE_SIZE); 853 from += part; 854 len -= part; 855 KUNIT_ASSERT_GE(test, len, 0); 856 if (len == 0) 857 break; 858 offset0 = 0; 859 } 860 861 if (test->status == KUNIT_FAILURE) 862 break; 863 } while (iov_iter_count(&iter) > 0); 864 865 stop: 866 KUNIT_EXPECT_EQ(test, size, 0); 867 KUNIT_EXPECT_EQ(test, iter.count, 0); 868 KUNIT_SUCCEED(test); 869 } 870 871 /* 872 * Test the extraction of ITER_FOLIOQ-type iterators. 873 */ 874 static void __init iov_kunit_extract_pages_folioq(struct kunit *test) 875 { 876 const struct kvec_test_range *pr; 877 struct folio_queue *folioq; 878 struct iov_iter iter; 879 struct page **bpages, *pagelist[8], **pages = pagelist; 880 ssize_t len; 881 size_t bufsize, size = 0, npages; 882 int i, from; 883 884 bufsize = 0x100000; 885 npages = bufsize / PAGE_SIZE; 886 887 folioq = iov_kunit_create_folioq(test); 888 889 iov_kunit_create_buffer(test, &bpages, npages); 890 iov_kunit_load_folioq(test, &iter, READ, folioq, bpages, npages); 891 892 for (pr = kvec_test_ranges; pr->from >= 0; pr++) { 893 from = pr->from; 894 size = pr->to - from; 895 KUNIT_ASSERT_LE(test, pr->to, bufsize); 896 897 iov_iter_folio_queue(&iter, WRITE, folioq, 0, 0, pr->to); 898 iov_iter_advance(&iter, from); 899 900 do { 901 size_t offset0 = LONG_MAX; 902 903 for (i = 0; i < ARRAY_SIZE(pagelist); i++) 904 pagelist[i] = (void *)(unsigned long)0xaa55aa55aa55aa55ULL; 905 906 len = iov_iter_extract_pages(&iter, &pages, 100 * 1024, 907 ARRAY_SIZE(pagelist), 0, &offset0); 908 KUNIT_EXPECT_GE(test, len, 0); 909 if (len < 0) 910 break; 911 KUNIT_EXPECT_LE(test, len, size); 912 KUNIT_EXPECT_EQ(test, iter.count, size - len); 913 if (len == 0) 914 break; 915 size -= len; 916 KUNIT_EXPECT_GE(test, (ssize_t)offset0, 0); 917 KUNIT_EXPECT_LT(test, offset0, PAGE_SIZE); 918 919 for (i = 0; i < ARRAY_SIZE(pagelist); i++) { 920 struct page *p; 921 ssize_t part = min_t(ssize_t, len, PAGE_SIZE - offset0); 922 int ix; 923 924 KUNIT_ASSERT_GE(test, part, 0); 925 ix = from / PAGE_SIZE; 926 KUNIT_ASSERT_LT(test, ix, npages); 927 p = bpages[ix]; 928 KUNIT_EXPECT_PTR_EQ(test, pagelist[i], p); 929 KUNIT_EXPECT_EQ(test, offset0, from % PAGE_SIZE); 930 from += part; 931 len -= part; 932 KUNIT_ASSERT_GE(test, len, 0); 933 if (len == 0) 934 break; 935 offset0 = 0; 936 } 937 938 if (test->status == KUNIT_FAILURE) 939 goto stop; 940 } while (iov_iter_count(&iter) > 0); 941 942 KUNIT_EXPECT_EQ(test, size, 0); 943 KUNIT_EXPECT_EQ(test, iter.count, 0); 944 } 945 946 stop: 947 KUNIT_SUCCEED(test); 948 } 949 950 /* 951 * Test the extraction of ITER_XARRAY-type iterators. 952 */ 953 static void __init iov_kunit_extract_pages_xarray(struct kunit *test) 954 { 955 const struct kvec_test_range *pr; 956 struct iov_iter iter; 957 struct xarray *xarray; 958 struct page **bpages, *pagelist[8], **pages = pagelist; 959 ssize_t len; 960 size_t bufsize, size = 0, npages; 961 int i, from; 962 963 bufsize = 0x100000; 964 npages = bufsize / PAGE_SIZE; 965 966 xarray = iov_kunit_create_xarray(test); 967 968 iov_kunit_create_buffer(test, &bpages, npages); 969 iov_kunit_load_xarray(test, &iter, READ, xarray, bpages, npages); 970 971 for (pr = kvec_test_ranges; pr->from >= 0; pr++) { 972 from = pr->from; 973 size = pr->to - from; 974 KUNIT_ASSERT_LE(test, pr->to, bufsize); 975 976 iov_iter_xarray(&iter, WRITE, xarray, from, size); 977 978 do { 979 size_t offset0 = LONG_MAX; 980 981 for (i = 0; i < ARRAY_SIZE(pagelist); i++) 982 pagelist[i] = (void *)(unsigned long)0xaa55aa55aa55aa55ULL; 983 984 len = iov_iter_extract_pages(&iter, &pages, 100 * 1024, 985 ARRAY_SIZE(pagelist), 0, &offset0); 986 KUNIT_EXPECT_GE(test, len, 0); 987 if (len < 0) 988 break; 989 KUNIT_EXPECT_LE(test, len, size); 990 KUNIT_EXPECT_EQ(test, iter.count, size - len); 991 if (len == 0) 992 break; 993 size -= len; 994 KUNIT_EXPECT_GE(test, (ssize_t)offset0, 0); 995 KUNIT_EXPECT_LT(test, offset0, PAGE_SIZE); 996 997 for (i = 0; i < ARRAY_SIZE(pagelist); i++) { 998 struct page *p; 999 ssize_t part = min_t(ssize_t, len, PAGE_SIZE - offset0); 1000 int ix; 1001 1002 KUNIT_ASSERT_GE(test, part, 0); 1003 ix = from / PAGE_SIZE; 1004 KUNIT_ASSERT_LT(test, ix, npages); 1005 p = bpages[ix]; 1006 KUNIT_EXPECT_PTR_EQ(test, pagelist[i], p); 1007 KUNIT_EXPECT_EQ(test, offset0, from % PAGE_SIZE); 1008 from += part; 1009 len -= part; 1010 KUNIT_ASSERT_GE(test, len, 0); 1011 if (len == 0) 1012 break; 1013 offset0 = 0; 1014 } 1015 1016 if (test->status == KUNIT_FAILURE) 1017 goto stop; 1018 } while (iov_iter_count(&iter) > 0); 1019 1020 KUNIT_EXPECT_EQ(test, size, 0); 1021 KUNIT_EXPECT_EQ(test, iter.count, 0); 1022 KUNIT_EXPECT_EQ(test, iter.iov_offset, pr->to - pr->from); 1023 } 1024 1025 stop: 1026 KUNIT_SUCCEED(test); 1027 } 1028 1029 struct iov_kunit_iter_to_sg_data { 1030 struct sg_table *sgt; 1031 u8 *buffer, *scratch; 1032 u8 __user *ubuf; 1033 struct page **pages; 1034 size_t npages; 1035 }; 1036 1037 static void __init 1038 iov_kunit_iter_unpin_sgt(void *data) 1039 { 1040 struct sg_table *sgt = data; 1041 1042 for (unsigned int i = 0; i < sgt->nents; ++i) 1043 unpin_user_page(sg_page(&sgt->sgl[i])); 1044 } 1045 1046 static void __init 1047 iov_kunit_iter_to_sg_init(struct kunit *test, size_t bufsize, bool user, 1048 struct iov_kunit_iter_to_sg_data *data) 1049 { 1050 struct page **spages; 1051 struct scatterlist *sg; 1052 unsigned long uaddr; 1053 size_t i; 1054 1055 data->npages = bufsize / PAGE_SIZE; 1056 sg = kunit_kmalloc_array(test, data->npages, sizeof(*sg), GFP_KERNEL); 1057 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sg); 1058 sg_init_table(sg, data->npages); 1059 data->sgt = kunit_kzalloc(test, sizeof(*data->sgt), GFP_KERNEL); 1060 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, data->sgt); 1061 data->sgt->orig_nents = 0; 1062 data->sgt->sgl = sg; 1063 1064 data->buffer = NULL; 1065 data->ubuf = NULL; 1066 if (user) { 1067 uaddr = kunit_vm_mmap(test, NULL, 0, bufsize, 1068 PROT_READ | PROT_WRITE, 1069 MAP_ANONYMOUS | MAP_PRIVATE, 0); 1070 KUNIT_ASSERT_NE(test, uaddr, 0); 1071 data->ubuf = (u8 __user *)uaddr; 1072 for (i = 0; i < bufsize; ++i) 1073 put_user(pattern(i), data->ubuf + i); 1074 } else { 1075 data->buffer = iov_kunit_create_buffer(test, &data->pages, 1076 data->npages); 1077 for (i = 0; i < bufsize; ++i) 1078 data->buffer[i] = pattern(i); 1079 } 1080 data->scratch = iov_kunit_create_buffer(test, &spages, data->npages); 1081 memset(data->scratch, 0, bufsize); 1082 } 1083 1084 static void __init 1085 iov_kunit_iter_to_sg_check(struct kunit *test, struct iov_iter *iter, 1086 size_t bufsize, 1087 struct iov_kunit_iter_to_sg_data *data) 1088 { 1089 static const size_t tail = 16 * PAGE_SIZE; 1090 size_t i; 1091 1092 KUNIT_ASSERT_LT(test, tail, bufsize); 1093 1094 if (iov_iter_extract_will_pin(iter)) 1095 kunit_add_action_or_reset(test, iov_kunit_iter_unpin_sgt, 1096 data->sgt); 1097 1098 i = extract_iter_to_sg(iter, bufsize, data->sgt, 0, 0); 1099 KUNIT_ASSERT_EQ(test, i, 0); 1100 KUNIT_ASSERT_EQ(test, data->sgt->nents, 0); 1101 1102 i = extract_iter_to_sg(iter, bufsize - tail, data->sgt, 1, 0); 1103 KUNIT_ASSERT_LE(test, i, bufsize - tail); 1104 KUNIT_ASSERT_EQ(test, data->sgt->nents, 1); 1105 1106 i += extract_iter_to_sg(iter, bufsize - tail - i, data->sgt, 1107 data->npages - data->sgt->nents, 0); 1108 KUNIT_ASSERT_EQ(test, i, bufsize - tail); 1109 KUNIT_ASSERT_LE(test, data->sgt->nents, data->npages); 1110 1111 i += extract_iter_to_sg(iter, tail, data->sgt, 1112 data->npages - data->sgt->nents, 0); 1113 KUNIT_ASSERT_EQ(test, i, bufsize); 1114 KUNIT_ASSERT_LE(test, data->sgt->nents, data->npages); 1115 1116 sg_mark_end(&data->sgt->sgl[data->sgt->nents - 1]); 1117 1118 i = sg_copy_to_buffer(data->sgt->sgl, data->sgt->nents, 1119 data->scratch, bufsize); 1120 KUNIT_ASSERT_EQ(test, i, bufsize); 1121 1122 for (i = 0; i < bufsize; ++i) { 1123 KUNIT_EXPECT_EQ_MSG(test, data->scratch[i], pattern(i), 1124 "at i=%zx", i); 1125 if (data->scratch[i] != pattern(i)) 1126 break; 1127 } 1128 1129 KUNIT_EXPECT_EQ(test, i, bufsize); 1130 } 1131 1132 static void __init iov_kunit_iter_to_sg_kvec(struct kunit *test) 1133 { 1134 struct iov_kunit_iter_to_sg_data data; 1135 struct iov_iter iter; 1136 struct kvec kvec; 1137 size_t bufsize; 1138 1139 bufsize = 0x200000; 1140 iov_kunit_iter_to_sg_init(test, bufsize, false, &data); 1141 1142 kvec.iov_base = data.buffer; 1143 kvec.iov_len = bufsize; 1144 iov_iter_kvec(&iter, READ, &kvec, 1, bufsize); 1145 1146 iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data); 1147 } 1148 1149 static void __init iov_kunit_iter_to_sg_bvec(struct kunit *test) 1150 { 1151 struct iov_kunit_iter_to_sg_data data; 1152 struct page *p, *can_merge = NULL; 1153 size_t i, k, bufsize; 1154 struct bio_vec *bvec; 1155 struct iov_iter iter; 1156 1157 bufsize = 0x200000; 1158 iov_kunit_iter_to_sg_init(test, bufsize, false, &data); 1159 1160 bvec = kunit_kmalloc_array(test, data.npages, sizeof(*bvec), 1161 GFP_KERNEL); 1162 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bvec); 1163 k = 0; 1164 for (i = 0; i < data.npages; ++i) { 1165 p = data.pages[i]; 1166 if (p == can_merge) 1167 bvec[k-1].bv_len += PAGE_SIZE; 1168 else 1169 bvec_set_page(&bvec[k++], p, PAGE_SIZE, 0); 1170 can_merge = p + 1; 1171 } 1172 iov_iter_bvec(&iter, READ, bvec, k, bufsize); 1173 1174 iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data); 1175 } 1176 1177 static void __init iov_kunit_iter_to_sg_folioq(struct kunit *test) 1178 { 1179 struct iov_kunit_iter_to_sg_data data; 1180 struct folio_queue *folioq; 1181 struct iov_iter iter; 1182 size_t bufsize; 1183 1184 bufsize = 0x200000; 1185 iov_kunit_iter_to_sg_init(test, bufsize, false, &data); 1186 1187 folioq = iov_kunit_create_folioq(test); 1188 iov_kunit_load_folioq(test, &iter, READ, folioq, data.pages, 1189 data.npages); 1190 1191 iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data); 1192 } 1193 1194 static void __init iov_kunit_iter_to_sg_xarray(struct kunit *test) 1195 { 1196 struct iov_kunit_iter_to_sg_data data; 1197 struct xarray *xarray; 1198 struct iov_iter iter; 1199 size_t bufsize; 1200 1201 bufsize = 0x200000; 1202 iov_kunit_iter_to_sg_init(test, bufsize, false, &data); 1203 1204 xarray = iov_kunit_create_xarray(test); 1205 iov_kunit_load_xarray(test, &iter, READ, xarray, data.pages, 1206 data.npages); 1207 1208 iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data); 1209 } 1210 1211 static void __init iov_kunit_iter_to_sg_ubuf(struct kunit *test) 1212 { 1213 struct iov_kunit_iter_to_sg_data data; 1214 struct iov_iter iter; 1215 size_t bufsize; 1216 1217 bufsize = 0x200000; 1218 iov_kunit_iter_to_sg_init(test, bufsize, true, &data); 1219 1220 iov_iter_ubuf(&iter, READ, data.ubuf, bufsize); 1221 1222 iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data); 1223 } 1224 1225 static struct kunit_case __refdata iov_kunit_cases[] = { 1226 KUNIT_CASE(iov_kunit_copy_to_kvec), 1227 KUNIT_CASE(iov_kunit_copy_from_kvec), 1228 KUNIT_CASE(iov_kunit_copy_to_bvec), 1229 KUNIT_CASE(iov_kunit_copy_from_bvec), 1230 KUNIT_CASE(iov_kunit_copy_to_folioq), 1231 KUNIT_CASE(iov_kunit_copy_from_folioq), 1232 KUNIT_CASE(iov_kunit_copy_to_xarray), 1233 KUNIT_CASE(iov_kunit_copy_from_xarray), 1234 KUNIT_CASE(iov_kunit_extract_pages_kvec), 1235 KUNIT_CASE(iov_kunit_extract_pages_bvec), 1236 KUNIT_CASE(iov_kunit_extract_pages_folioq), 1237 KUNIT_CASE(iov_kunit_extract_pages_xarray), 1238 KUNIT_CASE(iov_kunit_iter_to_sg_kvec), 1239 KUNIT_CASE(iov_kunit_iter_to_sg_bvec), 1240 KUNIT_CASE(iov_kunit_iter_to_sg_folioq), 1241 KUNIT_CASE(iov_kunit_iter_to_sg_xarray), 1242 KUNIT_CASE(iov_kunit_iter_to_sg_ubuf), 1243 {} 1244 }; 1245 1246 static struct kunit_suite iov_kunit_suite = { 1247 .name = "iov_iter", 1248 .test_cases = iov_kunit_cases, 1249 }; 1250 1251 kunit_test_suites(&iov_kunit_suite); 1252