1 // SPDX-License-Identifier: GPL-2.0-only 2 /* I/O iterator tests. This can only test kernel-backed iterator types. 3 * 4 * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 10 #include <linux/module.h> 11 #include <linux/vmalloc.h> 12 #include <linux/mm.h> 13 #include <linux/uio.h> 14 #include <linux/bvec.h> 15 #include <linux/folio_queue.h> 16 #include <linux/scatterlist.h> 17 #include <linux/minmax.h> 18 #include <linux/mman.h> 19 #include <kunit/test.h> 20 21 MODULE_DESCRIPTION("iov_iter testing"); 22 MODULE_AUTHOR("David Howells <dhowells@redhat.com>"); 23 MODULE_LICENSE("GPL"); 24 25 struct kvec_test_range { 26 int from, to; 27 }; 28 29 static const struct kvec_test_range kvec_test_ranges[] = { 30 { 0x00002, 0x00002 }, 31 { 0x00027, 0x03000 }, 32 { 0x05193, 0x18794 }, 33 { 0x20000, 0x20000 }, 34 { 0x20000, 0x24000 }, 35 { 0x24000, 0x27001 }, 36 { 0x29000, 0xffffb }, 37 { 0xffffd, 0xffffe }, 38 { -1 } 39 }; 40 41 static inline u8 pattern(unsigned long x) 42 { 43 return (u8)x + (u8)(x >> 8) + (u8)(x >> 16); 44 } 45 46 static void iov_kunit_unmap(void *data) 47 { 48 vfree(data); 49 } 50 51 static void *__init iov_kunit_create_buffer(struct kunit *test, 52 struct page ***ppages, 53 size_t npages) 54 { 55 struct page **pages; 56 unsigned long got, last; 57 void *buffer; 58 unsigned int i; 59 60 pages = kzalloc_objs(struct page *, npages, GFP_KERNEL); 61 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pages); 62 *ppages = pages; 63 64 got = 0; 65 while (true) { 66 last = got; 67 got = alloc_pages_bulk(GFP_KERNEL, npages, pages); 68 69 if (last == got || got == npages) 70 break; 71 } 72 73 if (got != npages) { 74 release_pages(pages, got); 75 kvfree(pages); 76 KUNIT_ASSERT_EQ(test, got, npages); 77 } 78 /* Make sure that we don't get a physically contiguous buffer. */ 79 for (i = 0; i < npages / 4; ++i) 80 swap(pages[i], pages[i + npages / 2]); 81 82 buffer = vmap(pages, npages, VM_MAP | VM_MAP_PUT_PAGES, PAGE_KERNEL); 83 if (buffer == NULL) { 84 release_pages(pages, got); 85 kvfree(pages); 86 } 87 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buffer); 88 89 kunit_add_action_or_reset(test, iov_kunit_unmap, buffer); 90 return buffer; 91 } 92 93 static void __init iov_kunit_load_kvec(struct kunit *test, 94 struct iov_iter *iter, int dir, 95 struct kvec *kvec, unsigned int kvmax, 96 void *buffer, size_t bufsize, 97 const struct kvec_test_range *pr) 98 { 99 size_t size = 0; 100 int i; 101 102 for (i = 0; i < kvmax; i++, pr++) { 103 if (pr->from < 0) 104 break; 105 KUNIT_ASSERT_GE(test, pr->to, pr->from); 106 KUNIT_ASSERT_LE(test, pr->to, bufsize); 107 kvec[i].iov_base = buffer + pr->from; 108 kvec[i].iov_len = pr->to - pr->from; 109 size += pr->to - pr->from; 110 } 111 KUNIT_ASSERT_LE(test, size, bufsize); 112 113 iov_iter_kvec(iter, dir, kvec, i, size); 114 } 115 116 /* 117 * Test copying to a ITER_KVEC-type iterator. 118 */ 119 static void __init iov_kunit_copy_to_kvec(struct kunit *test) 120 { 121 const struct kvec_test_range *pr; 122 struct iov_iter iter; 123 struct page **spages, **bpages; 124 struct kvec kvec[8]; 125 u8 *scratch, *buffer; 126 size_t bufsize, npages, size, copied; 127 int i, patt; 128 129 bufsize = 0x100000; 130 npages = bufsize / PAGE_SIZE; 131 132 scratch = iov_kunit_create_buffer(test, &spages, npages); 133 for (i = 0; i < bufsize; i++) 134 scratch[i] = pattern(i); 135 136 buffer = iov_kunit_create_buffer(test, &bpages, npages); 137 memset(buffer, 0, bufsize); 138 139 iov_kunit_load_kvec(test, &iter, READ, kvec, ARRAY_SIZE(kvec), 140 buffer, bufsize, kvec_test_ranges); 141 size = iter.count; 142 143 copied = copy_to_iter(scratch, size, &iter); 144 145 KUNIT_EXPECT_EQ(test, copied, size); 146 KUNIT_EXPECT_EQ(test, iter.count, 0); 147 KUNIT_EXPECT_EQ(test, iter.nr_segs, 0); 148 149 /* Build the expected image in the scratch buffer. */ 150 patt = 0; 151 memset(scratch, 0, bufsize); 152 for (pr = kvec_test_ranges; pr->from >= 0; pr++) 153 for (i = pr->from; i < pr->to; i++) 154 scratch[i] = pattern(patt++); 155 156 /* Compare the images */ 157 for (i = 0; i < bufsize; i++) { 158 KUNIT_EXPECT_EQ_MSG(test, buffer[i], scratch[i], "at i=%x", i); 159 if (buffer[i] != scratch[i]) 160 return; 161 } 162 163 KUNIT_SUCCEED(test); 164 } 165 166 /* 167 * Test copying from a ITER_KVEC-type iterator. 168 */ 169 static void __init iov_kunit_copy_from_kvec(struct kunit *test) 170 { 171 const struct kvec_test_range *pr; 172 struct iov_iter iter; 173 struct page **spages, **bpages; 174 struct kvec kvec[8]; 175 u8 *scratch, *buffer; 176 size_t bufsize, npages, size, copied; 177 int i, j; 178 179 bufsize = 0x100000; 180 npages = bufsize / PAGE_SIZE; 181 182 buffer = iov_kunit_create_buffer(test, &bpages, npages); 183 for (i = 0; i < bufsize; i++) 184 buffer[i] = pattern(i); 185 186 scratch = iov_kunit_create_buffer(test, &spages, npages); 187 memset(scratch, 0, bufsize); 188 189 iov_kunit_load_kvec(test, &iter, WRITE, kvec, ARRAY_SIZE(kvec), 190 buffer, bufsize, kvec_test_ranges); 191 size = min(iter.count, bufsize); 192 193 copied = copy_from_iter(scratch, size, &iter); 194 195 KUNIT_EXPECT_EQ(test, copied, size); 196 KUNIT_EXPECT_EQ(test, iter.count, 0); 197 KUNIT_EXPECT_EQ(test, iter.nr_segs, 0); 198 199 /* Build the expected image in the main buffer. */ 200 i = 0; 201 memset(buffer, 0, bufsize); 202 for (pr = kvec_test_ranges; pr->from >= 0; pr++) { 203 for (j = pr->from; j < pr->to; j++) { 204 buffer[i++] = pattern(j); 205 if (i >= bufsize) 206 goto stop; 207 } 208 } 209 stop: 210 211 /* Compare the images */ 212 for (i = 0; i < bufsize; i++) { 213 KUNIT_EXPECT_EQ_MSG(test, scratch[i], buffer[i], "at i=%x", i); 214 if (scratch[i] != buffer[i]) 215 return; 216 } 217 218 KUNIT_SUCCEED(test); 219 } 220 221 struct bvec_test_range { 222 int page, from, to; 223 }; 224 225 static const struct bvec_test_range bvec_test_ranges[] = { 226 { 0, 0x0002, 0x0002 }, 227 { 1, 0x0027, 0x0893 }, 228 { 2, 0x0193, 0x0794 }, 229 { 3, 0x0000, 0x1000 }, 230 { 4, 0x0000, 0x1000 }, 231 { 5, 0x0000, 0x1000 }, 232 { 6, 0x0000, 0x0ffb }, 233 { 6, 0x0ffd, 0x0ffe }, 234 { -1, -1, -1 } 235 }; 236 237 static void __init iov_kunit_load_bvec(struct kunit *test, 238 struct iov_iter *iter, int dir, 239 struct bio_vec *bvec, unsigned int bvmax, 240 struct page **pages, size_t npages, 241 size_t bufsize, 242 const struct bvec_test_range *pr) 243 { 244 struct page *can_merge = NULL, *page; 245 size_t size = 0; 246 int i; 247 248 for (i = 0; i < bvmax; i++, pr++) { 249 if (pr->from < 0) 250 break; 251 KUNIT_ASSERT_LT(test, pr->page, npages); 252 KUNIT_ASSERT_LT(test, pr->page * PAGE_SIZE, bufsize); 253 KUNIT_ASSERT_GE(test, pr->from, 0); 254 KUNIT_ASSERT_GE(test, pr->to, pr->from); 255 KUNIT_ASSERT_LE(test, pr->to, PAGE_SIZE); 256 257 page = pages[pr->page]; 258 if (pr->from == 0 && pr->from != pr->to && page == can_merge) { 259 i--; 260 bvec[i].bv_len += pr->to; 261 } else { 262 bvec_set_page(&bvec[i], page, pr->to - pr->from, pr->from); 263 } 264 265 size += pr->to - pr->from; 266 if ((pr->to & ~PAGE_MASK) == 0) 267 can_merge = page + pr->to / PAGE_SIZE; 268 else 269 can_merge = NULL; 270 } 271 272 iov_iter_bvec(iter, dir, bvec, i, size); 273 } 274 275 /* 276 * Test copying to a ITER_BVEC-type iterator. 277 */ 278 static void __init iov_kunit_copy_to_bvec(struct kunit *test) 279 { 280 const struct bvec_test_range *pr; 281 struct iov_iter iter; 282 struct bio_vec bvec[8]; 283 struct page **spages, **bpages; 284 u8 *scratch, *buffer; 285 size_t bufsize, npages, size, copied; 286 int i, patt; 287 288 bufsize = 0x100000; 289 npages = bufsize / PAGE_SIZE; 290 291 scratch = iov_kunit_create_buffer(test, &spages, npages); 292 for (i = 0; i < bufsize; i++) 293 scratch[i] = pattern(i); 294 295 buffer = iov_kunit_create_buffer(test, &bpages, npages); 296 memset(buffer, 0, bufsize); 297 298 iov_kunit_load_bvec(test, &iter, READ, bvec, ARRAY_SIZE(bvec), 299 bpages, npages, bufsize, bvec_test_ranges); 300 size = iter.count; 301 302 copied = copy_to_iter(scratch, size, &iter); 303 304 KUNIT_EXPECT_EQ(test, copied, size); 305 KUNIT_EXPECT_EQ(test, iter.count, 0); 306 KUNIT_EXPECT_EQ(test, iter.nr_segs, 0); 307 308 /* Build the expected image in the scratch buffer. */ 309 patt = 0; 310 memset(scratch, 0, bufsize); 311 for (pr = bvec_test_ranges; pr->from >= 0; pr++) { 312 u8 *p = scratch + pr->page * PAGE_SIZE; 313 314 for (i = pr->from; i < pr->to; i++) 315 p[i] = pattern(patt++); 316 } 317 318 /* Compare the images */ 319 for (i = 0; i < bufsize; i++) { 320 KUNIT_EXPECT_EQ_MSG(test, buffer[i], scratch[i], "at i=%x", i); 321 if (buffer[i] != scratch[i]) 322 return; 323 } 324 325 KUNIT_SUCCEED(test); 326 } 327 328 /* 329 * Test copying from a ITER_BVEC-type iterator. 330 */ 331 static void __init iov_kunit_copy_from_bvec(struct kunit *test) 332 { 333 const struct bvec_test_range *pr; 334 struct iov_iter iter; 335 struct bio_vec bvec[8]; 336 struct page **spages, **bpages; 337 u8 *scratch, *buffer; 338 size_t bufsize, npages, size, copied; 339 int i, j; 340 341 bufsize = 0x100000; 342 npages = bufsize / PAGE_SIZE; 343 344 buffer = iov_kunit_create_buffer(test, &bpages, npages); 345 for (i = 0; i < bufsize; i++) 346 buffer[i] = pattern(i); 347 348 scratch = iov_kunit_create_buffer(test, &spages, npages); 349 memset(scratch, 0, bufsize); 350 351 iov_kunit_load_bvec(test, &iter, WRITE, bvec, ARRAY_SIZE(bvec), 352 bpages, npages, bufsize, bvec_test_ranges); 353 size = iter.count; 354 355 copied = copy_from_iter(scratch, size, &iter); 356 357 KUNIT_EXPECT_EQ(test, copied, size); 358 KUNIT_EXPECT_EQ(test, iter.count, 0); 359 KUNIT_EXPECT_EQ(test, iter.nr_segs, 0); 360 361 /* Build the expected image in the main buffer. */ 362 i = 0; 363 memset(buffer, 0, bufsize); 364 for (pr = bvec_test_ranges; pr->from >= 0; pr++) { 365 size_t patt = pr->page * PAGE_SIZE; 366 367 for (j = pr->from; j < pr->to; j++) { 368 buffer[i++] = pattern(patt + j); 369 if (i >= bufsize) 370 goto stop; 371 } 372 } 373 stop: 374 375 /* Compare the images */ 376 for (i = 0; i < bufsize; i++) { 377 KUNIT_EXPECT_EQ_MSG(test, scratch[i], buffer[i], "at i=%x", i); 378 if (scratch[i] != buffer[i]) 379 return; 380 } 381 382 KUNIT_SUCCEED(test); 383 } 384 385 static void iov_kunit_destroy_folioq(void *data) 386 { 387 struct folio_queue *folioq, *next; 388 389 for (folioq = data; folioq; folioq = next) { 390 next = folioq->next; 391 kfree(folioq); 392 } 393 } 394 395 static void __init iov_kunit_load_folioq(struct kunit *test, 396 struct iov_iter *iter, int dir, 397 struct folio_queue *folioq, 398 struct page **pages, size_t npages) 399 { 400 struct folio_queue *p = folioq; 401 size_t size = 0; 402 int i; 403 404 for (i = 0; i < npages; i++) { 405 if (folioq_full(p)) { 406 p->next = kzalloc_obj(struct folio_queue); 407 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, p->next); 408 folioq_init(p->next, 0); 409 p->next->prev = p; 410 p = p->next; 411 } 412 folioq_append(p, page_folio(pages[i])); 413 size += PAGE_SIZE; 414 } 415 iov_iter_folio_queue(iter, dir, folioq, 0, 0, size); 416 } 417 418 static struct folio_queue *iov_kunit_create_folioq(struct kunit *test) 419 { 420 struct folio_queue *folioq; 421 422 folioq = kzalloc_obj(struct folio_queue); 423 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, folioq); 424 kunit_add_action_or_reset(test, iov_kunit_destroy_folioq, folioq); 425 folioq_init(folioq, 0); 426 return folioq; 427 } 428 429 /* 430 * Test copying to a ITER_FOLIOQ-type iterator. 431 */ 432 static void __init iov_kunit_copy_to_folioq(struct kunit *test) 433 { 434 const struct kvec_test_range *pr; 435 struct iov_iter iter; 436 struct folio_queue *folioq; 437 struct page **spages, **bpages; 438 u8 *scratch, *buffer; 439 size_t bufsize, npages, size, copied; 440 int i, patt; 441 442 bufsize = 0x100000; 443 npages = bufsize / PAGE_SIZE; 444 445 folioq = iov_kunit_create_folioq(test); 446 447 scratch = iov_kunit_create_buffer(test, &spages, npages); 448 for (i = 0; i < bufsize; i++) 449 scratch[i] = pattern(i); 450 451 buffer = iov_kunit_create_buffer(test, &bpages, npages); 452 memset(buffer, 0, bufsize); 453 454 iov_kunit_load_folioq(test, &iter, READ, folioq, bpages, npages); 455 456 i = 0; 457 for (pr = kvec_test_ranges; pr->from >= 0; pr++) { 458 size = pr->to - pr->from; 459 KUNIT_ASSERT_LE(test, pr->to, bufsize); 460 461 iov_iter_folio_queue(&iter, READ, folioq, 0, 0, pr->to); 462 iov_iter_advance(&iter, pr->from); 463 copied = copy_to_iter(scratch + i, size, &iter); 464 465 KUNIT_EXPECT_EQ(test, copied, size); 466 KUNIT_EXPECT_EQ(test, iter.count, 0); 467 KUNIT_EXPECT_EQ(test, iter.iov_offset, pr->to % PAGE_SIZE); 468 i += size; 469 if (test->status == KUNIT_FAILURE) 470 goto stop; 471 } 472 473 /* Build the expected image in the scratch buffer. */ 474 patt = 0; 475 memset(scratch, 0, bufsize); 476 for (pr = kvec_test_ranges; pr->from >= 0; pr++) 477 for (i = pr->from; i < pr->to; i++) 478 scratch[i] = pattern(patt++); 479 480 /* Compare the images */ 481 for (i = 0; i < bufsize; i++) { 482 KUNIT_EXPECT_EQ_MSG(test, buffer[i], scratch[i], "at i=%x", i); 483 if (buffer[i] != scratch[i]) 484 return; 485 } 486 487 stop: 488 KUNIT_SUCCEED(test); 489 } 490 491 /* 492 * Test copying from a ITER_FOLIOQ-type iterator. 493 */ 494 static void __init iov_kunit_copy_from_folioq(struct kunit *test) 495 { 496 const struct kvec_test_range *pr; 497 struct iov_iter iter; 498 struct folio_queue *folioq; 499 struct page **spages, **bpages; 500 u8 *scratch, *buffer; 501 size_t bufsize, npages, size, copied; 502 int i, j; 503 504 bufsize = 0x100000; 505 npages = bufsize / PAGE_SIZE; 506 507 folioq = iov_kunit_create_folioq(test); 508 509 buffer = iov_kunit_create_buffer(test, &bpages, npages); 510 for (i = 0; i < bufsize; i++) 511 buffer[i] = pattern(i); 512 513 scratch = iov_kunit_create_buffer(test, &spages, npages); 514 memset(scratch, 0, bufsize); 515 516 iov_kunit_load_folioq(test, &iter, READ, folioq, bpages, npages); 517 518 i = 0; 519 for (pr = kvec_test_ranges; pr->from >= 0; pr++) { 520 size = pr->to - pr->from; 521 KUNIT_ASSERT_LE(test, pr->to, bufsize); 522 523 iov_iter_folio_queue(&iter, WRITE, folioq, 0, 0, pr->to); 524 iov_iter_advance(&iter, pr->from); 525 copied = copy_from_iter(scratch + i, size, &iter); 526 527 KUNIT_EXPECT_EQ(test, copied, size); 528 KUNIT_EXPECT_EQ(test, iter.count, 0); 529 KUNIT_EXPECT_EQ(test, iter.iov_offset, pr->to % PAGE_SIZE); 530 i += size; 531 } 532 533 /* Build the expected image in the main buffer. */ 534 i = 0; 535 memset(buffer, 0, bufsize); 536 for (pr = kvec_test_ranges; pr->from >= 0; pr++) { 537 for (j = pr->from; j < pr->to; j++) { 538 buffer[i++] = pattern(j); 539 if (i >= bufsize) 540 goto stop; 541 } 542 } 543 stop: 544 545 /* Compare the images */ 546 for (i = 0; i < bufsize; i++) { 547 KUNIT_EXPECT_EQ_MSG(test, scratch[i], buffer[i], "at i=%x", i); 548 if (scratch[i] != buffer[i]) 549 return; 550 } 551 552 KUNIT_SUCCEED(test); 553 } 554 555 static void iov_kunit_destroy_xarray(void *data) 556 { 557 struct xarray *xarray = data; 558 559 xa_destroy(xarray); 560 kfree(xarray); 561 } 562 563 static void __init iov_kunit_load_xarray(struct kunit *test, 564 struct iov_iter *iter, int dir, 565 struct xarray *xarray, 566 struct page **pages, size_t npages) 567 { 568 size_t size = 0; 569 int i; 570 571 for (i = 0; i < npages; i++) { 572 void *x = xa_store(xarray, i, pages[i], GFP_KERNEL); 573 574 KUNIT_ASSERT_FALSE(test, xa_is_err(x)); 575 size += PAGE_SIZE; 576 } 577 iov_iter_xarray(iter, dir, xarray, 0, size); 578 } 579 580 static struct xarray *iov_kunit_create_xarray(struct kunit *test) 581 { 582 struct xarray *xarray; 583 584 xarray = kzalloc_obj(struct xarray); 585 xa_init(xarray); 586 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xarray); 587 kunit_add_action_or_reset(test, iov_kunit_destroy_xarray, xarray); 588 return xarray; 589 } 590 591 /* 592 * Test copying to a ITER_XARRAY-type iterator. 593 */ 594 static void __init iov_kunit_copy_to_xarray(struct kunit *test) 595 { 596 const struct kvec_test_range *pr; 597 struct iov_iter iter; 598 struct xarray *xarray; 599 struct page **spages, **bpages; 600 u8 *scratch, *buffer; 601 size_t bufsize, npages, size, copied; 602 int i, patt; 603 604 bufsize = 0x100000; 605 npages = bufsize / PAGE_SIZE; 606 607 xarray = iov_kunit_create_xarray(test); 608 609 scratch = iov_kunit_create_buffer(test, &spages, npages); 610 for (i = 0; i < bufsize; i++) 611 scratch[i] = pattern(i); 612 613 buffer = iov_kunit_create_buffer(test, &bpages, npages); 614 memset(buffer, 0, bufsize); 615 616 iov_kunit_load_xarray(test, &iter, READ, xarray, bpages, npages); 617 618 i = 0; 619 for (pr = kvec_test_ranges; pr->from >= 0; pr++) { 620 size = pr->to - pr->from; 621 KUNIT_ASSERT_LE(test, pr->to, bufsize); 622 623 iov_iter_xarray(&iter, READ, xarray, pr->from, size); 624 copied = copy_to_iter(scratch + i, size, &iter); 625 626 KUNIT_EXPECT_EQ(test, copied, size); 627 KUNIT_EXPECT_EQ(test, iter.count, 0); 628 KUNIT_EXPECT_EQ(test, iter.iov_offset, size); 629 i += size; 630 } 631 632 /* Build the expected image in the scratch buffer. */ 633 patt = 0; 634 memset(scratch, 0, bufsize); 635 for (pr = kvec_test_ranges; pr->from >= 0; pr++) 636 for (i = pr->from; i < pr->to; i++) 637 scratch[i] = pattern(patt++); 638 639 /* Compare the images */ 640 for (i = 0; i < bufsize; i++) { 641 KUNIT_EXPECT_EQ_MSG(test, buffer[i], scratch[i], "at i=%x", i); 642 if (buffer[i] != scratch[i]) 643 return; 644 } 645 646 KUNIT_SUCCEED(test); 647 } 648 649 /* 650 * Test copying from a ITER_XARRAY-type iterator. 651 */ 652 static void __init iov_kunit_copy_from_xarray(struct kunit *test) 653 { 654 const struct kvec_test_range *pr; 655 struct iov_iter iter; 656 struct xarray *xarray; 657 struct page **spages, **bpages; 658 u8 *scratch, *buffer; 659 size_t bufsize, npages, size, copied; 660 int i, j; 661 662 bufsize = 0x100000; 663 npages = bufsize / PAGE_SIZE; 664 665 xarray = iov_kunit_create_xarray(test); 666 667 buffer = iov_kunit_create_buffer(test, &bpages, npages); 668 for (i = 0; i < bufsize; i++) 669 buffer[i] = pattern(i); 670 671 scratch = iov_kunit_create_buffer(test, &spages, npages); 672 memset(scratch, 0, bufsize); 673 674 iov_kunit_load_xarray(test, &iter, READ, xarray, bpages, npages); 675 676 i = 0; 677 for (pr = kvec_test_ranges; pr->from >= 0; pr++) { 678 size = pr->to - pr->from; 679 KUNIT_ASSERT_LE(test, pr->to, bufsize); 680 681 iov_iter_xarray(&iter, WRITE, xarray, pr->from, size); 682 copied = copy_from_iter(scratch + i, size, &iter); 683 684 KUNIT_EXPECT_EQ(test, copied, size); 685 KUNIT_EXPECT_EQ(test, iter.count, 0); 686 KUNIT_EXPECT_EQ(test, iter.iov_offset, size); 687 i += size; 688 } 689 690 /* Build the expected image in the main buffer. */ 691 i = 0; 692 memset(buffer, 0, bufsize); 693 for (pr = kvec_test_ranges; pr->from >= 0; pr++) { 694 for (j = pr->from; j < pr->to; j++) { 695 buffer[i++] = pattern(j); 696 if (i >= bufsize) 697 goto stop; 698 } 699 } 700 stop: 701 702 /* Compare the images */ 703 for (i = 0; i < bufsize; i++) { 704 KUNIT_EXPECT_EQ_MSG(test, scratch[i], buffer[i], "at i=%x", i); 705 if (scratch[i] != buffer[i]) 706 return; 707 } 708 709 KUNIT_SUCCEED(test); 710 } 711 712 /* 713 * Test the extraction of ITER_KVEC-type iterators. 714 */ 715 static void __init iov_kunit_extract_pages_kvec(struct kunit *test) 716 { 717 const struct kvec_test_range *pr; 718 struct iov_iter iter; 719 struct page **bpages, *pagelist[8], **pages = pagelist; 720 struct kvec kvec[8]; 721 u8 *buffer; 722 ssize_t len; 723 size_t bufsize, size = 0, npages; 724 int i, from; 725 726 bufsize = 0x100000; 727 npages = bufsize / PAGE_SIZE; 728 729 buffer = iov_kunit_create_buffer(test, &bpages, npages); 730 731 iov_kunit_load_kvec(test, &iter, READ, kvec, ARRAY_SIZE(kvec), 732 buffer, bufsize, kvec_test_ranges); 733 size = iter.count; 734 735 pr = kvec_test_ranges; 736 from = pr->from; 737 do { 738 size_t offset0 = LONG_MAX; 739 740 for (i = 0; i < ARRAY_SIZE(pagelist); i++) 741 pagelist[i] = (void *)(unsigned long)0xaa55aa55aa55aa55ULL; 742 743 len = iov_iter_extract_pages(&iter, &pages, 100 * 1024, 744 ARRAY_SIZE(pagelist), 0, &offset0); 745 KUNIT_EXPECT_GE(test, len, 0); 746 if (len < 0) 747 break; 748 KUNIT_EXPECT_GE(test, (ssize_t)offset0, 0); 749 KUNIT_EXPECT_LT(test, offset0, PAGE_SIZE); 750 KUNIT_EXPECT_LE(test, len, size); 751 KUNIT_EXPECT_EQ(test, iter.count, size - len); 752 size -= len; 753 754 if (len == 0) 755 break; 756 757 for (i = 0; i < ARRAY_SIZE(pagelist); i++) { 758 struct page *p; 759 ssize_t part = min_t(ssize_t, len, PAGE_SIZE - offset0); 760 int ix; 761 762 KUNIT_ASSERT_GE(test, part, 0); 763 while (from == pr->to) { 764 pr++; 765 from = pr->from; 766 if (from < 0) 767 goto stop; 768 } 769 ix = from / PAGE_SIZE; 770 KUNIT_ASSERT_LT(test, ix, npages); 771 p = bpages[ix]; 772 KUNIT_EXPECT_PTR_EQ(test, pagelist[i], p); 773 KUNIT_EXPECT_EQ(test, offset0, from % PAGE_SIZE); 774 from += part; 775 len -= part; 776 KUNIT_ASSERT_GE(test, len, 0); 777 if (len == 0) 778 break; 779 offset0 = 0; 780 } 781 782 if (test->status == KUNIT_FAILURE) 783 break; 784 } while (iov_iter_count(&iter) > 0); 785 786 stop: 787 KUNIT_EXPECT_EQ(test, size, 0); 788 KUNIT_EXPECT_EQ(test, iter.count, 0); 789 KUNIT_SUCCEED(test); 790 } 791 792 /* 793 * Test the extraction of ITER_BVEC-type iterators. 794 */ 795 static void __init iov_kunit_extract_pages_bvec(struct kunit *test) 796 { 797 const struct bvec_test_range *pr; 798 struct iov_iter iter; 799 struct page **bpages, *pagelist[8], **pages = pagelist; 800 struct bio_vec bvec[8]; 801 ssize_t len; 802 size_t bufsize, size = 0, npages; 803 int i, from; 804 805 bufsize = 0x100000; 806 npages = bufsize / PAGE_SIZE; 807 808 iov_kunit_create_buffer(test, &bpages, npages); 809 iov_kunit_load_bvec(test, &iter, READ, bvec, ARRAY_SIZE(bvec), 810 bpages, npages, bufsize, bvec_test_ranges); 811 size = iter.count; 812 813 pr = bvec_test_ranges; 814 from = pr->from; 815 do { 816 size_t offset0 = LONG_MAX; 817 818 for (i = 0; i < ARRAY_SIZE(pagelist); i++) 819 pagelist[i] = (void *)(unsigned long)0xaa55aa55aa55aa55ULL; 820 821 len = iov_iter_extract_pages(&iter, &pages, 100 * 1024, 822 ARRAY_SIZE(pagelist), 0, &offset0); 823 KUNIT_EXPECT_GE(test, len, 0); 824 if (len < 0) 825 break; 826 KUNIT_EXPECT_GE(test, (ssize_t)offset0, 0); 827 KUNIT_EXPECT_LT(test, offset0, PAGE_SIZE); 828 KUNIT_EXPECT_LE(test, len, size); 829 KUNIT_EXPECT_EQ(test, iter.count, size - len); 830 size -= len; 831 832 if (len == 0) 833 break; 834 835 for (i = 0; i < ARRAY_SIZE(pagelist); i++) { 836 struct page *p; 837 ssize_t part = min_t(ssize_t, len, PAGE_SIZE - offset0); 838 int ix; 839 840 KUNIT_ASSERT_GE(test, part, 0); 841 while (from == pr->to) { 842 pr++; 843 from = pr->from; 844 if (from < 0) 845 goto stop; 846 } 847 ix = pr->page + from / PAGE_SIZE; 848 KUNIT_ASSERT_LT(test, ix, npages); 849 p = bpages[ix]; 850 KUNIT_EXPECT_PTR_EQ(test, pagelist[i], p); 851 KUNIT_EXPECT_EQ(test, offset0, from % PAGE_SIZE); 852 from += part; 853 len -= part; 854 KUNIT_ASSERT_GE(test, len, 0); 855 if (len == 0) 856 break; 857 offset0 = 0; 858 } 859 860 if (test->status == KUNIT_FAILURE) 861 break; 862 } while (iov_iter_count(&iter) > 0); 863 864 stop: 865 KUNIT_EXPECT_EQ(test, size, 0); 866 KUNIT_EXPECT_EQ(test, iter.count, 0); 867 KUNIT_SUCCEED(test); 868 } 869 870 /* 871 * Test the extraction of ITER_FOLIOQ-type iterators. 872 */ 873 static void __init iov_kunit_extract_pages_folioq(struct kunit *test) 874 { 875 const struct kvec_test_range *pr; 876 struct folio_queue *folioq; 877 struct iov_iter iter; 878 struct page **bpages, *pagelist[8], **pages = pagelist; 879 ssize_t len; 880 size_t bufsize, size = 0, npages; 881 int i, from; 882 883 bufsize = 0x100000; 884 npages = bufsize / PAGE_SIZE; 885 886 folioq = iov_kunit_create_folioq(test); 887 888 iov_kunit_create_buffer(test, &bpages, npages); 889 iov_kunit_load_folioq(test, &iter, READ, folioq, bpages, npages); 890 891 for (pr = kvec_test_ranges; pr->from >= 0; pr++) { 892 from = pr->from; 893 size = pr->to - from; 894 KUNIT_ASSERT_LE(test, pr->to, bufsize); 895 896 iov_iter_folio_queue(&iter, WRITE, folioq, 0, 0, pr->to); 897 iov_iter_advance(&iter, from); 898 899 do { 900 size_t offset0 = LONG_MAX; 901 902 for (i = 0; i < ARRAY_SIZE(pagelist); i++) 903 pagelist[i] = (void *)(unsigned long)0xaa55aa55aa55aa55ULL; 904 905 len = iov_iter_extract_pages(&iter, &pages, 100 * 1024, 906 ARRAY_SIZE(pagelist), 0, &offset0); 907 KUNIT_EXPECT_GE(test, len, 0); 908 if (len < 0) 909 break; 910 KUNIT_EXPECT_LE(test, len, size); 911 KUNIT_EXPECT_EQ(test, iter.count, size - len); 912 if (len == 0) 913 break; 914 size -= len; 915 KUNIT_EXPECT_GE(test, (ssize_t)offset0, 0); 916 KUNIT_EXPECT_LT(test, offset0, PAGE_SIZE); 917 918 for (i = 0; i < ARRAY_SIZE(pagelist); i++) { 919 struct page *p; 920 ssize_t part = min_t(ssize_t, len, PAGE_SIZE - offset0); 921 int ix; 922 923 KUNIT_ASSERT_GE(test, part, 0); 924 ix = from / PAGE_SIZE; 925 KUNIT_ASSERT_LT(test, ix, npages); 926 p = bpages[ix]; 927 KUNIT_EXPECT_PTR_EQ(test, pagelist[i], p); 928 KUNIT_EXPECT_EQ(test, offset0, from % PAGE_SIZE); 929 from += part; 930 len -= part; 931 KUNIT_ASSERT_GE(test, len, 0); 932 if (len == 0) 933 break; 934 offset0 = 0; 935 } 936 937 if (test->status == KUNIT_FAILURE) 938 goto stop; 939 } while (iov_iter_count(&iter) > 0); 940 941 KUNIT_EXPECT_EQ(test, size, 0); 942 KUNIT_EXPECT_EQ(test, iter.count, 0); 943 } 944 945 stop: 946 KUNIT_SUCCEED(test); 947 } 948 949 /* 950 * Test the extraction of ITER_XARRAY-type iterators. 951 */ 952 static void __init iov_kunit_extract_pages_xarray(struct kunit *test) 953 { 954 const struct kvec_test_range *pr; 955 struct iov_iter iter; 956 struct xarray *xarray; 957 struct page **bpages, *pagelist[8], **pages = pagelist; 958 ssize_t len; 959 size_t bufsize, size = 0, npages; 960 int i, from; 961 962 bufsize = 0x100000; 963 npages = bufsize / PAGE_SIZE; 964 965 xarray = iov_kunit_create_xarray(test); 966 967 iov_kunit_create_buffer(test, &bpages, npages); 968 iov_kunit_load_xarray(test, &iter, READ, xarray, bpages, npages); 969 970 for (pr = kvec_test_ranges; pr->from >= 0; pr++) { 971 from = pr->from; 972 size = pr->to - from; 973 KUNIT_ASSERT_LE(test, pr->to, bufsize); 974 975 iov_iter_xarray(&iter, WRITE, xarray, from, size); 976 977 do { 978 size_t offset0 = LONG_MAX; 979 980 for (i = 0; i < ARRAY_SIZE(pagelist); i++) 981 pagelist[i] = (void *)(unsigned long)0xaa55aa55aa55aa55ULL; 982 983 len = iov_iter_extract_pages(&iter, &pages, 100 * 1024, 984 ARRAY_SIZE(pagelist), 0, &offset0); 985 KUNIT_EXPECT_GE(test, len, 0); 986 if (len < 0) 987 break; 988 KUNIT_EXPECT_LE(test, len, size); 989 KUNIT_EXPECT_EQ(test, iter.count, size - len); 990 if (len == 0) 991 break; 992 size -= len; 993 KUNIT_EXPECT_GE(test, (ssize_t)offset0, 0); 994 KUNIT_EXPECT_LT(test, offset0, PAGE_SIZE); 995 996 for (i = 0; i < ARRAY_SIZE(pagelist); i++) { 997 struct page *p; 998 ssize_t part = min_t(ssize_t, len, PAGE_SIZE - offset0); 999 int ix; 1000 1001 KUNIT_ASSERT_GE(test, part, 0); 1002 ix = from / PAGE_SIZE; 1003 KUNIT_ASSERT_LT(test, ix, npages); 1004 p = bpages[ix]; 1005 KUNIT_EXPECT_PTR_EQ(test, pagelist[i], p); 1006 KUNIT_EXPECT_EQ(test, offset0, from % PAGE_SIZE); 1007 from += part; 1008 len -= part; 1009 KUNIT_ASSERT_GE(test, len, 0); 1010 if (len == 0) 1011 break; 1012 offset0 = 0; 1013 } 1014 1015 if (test->status == KUNIT_FAILURE) 1016 goto stop; 1017 } while (iov_iter_count(&iter) > 0); 1018 1019 KUNIT_EXPECT_EQ(test, size, 0); 1020 KUNIT_EXPECT_EQ(test, iter.count, 0); 1021 KUNIT_EXPECT_EQ(test, iter.iov_offset, pr->to - pr->from); 1022 } 1023 1024 stop: 1025 KUNIT_SUCCEED(test); 1026 } 1027 1028 struct iov_kunit_iter_to_sg_data { 1029 struct sg_table *sgt; 1030 u8 *buffer, *scratch; 1031 u8 __user *ubuf; 1032 struct page **pages; 1033 size_t npages; 1034 }; 1035 1036 static void __init 1037 iov_kunit_iter_unpin_sgt(void *data) 1038 { 1039 struct sg_table *sgt = data; 1040 1041 for (unsigned int i = 0; i < sgt->nents; ++i) 1042 unpin_user_page(sg_page(&sgt->sgl[i])); 1043 } 1044 1045 static void __init 1046 iov_kunit_iter_to_sg_init(struct kunit *test, size_t bufsize, bool user, 1047 struct iov_kunit_iter_to_sg_data *data) 1048 { 1049 struct page **spages; 1050 struct scatterlist *sg; 1051 unsigned long uaddr; 1052 size_t i; 1053 1054 data->npages = bufsize / PAGE_SIZE; 1055 sg = kunit_kmalloc_array(test, data->npages, sizeof(*sg), GFP_KERNEL); 1056 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, sg); 1057 sg_init_table(sg, data->npages); 1058 data->sgt = kunit_kzalloc(test, sizeof(*data->sgt), GFP_KERNEL); 1059 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, data->sgt); 1060 data->sgt->orig_nents = 0; 1061 data->sgt->sgl = sg; 1062 1063 data->buffer = NULL; 1064 data->ubuf = NULL; 1065 if (user) { 1066 uaddr = kunit_vm_mmap(test, NULL, 0, bufsize, 1067 PROT_READ | PROT_WRITE, 1068 MAP_ANONYMOUS | MAP_PRIVATE, 0); 1069 KUNIT_ASSERT_NE(test, uaddr, 0); 1070 data->ubuf = (u8 __user *)uaddr; 1071 for (i = 0; i < bufsize; ++i) 1072 put_user(pattern(i), data->ubuf + i); 1073 } else { 1074 data->buffer = iov_kunit_create_buffer(test, &data->pages, 1075 data->npages); 1076 for (i = 0; i < bufsize; ++i) 1077 data->buffer[i] = pattern(i); 1078 } 1079 data->scratch = iov_kunit_create_buffer(test, &spages, data->npages); 1080 memset(data->scratch, 0, bufsize); 1081 } 1082 1083 static void __init 1084 iov_kunit_iter_to_sg_check(struct kunit *test, struct iov_iter *iter, 1085 size_t bufsize, 1086 struct iov_kunit_iter_to_sg_data *data) 1087 { 1088 static const size_t tail = 16 * PAGE_SIZE; 1089 size_t i; 1090 1091 KUNIT_ASSERT_LT(test, tail, bufsize); 1092 1093 if (iov_iter_extract_will_pin(iter)) 1094 kunit_add_action_or_reset(test, iov_kunit_iter_unpin_sgt, 1095 data->sgt); 1096 1097 i = extract_iter_to_sg(iter, bufsize, data->sgt, 0, 0); 1098 KUNIT_ASSERT_EQ(test, i, 0); 1099 KUNIT_ASSERT_EQ(test, data->sgt->nents, 0); 1100 1101 i = extract_iter_to_sg(iter, bufsize - tail, data->sgt, 1, 0); 1102 KUNIT_ASSERT_LE(test, i, bufsize - tail); 1103 KUNIT_ASSERT_EQ(test, data->sgt->nents, 1); 1104 1105 i += extract_iter_to_sg(iter, bufsize - tail - i, data->sgt, 1106 data->npages - data->sgt->nents, 0); 1107 KUNIT_ASSERT_EQ(test, i, bufsize - tail); 1108 KUNIT_ASSERT_LE(test, data->sgt->nents, data->npages); 1109 1110 i += extract_iter_to_sg(iter, tail, data->sgt, 1111 data->npages - data->sgt->nents, 0); 1112 KUNIT_ASSERT_EQ(test, i, bufsize); 1113 KUNIT_ASSERT_LE(test, data->sgt->nents, data->npages); 1114 1115 sg_mark_end(&data->sgt->sgl[data->sgt->nents - 1]); 1116 1117 i = sg_copy_to_buffer(data->sgt->sgl, data->sgt->nents, 1118 data->scratch, bufsize); 1119 KUNIT_ASSERT_EQ(test, i, bufsize); 1120 1121 for (i = 0; i < bufsize; ++i) { 1122 KUNIT_EXPECT_EQ_MSG(test, data->scratch[i], pattern(i), 1123 "at i=%zx", i); 1124 if (data->scratch[i] != pattern(i)) 1125 break; 1126 } 1127 1128 KUNIT_EXPECT_EQ(test, i, bufsize); 1129 } 1130 1131 static void __init iov_kunit_iter_to_sg_kvec(struct kunit *test) 1132 { 1133 struct iov_kunit_iter_to_sg_data data; 1134 struct iov_iter iter; 1135 struct kvec kvec; 1136 size_t bufsize; 1137 1138 bufsize = 0x200000; 1139 iov_kunit_iter_to_sg_init(test, bufsize, false, &data); 1140 1141 kvec.iov_base = data.buffer; 1142 kvec.iov_len = bufsize; 1143 iov_iter_kvec(&iter, READ, &kvec, 1, bufsize); 1144 1145 iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data); 1146 } 1147 1148 static void __init iov_kunit_iter_to_sg_bvec(struct kunit *test) 1149 { 1150 struct iov_kunit_iter_to_sg_data data; 1151 struct page *p, *can_merge = NULL; 1152 size_t i, k, bufsize; 1153 struct bio_vec *bvec; 1154 struct iov_iter iter; 1155 1156 bufsize = 0x200000; 1157 iov_kunit_iter_to_sg_init(test, bufsize, false, &data); 1158 1159 bvec = kunit_kmalloc_array(test, data.npages, sizeof(*bvec), 1160 GFP_KERNEL); 1161 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, bvec); 1162 k = 0; 1163 for (i = 0; i < data.npages; ++i) { 1164 p = data.pages[i]; 1165 if (p == can_merge) 1166 bvec[k-1].bv_len += PAGE_SIZE; 1167 else 1168 bvec_set_page(&bvec[k++], p, PAGE_SIZE, 0); 1169 can_merge = p + 1; 1170 } 1171 iov_iter_bvec(&iter, READ, bvec, k, bufsize); 1172 1173 iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data); 1174 } 1175 1176 static void __init iov_kunit_iter_to_sg_folioq(struct kunit *test) 1177 { 1178 struct iov_kunit_iter_to_sg_data data; 1179 struct folio_queue *folioq; 1180 struct iov_iter iter; 1181 size_t bufsize; 1182 1183 bufsize = 0x200000; 1184 iov_kunit_iter_to_sg_init(test, bufsize, false, &data); 1185 1186 folioq = iov_kunit_create_folioq(test); 1187 iov_kunit_load_folioq(test, &iter, READ, folioq, data.pages, 1188 data.npages); 1189 1190 iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data); 1191 } 1192 1193 static void __init iov_kunit_iter_to_sg_xarray(struct kunit *test) 1194 { 1195 struct iov_kunit_iter_to_sg_data data; 1196 struct xarray *xarray; 1197 struct iov_iter iter; 1198 size_t bufsize; 1199 1200 bufsize = 0x200000; 1201 iov_kunit_iter_to_sg_init(test, bufsize, false, &data); 1202 1203 xarray = iov_kunit_create_xarray(test); 1204 iov_kunit_load_xarray(test, &iter, READ, xarray, data.pages, 1205 data.npages); 1206 1207 iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data); 1208 } 1209 1210 static void __init iov_kunit_iter_to_sg_ubuf(struct kunit *test) 1211 { 1212 struct iov_kunit_iter_to_sg_data data; 1213 struct iov_iter iter; 1214 size_t bufsize; 1215 1216 bufsize = 0x200000; 1217 iov_kunit_iter_to_sg_init(test, bufsize, true, &data); 1218 1219 iov_iter_ubuf(&iter, READ, data.ubuf, bufsize); 1220 1221 iov_kunit_iter_to_sg_check(test, &iter, bufsize, &data); 1222 } 1223 1224 static struct kunit_case __refdata iov_kunit_cases[] = { 1225 KUNIT_CASE(iov_kunit_copy_to_kvec), 1226 KUNIT_CASE(iov_kunit_copy_from_kvec), 1227 KUNIT_CASE(iov_kunit_copy_to_bvec), 1228 KUNIT_CASE(iov_kunit_copy_from_bvec), 1229 KUNIT_CASE(iov_kunit_copy_to_folioq), 1230 KUNIT_CASE(iov_kunit_copy_from_folioq), 1231 KUNIT_CASE(iov_kunit_copy_to_xarray), 1232 KUNIT_CASE(iov_kunit_copy_from_xarray), 1233 KUNIT_CASE(iov_kunit_extract_pages_kvec), 1234 KUNIT_CASE(iov_kunit_extract_pages_bvec), 1235 KUNIT_CASE(iov_kunit_extract_pages_folioq), 1236 KUNIT_CASE(iov_kunit_extract_pages_xarray), 1237 KUNIT_CASE(iov_kunit_iter_to_sg_kvec), 1238 KUNIT_CASE(iov_kunit_iter_to_sg_bvec), 1239 KUNIT_CASE(iov_kunit_iter_to_sg_folioq), 1240 KUNIT_CASE(iov_kunit_iter_to_sg_xarray), 1241 KUNIT_CASE(iov_kunit_iter_to_sg_ubuf), 1242 {} 1243 }; 1244 1245 static struct kunit_suite iov_kunit_suite = { 1246 .name = "iov_iter", 1247 .test_cases = iov_kunit_cases, 1248 }; 1249 1250 kunit_test_suites(&iov_kunit_suite); 1251