1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2018 Cambridge Greys Ltd 4 * Copyright (C) 2015-2016 Anton Ivanov (aivanov@brocade.com) 5 * Copyright (C) 2000 Jeff Dike (jdike@karaya.com) 6 */ 7 8 /* 2001-09-28...2002-04-17 9 * Partition stuff by James_McMechan@hotmail.com 10 * old style ubd by setting UBD_SHIFT to 0 11 * 2002-09-27...2002-10-18 massive tinkering for 2.5 12 * partitions have changed in 2.5 13 * 2003-01-29 more tinkering for 2.5.59-1 14 * This should now address the sysfs problems and has 15 * the symlink for devfs to allow for booting with 16 * the common /dev/ubd/discX/... names rather than 17 * only /dev/ubdN/discN this version also has lots of 18 * clean ups preparing for ubd-many. 19 * James McMechan 20 */ 21 22 #define UBD_SHIFT 4 23 24 #include <linux/module.h> 25 #include <linux/init.h> 26 #include <linux/blkdev.h> 27 #include <linux/blk-mq.h> 28 #include <linux/ata.h> 29 #include <linux/hdreg.h> 30 #include <linux/major.h> 31 #include <linux/cdrom.h> 32 #include <linux/proc_fs.h> 33 #include <linux/seq_file.h> 34 #include <linux/ctype.h> 35 #include <linux/slab.h> 36 #include <linux/vmalloc.h> 37 #include <linux/platform_device.h> 38 #include <linux/scatterlist.h> 39 #include <asm/tlbflush.h> 40 #include <kern_util.h> 41 #include "mconsole_kern.h" 42 #include <init.h> 43 #include <irq_kern.h> 44 #include "ubd.h" 45 #include <os.h> 46 #include "cow.h" 47 48 /* Max request size is determined by sector mask - 32K */ 49 #define UBD_MAX_REQUEST (8 * sizeof(long)) 50 51 struct io_desc { 52 char *buffer; 53 unsigned long length; 54 unsigned long sector_mask; 55 unsigned long long cow_offset; 56 unsigned long bitmap_words[2]; 57 }; 58 59 struct io_thread_req { 60 struct request *req; 61 int fds[2]; 62 unsigned long offsets[2]; 63 unsigned long long offset; 64 int sectorsize; 65 int error; 66 67 int desc_cnt; 68 /* io_desc has to be the last element of the struct */ 69 struct io_desc io_desc[]; 70 }; 71 72 73 static struct io_thread_req * (*irq_req_buffer)[]; 74 static struct io_thread_req *irq_remainder; 75 static int irq_remainder_size; 76 77 static struct io_thread_req * (*io_req_buffer)[]; 78 static struct io_thread_req *io_remainder; 79 static int io_remainder_size; 80 81 82 83 static inline int ubd_test_bit(__u64 bit, unsigned char *data) 84 { 85 __u64 n; 86 int bits, off; 87 88 bits = sizeof(data[0]) * 8; 89 n = bit / bits; 90 off = bit % bits; 91 return (data[n] & (1 << off)) != 0; 92 } 93 94 static inline void ubd_set_bit(__u64 bit, unsigned char *data) 95 { 96 __u64 n; 97 int bits, off; 98 99 bits = sizeof(data[0]) * 8; 100 n = bit / bits; 101 off = bit % bits; 102 data[n] |= (1 << off); 103 } 104 /*End stuff from ubd_user.h*/ 105 106 #define DRIVER_NAME "uml-blkdev" 107 108 static DEFINE_MUTEX(ubd_lock); 109 110 static int ubd_ioctl(struct block_device *bdev, blk_mode_t mode, 111 unsigned int cmd, unsigned long arg); 112 static int ubd_getgeo(struct block_device *bdev, struct hd_geometry *geo); 113 114 #define MAX_DEV (16) 115 116 static const struct block_device_operations ubd_blops = { 117 .owner = THIS_MODULE, 118 .ioctl = ubd_ioctl, 119 .compat_ioctl = blkdev_compat_ptr_ioctl, 120 .getgeo = ubd_getgeo, 121 }; 122 123 #ifdef CONFIG_BLK_DEV_UBD_SYNC 124 #define OPEN_FLAGS ((struct openflags) { .r = 1, .w = 1, .s = 1, .c = 0, \ 125 .cl = 1 }) 126 #else 127 #define OPEN_FLAGS ((struct openflags) { .r = 1, .w = 1, .s = 0, .c = 0, \ 128 .cl = 1 }) 129 #endif 130 static struct openflags global_openflags = OPEN_FLAGS; 131 132 struct cow { 133 /* backing file name */ 134 char *file; 135 /* backing file fd */ 136 int fd; 137 unsigned long *bitmap; 138 unsigned long bitmap_len; 139 int bitmap_offset; 140 int data_offset; 141 }; 142 143 #define MAX_SG 64 144 145 struct ubd { 146 /* name (and fd, below) of the file opened for writing, either the 147 * backing or the cow file. */ 148 char *file; 149 char *serial; 150 int fd; 151 __u64 size; 152 struct openflags boot_openflags; 153 struct openflags openflags; 154 unsigned shared:1; 155 unsigned no_cow:1; 156 unsigned no_trim:1; 157 struct cow cow; 158 struct platform_device pdev; 159 struct gendisk *disk; 160 struct blk_mq_tag_set tag_set; 161 spinlock_t lock; 162 }; 163 164 #define DEFAULT_COW { \ 165 .file = NULL, \ 166 .fd = -1, \ 167 .bitmap = NULL, \ 168 .bitmap_offset = 0, \ 169 .data_offset = 0, \ 170 } 171 172 #define DEFAULT_UBD { \ 173 .file = NULL, \ 174 .serial = NULL, \ 175 .fd = -1, \ 176 .size = -1, \ 177 .boot_openflags = OPEN_FLAGS, \ 178 .openflags = OPEN_FLAGS, \ 179 .no_cow = 0, \ 180 .no_trim = 0, \ 181 .shared = 0, \ 182 .cow = DEFAULT_COW, \ 183 .lock = __SPIN_LOCK_UNLOCKED(ubd_devs.lock), \ 184 } 185 186 /* Protected by ubd_lock */ 187 static struct ubd ubd_devs[MAX_DEV] = { [0 ... MAX_DEV - 1] = DEFAULT_UBD }; 188 189 static blk_status_t ubd_queue_rq(struct blk_mq_hw_ctx *hctx, 190 const struct blk_mq_queue_data *bd); 191 192 static int fake_ide_setup(char *str) 193 { 194 pr_warn("The fake_ide option has been removed\n"); 195 return 1; 196 } 197 __setup("fake_ide", fake_ide_setup); 198 199 __uml_help(fake_ide_setup, 200 "fake_ide\n" 201 " Obsolete stub.\n\n" 202 ); 203 204 static int parse_unit(char **ptr) 205 { 206 char *str = *ptr, *end; 207 int n = -1; 208 209 if(isdigit(*str)) { 210 n = simple_strtoul(str, &end, 0); 211 if(end == str) 212 return -1; 213 *ptr = end; 214 } 215 else if (('a' <= *str) && (*str <= 'z')) { 216 n = *str - 'a'; 217 str++; 218 *ptr = str; 219 } 220 return n; 221 } 222 223 /* If *index_out == -1 at exit, the passed option was a general one; 224 * otherwise, the str pointer is used (and owned) inside ubd_devs array, so it 225 * should not be freed on exit. 226 */ 227 static int ubd_setup_common(char *str, int *index_out, char **error_out) 228 { 229 struct ubd *ubd_dev; 230 struct openflags flags = global_openflags; 231 char *file, *backing_file, *serial; 232 int n, err = 0, i; 233 234 if(index_out) *index_out = -1; 235 n = *str; 236 if(n == '='){ 237 str++; 238 if(!strcmp(str, "sync")){ 239 global_openflags = of_sync(global_openflags); 240 return err; 241 } 242 243 pr_warn("fake major not supported any more\n"); 244 return 0; 245 } 246 247 n = parse_unit(&str); 248 if(n < 0){ 249 *error_out = "Couldn't parse device number"; 250 return -EINVAL; 251 } 252 if(n >= MAX_DEV){ 253 *error_out = "Device number out of range"; 254 return 1; 255 } 256 257 err = -EBUSY; 258 mutex_lock(&ubd_lock); 259 260 ubd_dev = &ubd_devs[n]; 261 if(ubd_dev->file != NULL){ 262 *error_out = "Device is already configured"; 263 goto out; 264 } 265 266 if (index_out) 267 *index_out = n; 268 269 err = -EINVAL; 270 for (i = 0; i < sizeof("rscdt="); i++) { 271 switch (*str) { 272 case 'r': 273 flags.w = 0; 274 break; 275 case 's': 276 flags.s = 1; 277 break; 278 case 'd': 279 ubd_dev->no_cow = 1; 280 break; 281 case 'c': 282 ubd_dev->shared = 1; 283 break; 284 case 't': 285 ubd_dev->no_trim = 1; 286 break; 287 case '=': 288 str++; 289 goto break_loop; 290 default: 291 *error_out = "Expected '=' or flag letter " 292 "(r, s, c, t or d)"; 293 goto out; 294 } 295 str++; 296 } 297 298 if (*str == '=') 299 *error_out = "Too many flags specified"; 300 else 301 *error_out = "Missing '='"; 302 goto out; 303 304 break_loop: 305 file = strsep(&str, ",:"); 306 if (*file == '\0') 307 file = NULL; 308 309 backing_file = strsep(&str, ",:"); 310 if (backing_file && *backing_file == '\0') 311 backing_file = NULL; 312 313 serial = strsep(&str, ",:"); 314 if (serial && *serial == '\0') 315 serial = NULL; 316 317 if (backing_file && ubd_dev->no_cow) { 318 *error_out = "Can't specify both 'd' and a cow file"; 319 goto out; 320 } 321 322 err = 0; 323 ubd_dev->file = file; 324 ubd_dev->cow.file = backing_file; 325 ubd_dev->serial = serial; 326 ubd_dev->boot_openflags = flags; 327 out: 328 mutex_unlock(&ubd_lock); 329 return err; 330 } 331 332 static int ubd_setup(char *str) 333 { 334 char *error; 335 int err; 336 337 err = ubd_setup_common(str, NULL, &error); 338 if(err) 339 printk(KERN_ERR "Failed to initialize device with \"%s\" : " 340 "%s\n", str, error); 341 return 1; 342 } 343 344 __setup("ubd", ubd_setup); 345 __uml_help(ubd_setup, 346 "ubd<n><flags>=<filename>[(:|,)<filename2>][(:|,)<serial>]\n" 347 " This is used to associate a device with a file in the underlying\n" 348 " filesystem. When specifying two filenames, the first one is the\n" 349 " COW name and the second is the backing file name. As separator you can\n" 350 " use either a ':' or a ',': the first one allows writing things like;\n" 351 " ubd0=~/Uml/root_cow:~/Uml/root_backing_file\n" 352 " while with a ',' the shell would not expand the 2nd '~'.\n" 353 " When using only one filename, UML will detect whether to treat it like\n" 354 " a COW file or a backing file. To override this detection, add the 'd'\n" 355 " flag:\n" 356 " ubd0d=BackingFile\n" 357 " Usually, there is a filesystem in the file, but \n" 358 " that's not required. Swap devices containing swap files can be\n" 359 " specified like this. Also, a file which doesn't contain a\n" 360 " filesystem can have its contents read in the virtual \n" 361 " machine by running 'dd' on the device. <n> must be in the range\n" 362 " 0 to 7. Appending an 'r' to the number will cause that device\n" 363 " to be mounted read-only. For example ubd1r=./ext_fs. Appending\n" 364 " an 's' will cause data to be written to disk on the host immediately.\n" 365 " 'c' will cause the device to be treated as being shared between multiple\n" 366 " UMLs and file locking will be turned off - this is appropriate for a\n" 367 " cluster filesystem and inappropriate at almost all other times.\n\n" 368 " 't' will disable trim/discard support on the device (enabled by default).\n\n" 369 " An optional device serial number can be exposed using the serial parameter\n" 370 " on the cmdline which is exposed as a sysfs entry. This is particularly\n" 371 " useful when a unique number should be given to the device. Note when\n" 372 " specifying a label, the filename2 must be also presented. It can be\n" 373 " an empty string, in which case the backing file is not used:\n" 374 " ubd0=File,,Serial\n" 375 ); 376 377 static int udb_setup(char *str) 378 { 379 printk("udb%s specified on command line is almost certainly a ubd -> " 380 "udb TYPO\n", str); 381 return 1; 382 } 383 384 __setup("udb", udb_setup); 385 __uml_help(udb_setup, 386 "udb\n" 387 " This option is here solely to catch ubd -> udb typos, which can be\n" 388 " to impossible to catch visually unless you specifically look for\n" 389 " them. The only result of any option starting with 'udb' is an error\n" 390 " in the boot output.\n\n" 391 ); 392 393 /* Only changed by ubd_init, which is an initcall. */ 394 static int thread_fd = -1; 395 396 /* Function to read several request pointers at a time 397 * handling fractional reads if (and as) needed 398 */ 399 400 static int bulk_req_safe_read( 401 int fd, 402 struct io_thread_req * (*request_buffer)[], 403 struct io_thread_req **remainder, 404 int *remainder_size, 405 int max_recs 406 ) 407 { 408 int n = 0; 409 int res = 0; 410 411 if (*remainder_size > 0) { 412 memmove( 413 (char *) request_buffer, 414 (char *) remainder, *remainder_size 415 ); 416 n = *remainder_size; 417 } 418 419 res = os_read_file( 420 fd, 421 ((char *) request_buffer) + *remainder_size, 422 sizeof(struct io_thread_req *)*max_recs 423 - *remainder_size 424 ); 425 if (res > 0) { 426 n += res; 427 if ((n % sizeof(struct io_thread_req *)) > 0) { 428 /* 429 * Read somehow returned not a multiple of dword 430 * theoretically possible, but never observed in the 431 * wild, so read routine must be able to handle it 432 */ 433 *remainder_size = n % sizeof(struct io_thread_req *); 434 WARN(*remainder_size > 0, "UBD IPC read returned a partial result"); 435 memmove( 436 remainder, 437 ((char *) request_buffer) + 438 (n/sizeof(struct io_thread_req *))*sizeof(struct io_thread_req *), 439 *remainder_size 440 ); 441 n = n - *remainder_size; 442 } 443 } else { 444 n = res; 445 } 446 return n; 447 } 448 449 /* Called without dev->lock held, and only in interrupt context. */ 450 static void ubd_handler(void) 451 { 452 int n; 453 int count; 454 455 while(1){ 456 n = bulk_req_safe_read( 457 thread_fd, 458 irq_req_buffer, 459 &irq_remainder, 460 &irq_remainder_size, 461 UBD_REQ_BUFFER_SIZE 462 ); 463 if (n < 0) { 464 if(n == -EAGAIN) 465 break; 466 printk(KERN_ERR "spurious interrupt in ubd_handler, " 467 "err = %d\n", -n); 468 return; 469 } 470 for (count = 0; count < n/sizeof(struct io_thread_req *); count++) { 471 struct io_thread_req *io_req = (*irq_req_buffer)[count]; 472 473 if ((io_req->error == BLK_STS_NOTSUPP) && (req_op(io_req->req) == REQ_OP_DISCARD)) { 474 blk_queue_max_discard_sectors(io_req->req->q, 0); 475 blk_queue_max_write_zeroes_sectors(io_req->req->q, 0); 476 } 477 blk_mq_end_request(io_req->req, io_req->error); 478 kfree(io_req); 479 } 480 } 481 } 482 483 static irqreturn_t ubd_intr(int irq, void *dev) 484 { 485 ubd_handler(); 486 return IRQ_HANDLED; 487 } 488 489 /* Only changed by ubd_init, which is an initcall. */ 490 static int io_pid = -1; 491 492 static void kill_io_thread(void) 493 { 494 if(io_pid != -1) 495 os_kill_process(io_pid, 1); 496 } 497 498 __uml_exitcall(kill_io_thread); 499 500 static inline int ubd_file_size(struct ubd *ubd_dev, __u64 *size_out) 501 { 502 char *file; 503 int fd; 504 int err; 505 506 __u32 version; 507 __u32 align; 508 char *backing_file; 509 time64_t mtime; 510 unsigned long long size; 511 int sector_size; 512 int bitmap_offset; 513 514 if (ubd_dev->file && ubd_dev->cow.file) { 515 file = ubd_dev->cow.file; 516 517 goto out; 518 } 519 520 fd = os_open_file(ubd_dev->file, of_read(OPENFLAGS()), 0); 521 if (fd < 0) 522 return fd; 523 524 err = read_cow_header(file_reader, &fd, &version, &backing_file, \ 525 &mtime, &size, §or_size, &align, &bitmap_offset); 526 os_close_file(fd); 527 528 if(err == -EINVAL) 529 file = ubd_dev->file; 530 else 531 file = backing_file; 532 533 out: 534 return os_file_size(file, size_out); 535 } 536 537 static int read_cow_bitmap(int fd, void *buf, int offset, int len) 538 { 539 int err; 540 541 err = os_pread_file(fd, buf, len, offset); 542 if (err < 0) 543 return err; 544 545 return 0; 546 } 547 548 static int backing_file_mismatch(char *file, __u64 size, time64_t mtime) 549 { 550 time64_t modtime; 551 unsigned long long actual; 552 int err; 553 554 err = os_file_modtime(file, &modtime); 555 if (err < 0) { 556 printk(KERN_ERR "Failed to get modification time of backing " 557 "file \"%s\", err = %d\n", file, -err); 558 return err; 559 } 560 561 err = os_file_size(file, &actual); 562 if (err < 0) { 563 printk(KERN_ERR "Failed to get size of backing file \"%s\", " 564 "err = %d\n", file, -err); 565 return err; 566 } 567 568 if (actual != size) { 569 /*__u64 can be a long on AMD64 and with %lu GCC complains; so 570 * the typecast.*/ 571 printk(KERN_ERR "Size mismatch (%llu vs %llu) of COW header " 572 "vs backing file\n", (unsigned long long) size, actual); 573 return -EINVAL; 574 } 575 if (modtime != mtime) { 576 printk(KERN_ERR "mtime mismatch (%lld vs %lld) of COW header vs " 577 "backing file\n", mtime, modtime); 578 return -EINVAL; 579 } 580 return 0; 581 } 582 583 static int path_requires_switch(char *from_cmdline, char *from_cow, char *cow) 584 { 585 struct uml_stat buf1, buf2; 586 int err; 587 588 if (from_cmdline == NULL) 589 return 0; 590 if (!strcmp(from_cmdline, from_cow)) 591 return 0; 592 593 err = os_stat_file(from_cmdline, &buf1); 594 if (err < 0) { 595 printk(KERN_ERR "Couldn't stat '%s', err = %d\n", from_cmdline, 596 -err); 597 return 0; 598 } 599 err = os_stat_file(from_cow, &buf2); 600 if (err < 0) { 601 printk(KERN_ERR "Couldn't stat '%s', err = %d\n", from_cow, 602 -err); 603 return 1; 604 } 605 if ((buf1.ust_dev == buf2.ust_dev) && (buf1.ust_ino == buf2.ust_ino)) 606 return 0; 607 608 printk(KERN_ERR "Backing file mismatch - \"%s\" requested, " 609 "\"%s\" specified in COW header of \"%s\"\n", 610 from_cmdline, from_cow, cow); 611 return 1; 612 } 613 614 static int open_ubd_file(char *file, struct openflags *openflags, int shared, 615 char **backing_file_out, int *bitmap_offset_out, 616 unsigned long *bitmap_len_out, int *data_offset_out, 617 int *create_cow_out) 618 { 619 time64_t mtime; 620 unsigned long long size; 621 __u32 version, align; 622 char *backing_file; 623 int fd, err, sectorsize, asked_switch, mode = 0644; 624 625 fd = os_open_file(file, *openflags, mode); 626 if (fd < 0) { 627 if ((fd == -ENOENT) && (create_cow_out != NULL)) 628 *create_cow_out = 1; 629 if (!openflags->w || 630 ((fd != -EROFS) && (fd != -EACCES))) 631 return fd; 632 openflags->w = 0; 633 fd = os_open_file(file, *openflags, mode); 634 if (fd < 0) 635 return fd; 636 } 637 638 if (shared) 639 printk(KERN_INFO "Not locking \"%s\" on the host\n", file); 640 else { 641 err = os_lock_file(fd, openflags->w); 642 if (err < 0) { 643 printk(KERN_ERR "Failed to lock '%s', err = %d\n", 644 file, -err); 645 goto out_close; 646 } 647 } 648 649 /* Successful return case! */ 650 if (backing_file_out == NULL) 651 return fd; 652 653 err = read_cow_header(file_reader, &fd, &version, &backing_file, &mtime, 654 &size, §orsize, &align, bitmap_offset_out); 655 if (err && (*backing_file_out != NULL)) { 656 printk(KERN_ERR "Failed to read COW header from COW file " 657 "\"%s\", errno = %d\n", file, -err); 658 goto out_close; 659 } 660 if (err) 661 return fd; 662 663 asked_switch = path_requires_switch(*backing_file_out, backing_file, 664 file); 665 666 /* Allow switching only if no mismatch. */ 667 if (asked_switch && !backing_file_mismatch(*backing_file_out, size, 668 mtime)) { 669 printk(KERN_ERR "Switching backing file to '%s'\n", 670 *backing_file_out); 671 err = write_cow_header(file, fd, *backing_file_out, 672 sectorsize, align, &size); 673 if (err) { 674 printk(KERN_ERR "Switch failed, errno = %d\n", -err); 675 goto out_close; 676 } 677 } else { 678 *backing_file_out = backing_file; 679 err = backing_file_mismatch(*backing_file_out, size, mtime); 680 if (err) 681 goto out_close; 682 } 683 684 cow_sizes(version, size, sectorsize, align, *bitmap_offset_out, 685 bitmap_len_out, data_offset_out); 686 687 return fd; 688 out_close: 689 os_close_file(fd); 690 return err; 691 } 692 693 static int create_cow_file(char *cow_file, char *backing_file, 694 struct openflags flags, 695 int sectorsize, int alignment, int *bitmap_offset_out, 696 unsigned long *bitmap_len_out, int *data_offset_out) 697 { 698 int err, fd; 699 700 flags.c = 1; 701 fd = open_ubd_file(cow_file, &flags, 0, NULL, NULL, NULL, NULL, NULL); 702 if (fd < 0) { 703 err = fd; 704 printk(KERN_ERR "Open of COW file '%s' failed, errno = %d\n", 705 cow_file, -err); 706 goto out; 707 } 708 709 err = init_cow_file(fd, cow_file, backing_file, sectorsize, alignment, 710 bitmap_offset_out, bitmap_len_out, 711 data_offset_out); 712 if (!err) 713 return fd; 714 os_close_file(fd); 715 out: 716 return err; 717 } 718 719 static void ubd_close_dev(struct ubd *ubd_dev) 720 { 721 os_close_file(ubd_dev->fd); 722 if(ubd_dev->cow.file == NULL) 723 return; 724 725 os_close_file(ubd_dev->cow.fd); 726 vfree(ubd_dev->cow.bitmap); 727 ubd_dev->cow.bitmap = NULL; 728 } 729 730 static int ubd_open_dev(struct ubd *ubd_dev) 731 { 732 struct openflags flags; 733 char **back_ptr; 734 int err, create_cow, *create_ptr; 735 int fd; 736 737 ubd_dev->openflags = ubd_dev->boot_openflags; 738 create_cow = 0; 739 create_ptr = (ubd_dev->cow.file != NULL) ? &create_cow : NULL; 740 back_ptr = ubd_dev->no_cow ? NULL : &ubd_dev->cow.file; 741 742 fd = open_ubd_file(ubd_dev->file, &ubd_dev->openflags, ubd_dev->shared, 743 back_ptr, &ubd_dev->cow.bitmap_offset, 744 &ubd_dev->cow.bitmap_len, &ubd_dev->cow.data_offset, 745 create_ptr); 746 747 if((fd == -ENOENT) && create_cow){ 748 fd = create_cow_file(ubd_dev->file, ubd_dev->cow.file, 749 ubd_dev->openflags, SECTOR_SIZE, PAGE_SIZE, 750 &ubd_dev->cow.bitmap_offset, 751 &ubd_dev->cow.bitmap_len, 752 &ubd_dev->cow.data_offset); 753 if(fd >= 0){ 754 printk(KERN_INFO "Creating \"%s\" as COW file for " 755 "\"%s\"\n", ubd_dev->file, ubd_dev->cow.file); 756 } 757 } 758 759 if(fd < 0){ 760 printk("Failed to open '%s', errno = %d\n", ubd_dev->file, 761 -fd); 762 return fd; 763 } 764 ubd_dev->fd = fd; 765 766 if(ubd_dev->cow.file != NULL){ 767 err = -ENOMEM; 768 ubd_dev->cow.bitmap = vmalloc(ubd_dev->cow.bitmap_len); 769 if(ubd_dev->cow.bitmap == NULL){ 770 printk(KERN_ERR "Failed to vmalloc COW bitmap\n"); 771 goto error; 772 } 773 flush_tlb_kernel_vm(); 774 775 err = read_cow_bitmap(ubd_dev->fd, ubd_dev->cow.bitmap, 776 ubd_dev->cow.bitmap_offset, 777 ubd_dev->cow.bitmap_len); 778 if(err < 0) 779 goto error; 780 781 flags = ubd_dev->openflags; 782 flags.w = 0; 783 err = open_ubd_file(ubd_dev->cow.file, &flags, ubd_dev->shared, NULL, 784 NULL, NULL, NULL, NULL); 785 if(err < 0) goto error; 786 ubd_dev->cow.fd = err; 787 } 788 return 0; 789 error: 790 os_close_file(ubd_dev->fd); 791 return err; 792 } 793 794 static void ubd_device_release(struct device *dev) 795 { 796 struct ubd *ubd_dev = dev_get_drvdata(dev); 797 798 blk_mq_free_tag_set(&ubd_dev->tag_set); 799 *ubd_dev = ((struct ubd) DEFAULT_UBD); 800 } 801 802 static ssize_t serial_show(struct device *dev, 803 struct device_attribute *attr, char *buf) 804 { 805 struct gendisk *disk = dev_to_disk(dev); 806 struct ubd *ubd_dev = disk->private_data; 807 808 if (!ubd_dev) 809 return 0; 810 811 return sprintf(buf, "%s", ubd_dev->serial); 812 } 813 814 static DEVICE_ATTR_RO(serial); 815 816 static struct attribute *ubd_attrs[] = { 817 &dev_attr_serial.attr, 818 NULL, 819 }; 820 821 static umode_t ubd_attrs_are_visible(struct kobject *kobj, 822 struct attribute *a, int n) 823 { 824 return a->mode; 825 } 826 827 static const struct attribute_group ubd_attr_group = { 828 .attrs = ubd_attrs, 829 .is_visible = ubd_attrs_are_visible, 830 }; 831 832 static const struct attribute_group *ubd_attr_groups[] = { 833 &ubd_attr_group, 834 NULL, 835 }; 836 837 #define ROUND_BLOCK(n) ((n + (SECTOR_SIZE - 1)) & (-SECTOR_SIZE)) 838 839 static const struct blk_mq_ops ubd_mq_ops = { 840 .queue_rq = ubd_queue_rq, 841 }; 842 843 static int ubd_add(int n, char **error_out) 844 { 845 struct ubd *ubd_dev = &ubd_devs[n]; 846 struct queue_limits lim = { 847 .max_segments = MAX_SG, 848 .seg_boundary_mask = PAGE_SIZE - 1, 849 }; 850 struct gendisk *disk; 851 int err = 0; 852 853 if(ubd_dev->file == NULL) 854 goto out; 855 856 if (ubd_dev->cow.file) 857 lim.max_hw_sectors = 8 * sizeof(long); 858 if (!ubd_dev->no_trim) { 859 lim.max_hw_discard_sectors = UBD_MAX_REQUEST; 860 lim.max_write_zeroes_sectors = UBD_MAX_REQUEST; 861 } 862 863 err = ubd_file_size(ubd_dev, &ubd_dev->size); 864 if(err < 0){ 865 *error_out = "Couldn't determine size of device's file"; 866 goto out; 867 } 868 869 err = ubd_open_dev(ubd_dev); 870 if (err) { 871 pr_err("ubd%c: Can't open \"%s\": errno = %d\n", 872 'a' + n, ubd_dev->file, -err); 873 goto out; 874 } 875 876 ubd_dev->size = ROUND_BLOCK(ubd_dev->size); 877 878 ubd_dev->tag_set.ops = &ubd_mq_ops; 879 ubd_dev->tag_set.queue_depth = 64; 880 ubd_dev->tag_set.numa_node = NUMA_NO_NODE; 881 ubd_dev->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; 882 ubd_dev->tag_set.driver_data = ubd_dev; 883 ubd_dev->tag_set.nr_hw_queues = 1; 884 885 err = blk_mq_alloc_tag_set(&ubd_dev->tag_set); 886 if (err) 887 goto out_close; 888 889 disk = blk_mq_alloc_disk(&ubd_dev->tag_set, &lim, ubd_dev); 890 if (IS_ERR(disk)) { 891 err = PTR_ERR(disk); 892 goto out_cleanup_tags; 893 } 894 895 blk_queue_flag_set(QUEUE_FLAG_NONROT, disk->queue); 896 blk_queue_write_cache(disk->queue, true, false); 897 disk->major = UBD_MAJOR; 898 disk->first_minor = n << UBD_SHIFT; 899 disk->minors = 1 << UBD_SHIFT; 900 disk->fops = &ubd_blops; 901 set_capacity(disk, ubd_dev->size / 512); 902 sprintf(disk->disk_name, "ubd%c", 'a' + n); 903 disk->private_data = ubd_dev; 904 set_disk_ro(disk, !ubd_dev->openflags.w); 905 906 ubd_dev->pdev.id = n; 907 ubd_dev->pdev.name = DRIVER_NAME; 908 ubd_dev->pdev.dev.release = ubd_device_release; 909 dev_set_drvdata(&ubd_dev->pdev.dev, ubd_dev); 910 platform_device_register(&ubd_dev->pdev); 911 912 err = device_add_disk(&ubd_dev->pdev.dev, disk, ubd_attr_groups); 913 if (err) 914 goto out_cleanup_disk; 915 916 return 0; 917 918 out_cleanup_disk: 919 put_disk(disk); 920 out_cleanup_tags: 921 blk_mq_free_tag_set(&ubd_dev->tag_set); 922 out_close: 923 ubd_close_dev(ubd_dev); 924 out: 925 return err; 926 } 927 928 static int ubd_config(char *str, char **error_out) 929 { 930 int n, ret; 931 932 /* This string is possibly broken up and stored, so it's only 933 * freed if ubd_setup_common fails, or if only general options 934 * were set. 935 */ 936 str = kstrdup(str, GFP_KERNEL); 937 if (str == NULL) { 938 *error_out = "Failed to allocate memory"; 939 return -ENOMEM; 940 } 941 942 ret = ubd_setup_common(str, &n, error_out); 943 if (ret) 944 goto err_free; 945 946 if (n == -1) { 947 ret = 0; 948 goto err_free; 949 } 950 951 mutex_lock(&ubd_lock); 952 ret = ubd_add(n, error_out); 953 if (ret) 954 ubd_devs[n].file = NULL; 955 mutex_unlock(&ubd_lock); 956 957 out: 958 return ret; 959 960 err_free: 961 kfree(str); 962 goto out; 963 } 964 965 static int ubd_get_config(char *name, char *str, int size, char **error_out) 966 { 967 struct ubd *ubd_dev; 968 int n, len = 0; 969 970 n = parse_unit(&name); 971 if((n >= MAX_DEV) || (n < 0)){ 972 *error_out = "ubd_get_config : device number out of range"; 973 return -1; 974 } 975 976 ubd_dev = &ubd_devs[n]; 977 mutex_lock(&ubd_lock); 978 979 if(ubd_dev->file == NULL){ 980 CONFIG_CHUNK(str, size, len, "", 1); 981 goto out; 982 } 983 984 CONFIG_CHUNK(str, size, len, ubd_dev->file, 0); 985 986 if(ubd_dev->cow.file != NULL){ 987 CONFIG_CHUNK(str, size, len, ",", 0); 988 CONFIG_CHUNK(str, size, len, ubd_dev->cow.file, 1); 989 } 990 else CONFIG_CHUNK(str, size, len, "", 1); 991 992 out: 993 mutex_unlock(&ubd_lock); 994 return len; 995 } 996 997 static int ubd_id(char **str, int *start_out, int *end_out) 998 { 999 int n; 1000 1001 n = parse_unit(str); 1002 *start_out = 0; 1003 *end_out = MAX_DEV - 1; 1004 return n; 1005 } 1006 1007 static int ubd_remove(int n, char **error_out) 1008 { 1009 struct ubd *ubd_dev; 1010 int err = -ENODEV; 1011 1012 mutex_lock(&ubd_lock); 1013 1014 ubd_dev = &ubd_devs[n]; 1015 1016 if(ubd_dev->file == NULL) 1017 goto out; 1018 1019 if (ubd_dev->disk) { 1020 /* you cannot remove a open disk */ 1021 err = -EBUSY; 1022 if (disk_openers(ubd_dev->disk)) 1023 goto out; 1024 1025 del_gendisk(ubd_dev->disk); 1026 ubd_close_dev(ubd_dev); 1027 put_disk(ubd_dev->disk); 1028 } 1029 1030 err = 0; 1031 platform_device_unregister(&ubd_dev->pdev); 1032 out: 1033 mutex_unlock(&ubd_lock); 1034 return err; 1035 } 1036 1037 /* All these are called by mconsole in process context and without 1038 * ubd-specific locks. The structure itself is const except for .list. 1039 */ 1040 static struct mc_device ubd_mc = { 1041 .list = LIST_HEAD_INIT(ubd_mc.list), 1042 .name = "ubd", 1043 .config = ubd_config, 1044 .get_config = ubd_get_config, 1045 .id = ubd_id, 1046 .remove = ubd_remove, 1047 }; 1048 1049 static int __init ubd_mc_init(void) 1050 { 1051 mconsole_register_dev(&ubd_mc); 1052 return 0; 1053 } 1054 1055 __initcall(ubd_mc_init); 1056 1057 static int __init ubd0_init(void) 1058 { 1059 struct ubd *ubd_dev = &ubd_devs[0]; 1060 1061 mutex_lock(&ubd_lock); 1062 if(ubd_dev->file == NULL) 1063 ubd_dev->file = "root_fs"; 1064 mutex_unlock(&ubd_lock); 1065 1066 return 0; 1067 } 1068 1069 __initcall(ubd0_init); 1070 1071 /* Used in ubd_init, which is an initcall */ 1072 static struct platform_driver ubd_driver = { 1073 .driver = { 1074 .name = DRIVER_NAME, 1075 }, 1076 }; 1077 1078 static int __init ubd_init(void) 1079 { 1080 char *error; 1081 int i, err; 1082 1083 if (register_blkdev(UBD_MAJOR, "ubd")) 1084 return -1; 1085 1086 irq_req_buffer = kmalloc_array(UBD_REQ_BUFFER_SIZE, 1087 sizeof(struct io_thread_req *), 1088 GFP_KERNEL 1089 ); 1090 irq_remainder = 0; 1091 1092 if (irq_req_buffer == NULL) { 1093 printk(KERN_ERR "Failed to initialize ubd buffering\n"); 1094 return -ENOMEM; 1095 } 1096 io_req_buffer = kmalloc_array(UBD_REQ_BUFFER_SIZE, 1097 sizeof(struct io_thread_req *), 1098 GFP_KERNEL 1099 ); 1100 1101 io_remainder = 0; 1102 1103 if (io_req_buffer == NULL) { 1104 printk(KERN_ERR "Failed to initialize ubd buffering\n"); 1105 return -ENOMEM; 1106 } 1107 platform_driver_register(&ubd_driver); 1108 mutex_lock(&ubd_lock); 1109 for (i = 0; i < MAX_DEV; i++){ 1110 err = ubd_add(i, &error); 1111 if(err) 1112 printk(KERN_ERR "Failed to initialize ubd device %d :" 1113 "%s\n", i, error); 1114 } 1115 mutex_unlock(&ubd_lock); 1116 return 0; 1117 } 1118 1119 late_initcall(ubd_init); 1120 1121 static int __init ubd_driver_init(void){ 1122 unsigned long stack; 1123 int err; 1124 1125 /* Set by CONFIG_BLK_DEV_UBD_SYNC or ubd=sync.*/ 1126 if(global_openflags.s){ 1127 printk(KERN_INFO "ubd: Synchronous mode\n"); 1128 /* Letting ubd=sync be like using ubd#s= instead of ubd#= is 1129 * enough. So use anyway the io thread. */ 1130 } 1131 stack = alloc_stack(0, 0); 1132 io_pid = start_io_thread(stack + PAGE_SIZE, &thread_fd); 1133 if(io_pid < 0){ 1134 printk(KERN_ERR 1135 "ubd : Failed to start I/O thread (errno = %d) - " 1136 "falling back to synchronous I/O\n", -io_pid); 1137 io_pid = -1; 1138 return 0; 1139 } 1140 err = um_request_irq(UBD_IRQ, thread_fd, IRQ_READ, ubd_intr, 1141 0, "ubd", ubd_devs); 1142 if(err < 0) 1143 printk(KERN_ERR "um_request_irq failed - errno = %d\n", -err); 1144 return 0; 1145 } 1146 1147 device_initcall(ubd_driver_init); 1148 1149 static void cowify_bitmap(__u64 io_offset, int length, unsigned long *cow_mask, 1150 __u64 *cow_offset, unsigned long *bitmap, 1151 __u64 bitmap_offset, unsigned long *bitmap_words, 1152 __u64 bitmap_len) 1153 { 1154 __u64 sector = io_offset >> SECTOR_SHIFT; 1155 int i, update_bitmap = 0; 1156 1157 for (i = 0; i < length >> SECTOR_SHIFT; i++) { 1158 if(cow_mask != NULL) 1159 ubd_set_bit(i, (unsigned char *) cow_mask); 1160 if(ubd_test_bit(sector + i, (unsigned char *) bitmap)) 1161 continue; 1162 1163 update_bitmap = 1; 1164 ubd_set_bit(sector + i, (unsigned char *) bitmap); 1165 } 1166 1167 if(!update_bitmap) 1168 return; 1169 1170 *cow_offset = sector / (sizeof(unsigned long) * 8); 1171 1172 /* This takes care of the case where we're exactly at the end of the 1173 * device, and *cow_offset + 1 is off the end. So, just back it up 1174 * by one word. Thanks to Lynn Kerby for the fix and James McMechan 1175 * for the original diagnosis. 1176 */ 1177 if (*cow_offset == (DIV_ROUND_UP(bitmap_len, 1178 sizeof(unsigned long)) - 1)) 1179 (*cow_offset)--; 1180 1181 bitmap_words[0] = bitmap[*cow_offset]; 1182 bitmap_words[1] = bitmap[*cow_offset + 1]; 1183 1184 *cow_offset *= sizeof(unsigned long); 1185 *cow_offset += bitmap_offset; 1186 } 1187 1188 static void cowify_req(struct io_thread_req *req, struct io_desc *segment, 1189 unsigned long offset, unsigned long *bitmap, 1190 __u64 bitmap_offset, __u64 bitmap_len) 1191 { 1192 __u64 sector = offset >> SECTOR_SHIFT; 1193 int i; 1194 1195 if (segment->length > (sizeof(segment->sector_mask) * 8) << SECTOR_SHIFT) 1196 panic("Operation too long"); 1197 1198 if (req_op(req->req) == REQ_OP_READ) { 1199 for (i = 0; i < segment->length >> SECTOR_SHIFT; i++) { 1200 if(ubd_test_bit(sector + i, (unsigned char *) bitmap)) 1201 ubd_set_bit(i, (unsigned char *) 1202 &segment->sector_mask); 1203 } 1204 } else { 1205 cowify_bitmap(offset, segment->length, &segment->sector_mask, 1206 &segment->cow_offset, bitmap, bitmap_offset, 1207 segment->bitmap_words, bitmap_len); 1208 } 1209 } 1210 1211 static void ubd_map_req(struct ubd *dev, struct io_thread_req *io_req, 1212 struct request *req) 1213 { 1214 struct bio_vec bvec; 1215 struct req_iterator iter; 1216 int i = 0; 1217 unsigned long byte_offset = io_req->offset; 1218 enum req_op op = req_op(req); 1219 1220 if (op == REQ_OP_WRITE_ZEROES || op == REQ_OP_DISCARD) { 1221 io_req->io_desc[0].buffer = NULL; 1222 io_req->io_desc[0].length = blk_rq_bytes(req); 1223 } else { 1224 rq_for_each_segment(bvec, req, iter) { 1225 BUG_ON(i >= io_req->desc_cnt); 1226 1227 io_req->io_desc[i].buffer = bvec_virt(&bvec); 1228 io_req->io_desc[i].length = bvec.bv_len; 1229 i++; 1230 } 1231 } 1232 1233 if (dev->cow.file) { 1234 for (i = 0; i < io_req->desc_cnt; i++) { 1235 cowify_req(io_req, &io_req->io_desc[i], byte_offset, 1236 dev->cow.bitmap, dev->cow.bitmap_offset, 1237 dev->cow.bitmap_len); 1238 byte_offset += io_req->io_desc[i].length; 1239 } 1240 1241 } 1242 } 1243 1244 static struct io_thread_req *ubd_alloc_req(struct ubd *dev, struct request *req, 1245 int desc_cnt) 1246 { 1247 struct io_thread_req *io_req; 1248 int i; 1249 1250 io_req = kmalloc(sizeof(*io_req) + 1251 (desc_cnt * sizeof(struct io_desc)), 1252 GFP_ATOMIC); 1253 if (!io_req) 1254 return NULL; 1255 1256 io_req->req = req; 1257 if (dev->cow.file) 1258 io_req->fds[0] = dev->cow.fd; 1259 else 1260 io_req->fds[0] = dev->fd; 1261 io_req->error = 0; 1262 io_req->sectorsize = SECTOR_SIZE; 1263 io_req->fds[1] = dev->fd; 1264 io_req->offset = (u64) blk_rq_pos(req) << SECTOR_SHIFT; 1265 io_req->offsets[0] = 0; 1266 io_req->offsets[1] = dev->cow.data_offset; 1267 1268 for (i = 0 ; i < desc_cnt; i++) { 1269 io_req->io_desc[i].sector_mask = 0; 1270 io_req->io_desc[i].cow_offset = -1; 1271 } 1272 1273 return io_req; 1274 } 1275 1276 static int ubd_submit_request(struct ubd *dev, struct request *req) 1277 { 1278 int segs = 0; 1279 struct io_thread_req *io_req; 1280 int ret; 1281 enum req_op op = req_op(req); 1282 1283 if (op == REQ_OP_FLUSH) 1284 segs = 0; 1285 else if (op == REQ_OP_WRITE_ZEROES || op == REQ_OP_DISCARD) 1286 segs = 1; 1287 else 1288 segs = blk_rq_nr_phys_segments(req); 1289 1290 io_req = ubd_alloc_req(dev, req, segs); 1291 if (!io_req) 1292 return -ENOMEM; 1293 1294 io_req->desc_cnt = segs; 1295 if (segs) 1296 ubd_map_req(dev, io_req, req); 1297 1298 ret = os_write_file(thread_fd, &io_req, sizeof(io_req)); 1299 if (ret != sizeof(io_req)) { 1300 if (ret != -EAGAIN) 1301 pr_err("write to io thread failed: %d\n", -ret); 1302 kfree(io_req); 1303 } 1304 return ret; 1305 } 1306 1307 static blk_status_t ubd_queue_rq(struct blk_mq_hw_ctx *hctx, 1308 const struct blk_mq_queue_data *bd) 1309 { 1310 struct ubd *ubd_dev = hctx->queue->queuedata; 1311 struct request *req = bd->rq; 1312 int ret = 0, res = BLK_STS_OK; 1313 1314 blk_mq_start_request(req); 1315 1316 spin_lock_irq(&ubd_dev->lock); 1317 1318 switch (req_op(req)) { 1319 case REQ_OP_FLUSH: 1320 case REQ_OP_READ: 1321 case REQ_OP_WRITE: 1322 case REQ_OP_DISCARD: 1323 case REQ_OP_WRITE_ZEROES: 1324 ret = ubd_submit_request(ubd_dev, req); 1325 break; 1326 default: 1327 WARN_ON_ONCE(1); 1328 res = BLK_STS_NOTSUPP; 1329 } 1330 1331 spin_unlock_irq(&ubd_dev->lock); 1332 1333 if (ret < 0) { 1334 if (ret == -ENOMEM) 1335 res = BLK_STS_RESOURCE; 1336 else 1337 res = BLK_STS_DEV_RESOURCE; 1338 } 1339 1340 return res; 1341 } 1342 1343 static int ubd_getgeo(struct block_device *bdev, struct hd_geometry *geo) 1344 { 1345 struct ubd *ubd_dev = bdev->bd_disk->private_data; 1346 1347 geo->heads = 128; 1348 geo->sectors = 32; 1349 geo->cylinders = ubd_dev->size / (128 * 32 * 512); 1350 return 0; 1351 } 1352 1353 static int ubd_ioctl(struct block_device *bdev, blk_mode_t mode, 1354 unsigned int cmd, unsigned long arg) 1355 { 1356 struct ubd *ubd_dev = bdev->bd_disk->private_data; 1357 u16 ubd_id[ATA_ID_WORDS]; 1358 1359 switch (cmd) { 1360 struct cdrom_volctrl volume; 1361 case HDIO_GET_IDENTITY: 1362 memset(&ubd_id, 0, ATA_ID_WORDS * 2); 1363 ubd_id[ATA_ID_CYLS] = ubd_dev->size / (128 * 32 * 512); 1364 ubd_id[ATA_ID_HEADS] = 128; 1365 ubd_id[ATA_ID_SECTORS] = 32; 1366 if(copy_to_user((char __user *) arg, (char *) &ubd_id, 1367 sizeof(ubd_id))) 1368 return -EFAULT; 1369 return 0; 1370 1371 case CDROMVOLREAD: 1372 if(copy_from_user(&volume, (char __user *) arg, sizeof(volume))) 1373 return -EFAULT; 1374 volume.channel0 = 255; 1375 volume.channel1 = 255; 1376 volume.channel2 = 255; 1377 volume.channel3 = 255; 1378 if(copy_to_user((char __user *) arg, &volume, sizeof(volume))) 1379 return -EFAULT; 1380 return 0; 1381 } 1382 return -EINVAL; 1383 } 1384 1385 static int map_error(int error_code) 1386 { 1387 switch (error_code) { 1388 case 0: 1389 return BLK_STS_OK; 1390 case ENOSYS: 1391 case EOPNOTSUPP: 1392 return BLK_STS_NOTSUPP; 1393 case ENOSPC: 1394 return BLK_STS_NOSPC; 1395 } 1396 return BLK_STS_IOERR; 1397 } 1398 1399 /* 1400 * Everything from here onwards *IS NOT PART OF THE KERNEL* 1401 * 1402 * The following functions are part of UML hypervisor code. 1403 * All functions from here onwards are executed as a helper 1404 * thread and are not allowed to execute any kernel functions. 1405 * 1406 * Any communication must occur strictly via shared memory and IPC. 1407 * 1408 * Do not add printks, locks, kernel memory operations, etc - it 1409 * will result in unpredictable behaviour and/or crashes. 1410 */ 1411 1412 static int update_bitmap(struct io_thread_req *req, struct io_desc *segment) 1413 { 1414 int n; 1415 1416 if (segment->cow_offset == -1) 1417 return map_error(0); 1418 1419 n = os_pwrite_file(req->fds[1], &segment->bitmap_words, 1420 sizeof(segment->bitmap_words), segment->cow_offset); 1421 if (n != sizeof(segment->bitmap_words)) 1422 return map_error(-n); 1423 1424 return map_error(0); 1425 } 1426 1427 static void do_io(struct io_thread_req *req, struct io_desc *desc) 1428 { 1429 char *buf = NULL; 1430 unsigned long len; 1431 int n, nsectors, start, end, bit; 1432 __u64 off; 1433 1434 /* FLUSH is really a special case, we cannot "case" it with others */ 1435 1436 if (req_op(req->req) == REQ_OP_FLUSH) { 1437 /* fds[0] is always either the rw image or our cow file */ 1438 req->error = map_error(-os_sync_file(req->fds[0])); 1439 return; 1440 } 1441 1442 nsectors = desc->length / req->sectorsize; 1443 start = 0; 1444 do { 1445 bit = ubd_test_bit(start, (unsigned char *) &desc->sector_mask); 1446 end = start; 1447 while((end < nsectors) && 1448 (ubd_test_bit(end, (unsigned char *) &desc->sector_mask) == bit)) 1449 end++; 1450 1451 off = req->offset + req->offsets[bit] + 1452 start * req->sectorsize; 1453 len = (end - start) * req->sectorsize; 1454 if (desc->buffer != NULL) 1455 buf = &desc->buffer[start * req->sectorsize]; 1456 1457 switch (req_op(req->req)) { 1458 case REQ_OP_READ: 1459 n = 0; 1460 do { 1461 buf = &buf[n]; 1462 len -= n; 1463 n = os_pread_file(req->fds[bit], buf, len, off); 1464 if (n < 0) { 1465 req->error = map_error(-n); 1466 return; 1467 } 1468 } while((n < len) && (n != 0)); 1469 if (n < len) memset(&buf[n], 0, len - n); 1470 break; 1471 case REQ_OP_WRITE: 1472 n = os_pwrite_file(req->fds[bit], buf, len, off); 1473 if(n != len){ 1474 req->error = map_error(-n); 1475 return; 1476 } 1477 break; 1478 case REQ_OP_DISCARD: 1479 n = os_falloc_punch(req->fds[bit], off, len); 1480 if (n) { 1481 req->error = map_error(-n); 1482 return; 1483 } 1484 break; 1485 case REQ_OP_WRITE_ZEROES: 1486 n = os_falloc_zeroes(req->fds[bit], off, len); 1487 if (n) { 1488 req->error = map_error(-n); 1489 return; 1490 } 1491 break; 1492 default: 1493 WARN_ON_ONCE(1); 1494 req->error = BLK_STS_NOTSUPP; 1495 return; 1496 } 1497 1498 start = end; 1499 } while(start < nsectors); 1500 1501 req->offset += len; 1502 req->error = update_bitmap(req, desc); 1503 } 1504 1505 /* Changed in start_io_thread, which is serialized by being called only 1506 * from ubd_init, which is an initcall. 1507 */ 1508 int kernel_fd = -1; 1509 1510 /* Only changed by the io thread. XXX: currently unused. */ 1511 static int io_count; 1512 1513 int io_thread(void *arg) 1514 { 1515 int n, count, written, res; 1516 1517 os_fix_helper_signals(); 1518 1519 while(1){ 1520 n = bulk_req_safe_read( 1521 kernel_fd, 1522 io_req_buffer, 1523 &io_remainder, 1524 &io_remainder_size, 1525 UBD_REQ_BUFFER_SIZE 1526 ); 1527 if (n <= 0) { 1528 if (n == -EAGAIN) 1529 ubd_read_poll(-1); 1530 1531 continue; 1532 } 1533 1534 for (count = 0; count < n/sizeof(struct io_thread_req *); count++) { 1535 struct io_thread_req *req = (*io_req_buffer)[count]; 1536 int i; 1537 1538 io_count++; 1539 for (i = 0; !req->error && i < req->desc_cnt; i++) 1540 do_io(req, &(req->io_desc[i])); 1541 1542 } 1543 1544 written = 0; 1545 1546 do { 1547 res = os_write_file(kernel_fd, 1548 ((char *) io_req_buffer) + written, 1549 n - written); 1550 if (res >= 0) { 1551 written += res; 1552 } 1553 if (written < n) { 1554 ubd_write_poll(-1); 1555 } 1556 } while (written < n); 1557 } 1558 1559 return 0; 1560 } 1561