1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2016 Mellanox Technologies, Ltd. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice unmodified, this list of conditions, and the following 13 * disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/malloc.h> 36 #include <sys/kernel.h> 37 #include <sys/sysctl.h> 38 #include <sys/proc.h> 39 #include <sys/sglist.h> 40 #include <sys/sleepqueue.h> 41 #include <sys/lock.h> 42 #include <sys/mutex.h> 43 #include <sys/bus.h> 44 #include <sys/fcntl.h> 45 #include <sys/file.h> 46 #include <sys/filio.h> 47 #include <sys/rwlock.h> 48 49 #include <vm/vm.h> 50 #include <vm/pmap.h> 51 52 #include <machine/stdarg.h> 53 54 #include <linux/kobject.h> 55 #include <linux/device.h> 56 #include <linux/slab.h> 57 #include <linux/module.h> 58 #include <linux/cdev.h> 59 #include <linux/file.h> 60 #include <linux/sysfs.h> 61 #include <linux/mm.h> 62 #include <linux/io.h> 63 #include <linux/vmalloc.h> 64 #include <linux/netdevice.h> 65 #include <linux/timer.h> 66 #include <linux/workqueue.h> 67 #include <linux/rcupdate.h> 68 #include <linux/interrupt.h> 69 70 #include <vm/vm_pager.h> 71 72 MALLOC_DEFINE(M_KMALLOC, "linux", "Linux kmalloc compat"); 73 74 #include <linux/rbtree.h> 75 /* Undo Linux compat changes. */ 76 #undef RB_ROOT 77 #undef file 78 #undef cdev 79 #define RB_ROOT(head) (head)->rbh_root 80 81 struct kobject linux_class_root; 82 struct device linux_root_device; 83 struct class linux_class_misc; 84 struct list_head pci_drivers; 85 struct list_head pci_devices; 86 struct net init_net; 87 spinlock_t pci_lock; 88 struct sx linux_global_rcu_lock; 89 90 unsigned long linux_timer_hz_mask; 91 92 int 93 panic_cmp(struct rb_node *one, struct rb_node *two) 94 { 95 panic("no cmp"); 96 } 97 98 RB_GENERATE(linux_root, rb_node, __entry, panic_cmp); 99 100 int 101 kobject_set_name_vargs(struct kobject *kobj, const char *fmt, va_list args) 102 { 103 va_list tmp_va; 104 int len; 105 char *old; 106 char *name; 107 char dummy; 108 109 old = kobj->name; 110 111 if (old && fmt == NULL) 112 return (0); 113 114 /* compute length of string */ 115 va_copy(tmp_va, args); 116 len = vsnprintf(&dummy, 0, fmt, tmp_va); 117 va_end(tmp_va); 118 119 /* account for zero termination */ 120 len++; 121 122 /* check for error */ 123 if (len < 1) 124 return (-EINVAL); 125 126 /* allocate memory for string */ 127 name = kzalloc(len, GFP_KERNEL); 128 if (name == NULL) 129 return (-ENOMEM); 130 vsnprintf(name, len, fmt, args); 131 kobj->name = name; 132 133 /* free old string */ 134 kfree(old); 135 136 /* filter new string */ 137 for (; *name != '\0'; name++) 138 if (*name == '/') 139 *name = '!'; 140 return (0); 141 } 142 143 int 144 kobject_set_name(struct kobject *kobj, const char *fmt, ...) 145 { 146 va_list args; 147 int error; 148 149 va_start(args, fmt); 150 error = kobject_set_name_vargs(kobj, fmt, args); 151 va_end(args); 152 153 return (error); 154 } 155 156 static int 157 kobject_add_complete(struct kobject *kobj, struct kobject *parent) 158 { 159 const struct kobj_type *t; 160 int error; 161 162 kobj->parent = parent; 163 error = sysfs_create_dir(kobj); 164 if (error == 0 && kobj->ktype && kobj->ktype->default_attrs) { 165 struct attribute **attr; 166 t = kobj->ktype; 167 168 for (attr = t->default_attrs; *attr != NULL; attr++) { 169 error = sysfs_create_file(kobj, *attr); 170 if (error) 171 break; 172 } 173 if (error) 174 sysfs_remove_dir(kobj); 175 176 } 177 return (error); 178 } 179 180 int 181 kobject_add(struct kobject *kobj, struct kobject *parent, const char *fmt, ...) 182 { 183 va_list args; 184 int error; 185 186 va_start(args, fmt); 187 error = kobject_set_name_vargs(kobj, fmt, args); 188 va_end(args); 189 if (error) 190 return (error); 191 192 return kobject_add_complete(kobj, parent); 193 } 194 195 void 196 linux_kobject_release(struct kref *kref) 197 { 198 struct kobject *kobj; 199 char *name; 200 201 kobj = container_of(kref, struct kobject, kref); 202 sysfs_remove_dir(kobj); 203 name = kobj->name; 204 if (kobj->ktype && kobj->ktype->release) 205 kobj->ktype->release(kobj); 206 kfree(name); 207 } 208 209 static void 210 linux_kobject_kfree(struct kobject *kobj) 211 { 212 kfree(kobj); 213 } 214 215 static void 216 linux_kobject_kfree_name(struct kobject *kobj) 217 { 218 if (kobj) { 219 kfree(kobj->name); 220 } 221 } 222 223 const struct kobj_type linux_kfree_type = { 224 .release = linux_kobject_kfree 225 }; 226 227 static void 228 linux_device_release(struct device *dev) 229 { 230 pr_debug("linux_device_release: %s\n", dev_name(dev)); 231 kfree(dev); 232 } 233 234 static ssize_t 235 linux_class_show(struct kobject *kobj, struct attribute *attr, char *buf) 236 { 237 struct class_attribute *dattr; 238 ssize_t error; 239 240 dattr = container_of(attr, struct class_attribute, attr); 241 error = -EIO; 242 if (dattr->show) 243 error = dattr->show(container_of(kobj, struct class, kobj), 244 dattr, buf); 245 return (error); 246 } 247 248 static ssize_t 249 linux_class_store(struct kobject *kobj, struct attribute *attr, const char *buf, 250 size_t count) 251 { 252 struct class_attribute *dattr; 253 ssize_t error; 254 255 dattr = container_of(attr, struct class_attribute, attr); 256 error = -EIO; 257 if (dattr->store) 258 error = dattr->store(container_of(kobj, struct class, kobj), 259 dattr, buf, count); 260 return (error); 261 } 262 263 static void 264 linux_class_release(struct kobject *kobj) 265 { 266 struct class *class; 267 268 class = container_of(kobj, struct class, kobj); 269 if (class->class_release) 270 class->class_release(class); 271 } 272 273 static const struct sysfs_ops linux_class_sysfs = { 274 .show = linux_class_show, 275 .store = linux_class_store, 276 }; 277 278 const struct kobj_type linux_class_ktype = { 279 .release = linux_class_release, 280 .sysfs_ops = &linux_class_sysfs 281 }; 282 283 static void 284 linux_dev_release(struct kobject *kobj) 285 { 286 struct device *dev; 287 288 dev = container_of(kobj, struct device, kobj); 289 /* This is the precedence defined by linux. */ 290 if (dev->release) 291 dev->release(dev); 292 else if (dev->class && dev->class->dev_release) 293 dev->class->dev_release(dev); 294 } 295 296 static ssize_t 297 linux_dev_show(struct kobject *kobj, struct attribute *attr, char *buf) 298 { 299 struct device_attribute *dattr; 300 ssize_t error; 301 302 dattr = container_of(attr, struct device_attribute, attr); 303 error = -EIO; 304 if (dattr->show) 305 error = dattr->show(container_of(kobj, struct device, kobj), 306 dattr, buf); 307 return (error); 308 } 309 310 static ssize_t 311 linux_dev_store(struct kobject *kobj, struct attribute *attr, const char *buf, 312 size_t count) 313 { 314 struct device_attribute *dattr; 315 ssize_t error; 316 317 dattr = container_of(attr, struct device_attribute, attr); 318 error = -EIO; 319 if (dattr->store) 320 error = dattr->store(container_of(kobj, struct device, kobj), 321 dattr, buf, count); 322 return (error); 323 } 324 325 static const struct sysfs_ops linux_dev_sysfs = { 326 .show = linux_dev_show, 327 .store = linux_dev_store, 328 }; 329 330 const struct kobj_type linux_dev_ktype = { 331 .release = linux_dev_release, 332 .sysfs_ops = &linux_dev_sysfs 333 }; 334 335 struct device * 336 device_create(struct class *class, struct device *parent, dev_t devt, 337 void *drvdata, const char *fmt, ...) 338 { 339 struct device *dev; 340 va_list args; 341 342 dev = kzalloc(sizeof(*dev), M_WAITOK); 343 dev->parent = parent; 344 dev->class = class; 345 dev->devt = devt; 346 dev->driver_data = drvdata; 347 dev->release = linux_device_release; 348 va_start(args, fmt); 349 kobject_set_name_vargs(&dev->kobj, fmt, args); 350 va_end(args); 351 device_register(dev); 352 353 return (dev); 354 } 355 356 int 357 kobject_init_and_add(struct kobject *kobj, const struct kobj_type *ktype, 358 struct kobject *parent, const char *fmt, ...) 359 { 360 va_list args; 361 int error; 362 363 kobject_init(kobj, ktype); 364 kobj->ktype = ktype; 365 kobj->parent = parent; 366 kobj->name = NULL; 367 368 va_start(args, fmt); 369 error = kobject_set_name_vargs(kobj, fmt, args); 370 va_end(args); 371 if (error) 372 return (error); 373 return kobject_add_complete(kobj, parent); 374 } 375 376 static void 377 linux_file_dtor(void *cdp) 378 { 379 struct linux_file *filp; 380 381 filp = cdp; 382 filp->f_op->release(filp->f_vnode, filp); 383 vdrop(filp->f_vnode); 384 kfree(filp); 385 } 386 387 static int 388 linux_dev_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 389 { 390 struct linux_cdev *ldev; 391 struct linux_file *filp; 392 struct file *file; 393 int error; 394 395 file = curthread->td_fpop; 396 ldev = dev->si_drv1; 397 if (ldev == NULL) 398 return (ENODEV); 399 filp = kzalloc(sizeof(*filp), GFP_KERNEL); 400 filp->f_dentry = &filp->f_dentry_store; 401 filp->f_op = ldev->ops; 402 filp->f_flags = file->f_flag; 403 vhold(file->f_vnode); 404 filp->f_vnode = file->f_vnode; 405 if (filp->f_op->open) { 406 error = -filp->f_op->open(file->f_vnode, filp); 407 if (error) { 408 kfree(filp); 409 return (error); 410 } 411 } 412 error = devfs_set_cdevpriv(filp, linux_file_dtor); 413 if (error) { 414 filp->f_op->release(file->f_vnode, filp); 415 kfree(filp); 416 return (error); 417 } 418 419 return 0; 420 } 421 422 static int 423 linux_dev_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 424 { 425 struct linux_cdev *ldev; 426 struct linux_file *filp; 427 struct file *file; 428 int error; 429 430 file = curthread->td_fpop; 431 ldev = dev->si_drv1; 432 if (ldev == NULL) 433 return (0); 434 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 435 return (error); 436 filp->f_flags = file->f_flag; 437 devfs_clear_cdevpriv(); 438 439 440 return (0); 441 } 442 443 static int 444 linux_dev_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, 445 struct thread *td) 446 { 447 struct linux_cdev *ldev; 448 struct linux_file *filp; 449 struct file *file; 450 int error; 451 452 file = curthread->td_fpop; 453 ldev = dev->si_drv1; 454 if (ldev == NULL) 455 return (0); 456 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 457 return (error); 458 filp->f_flags = file->f_flag; 459 /* 460 * Linux does not have a generic ioctl copyin/copyout layer. All 461 * linux ioctls must be converted to void ioctls which pass a 462 * pointer to the address of the data. We want the actual user 463 * address so we dereference here. 464 */ 465 data = *(void **)data; 466 if (filp->f_op->unlocked_ioctl) 467 error = -filp->f_op->unlocked_ioctl(filp, cmd, (u_long)data); 468 else 469 error = ENOTTY; 470 471 return (error); 472 } 473 474 static int 475 linux_dev_read(struct cdev *dev, struct uio *uio, int ioflag) 476 { 477 struct linux_cdev *ldev; 478 struct linux_file *filp; 479 struct file *file; 480 ssize_t bytes; 481 int error; 482 483 file = curthread->td_fpop; 484 ldev = dev->si_drv1; 485 if (ldev == NULL) 486 return (0); 487 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 488 return (error); 489 filp->f_flags = file->f_flag; 490 if (uio->uio_iovcnt != 1) 491 panic("linux_dev_read: uio %p iovcnt %d", 492 uio, uio->uio_iovcnt); 493 if (filp->f_op->read) { 494 bytes = filp->f_op->read(filp, uio->uio_iov->iov_base, 495 uio->uio_iov->iov_len, &uio->uio_offset); 496 if (bytes >= 0) { 497 uio->uio_iov->iov_base = 498 ((uint8_t *)uio->uio_iov->iov_base) + bytes; 499 uio->uio_iov->iov_len -= bytes; 500 uio->uio_resid -= bytes; 501 } else 502 error = -bytes; 503 } else 504 error = ENXIO; 505 506 return (error); 507 } 508 509 static int 510 linux_dev_write(struct cdev *dev, struct uio *uio, int ioflag) 511 { 512 struct linux_cdev *ldev; 513 struct linux_file *filp; 514 struct file *file; 515 ssize_t bytes; 516 int error; 517 518 file = curthread->td_fpop; 519 ldev = dev->si_drv1; 520 if (ldev == NULL) 521 return (0); 522 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 523 return (error); 524 filp->f_flags = file->f_flag; 525 if (uio->uio_iovcnt != 1) 526 panic("linux_dev_write: uio %p iovcnt %d", 527 uio, uio->uio_iovcnt); 528 if (filp->f_op->write) { 529 bytes = filp->f_op->write(filp, uio->uio_iov->iov_base, 530 uio->uio_iov->iov_len, &uio->uio_offset); 531 if (bytes >= 0) { 532 uio->uio_iov->iov_base = 533 ((uint8_t *)uio->uio_iov->iov_base) + bytes; 534 uio->uio_iov->iov_len -= bytes; 535 uio->uio_resid -= bytes; 536 } else 537 error = -bytes; 538 } else 539 error = ENXIO; 540 541 return (error); 542 } 543 544 static int 545 linux_dev_poll(struct cdev *dev, int events, struct thread *td) 546 { 547 struct linux_cdev *ldev; 548 struct linux_file *filp; 549 struct file *file; 550 int revents; 551 int error; 552 553 file = curthread->td_fpop; 554 ldev = dev->si_drv1; 555 if (ldev == NULL) 556 return (0); 557 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 558 return (error); 559 filp->f_flags = file->f_flag; 560 if (filp->f_op->poll) 561 revents = filp->f_op->poll(filp, NULL) & events; 562 else 563 revents = 0; 564 565 return (revents); 566 } 567 568 static int 569 linux_dev_mmap_single(struct cdev *dev, vm_ooffset_t *offset, 570 vm_size_t size, struct vm_object **object, int nprot) 571 { 572 struct linux_cdev *ldev; 573 struct linux_file *filp; 574 struct file *file; 575 struct vm_area_struct vma; 576 int error; 577 578 file = curthread->td_fpop; 579 ldev = dev->si_drv1; 580 if (ldev == NULL) 581 return (ENODEV); 582 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 583 return (error); 584 filp->f_flags = file->f_flag; 585 vma.vm_start = 0; 586 vma.vm_end = size; 587 vma.vm_pgoff = *offset / PAGE_SIZE; 588 vma.vm_pfn = 0; 589 vma.vm_page_prot = VM_MEMATTR_DEFAULT; 590 if (filp->f_op->mmap) { 591 error = -filp->f_op->mmap(filp, &vma); 592 if (error == 0) { 593 struct sglist *sg; 594 595 sg = sglist_alloc(1, M_WAITOK); 596 sglist_append_phys(sg, 597 (vm_paddr_t)vma.vm_pfn << PAGE_SHIFT, vma.vm_len); 598 *object = vm_pager_allocate(OBJT_SG, sg, vma.vm_len, 599 nprot, 0, curthread->td_ucred); 600 if (*object == NULL) { 601 sglist_free(sg); 602 return (EINVAL); 603 } 604 *offset = 0; 605 if (vma.vm_page_prot != VM_MEMATTR_DEFAULT) { 606 VM_OBJECT_WLOCK(*object); 607 vm_object_set_memattr(*object, 608 vma.vm_page_prot); 609 VM_OBJECT_WUNLOCK(*object); 610 } 611 } 612 } else 613 error = ENODEV; 614 615 return (error); 616 } 617 618 struct cdevsw linuxcdevsw = { 619 .d_version = D_VERSION, 620 .d_flags = D_TRACKCLOSE, 621 .d_open = linux_dev_open, 622 .d_close = linux_dev_close, 623 .d_read = linux_dev_read, 624 .d_write = linux_dev_write, 625 .d_ioctl = linux_dev_ioctl, 626 .d_mmap_single = linux_dev_mmap_single, 627 .d_poll = linux_dev_poll, 628 }; 629 630 static int 631 linux_file_read(struct file *file, struct uio *uio, struct ucred *active_cred, 632 int flags, struct thread *td) 633 { 634 struct linux_file *filp; 635 ssize_t bytes; 636 int error; 637 638 error = 0; 639 filp = (struct linux_file *)file->f_data; 640 filp->f_flags = file->f_flag; 641 if (uio->uio_iovcnt != 1) 642 panic("linux_file_read: uio %p iovcnt %d", 643 uio, uio->uio_iovcnt); 644 if (filp->f_op->read) { 645 bytes = filp->f_op->read(filp, uio->uio_iov->iov_base, 646 uio->uio_iov->iov_len, &uio->uio_offset); 647 if (bytes >= 0) { 648 uio->uio_iov->iov_base = 649 ((uint8_t *)uio->uio_iov->iov_base) + bytes; 650 uio->uio_iov->iov_len -= bytes; 651 uio->uio_resid -= bytes; 652 } else 653 error = -bytes; 654 } else 655 error = ENXIO; 656 657 return (error); 658 } 659 660 static int 661 linux_file_poll(struct file *file, int events, struct ucred *active_cred, 662 struct thread *td) 663 { 664 struct linux_file *filp; 665 int revents; 666 667 filp = (struct linux_file *)file->f_data; 668 filp->f_flags = file->f_flag; 669 if (filp->f_op->poll) 670 revents = filp->f_op->poll(filp, NULL) & events; 671 else 672 revents = 0; 673 674 return (0); 675 } 676 677 static int 678 linux_file_close(struct file *file, struct thread *td) 679 { 680 struct linux_file *filp; 681 int error; 682 683 filp = (struct linux_file *)file->f_data; 684 filp->f_flags = file->f_flag; 685 error = -filp->f_op->release(NULL, filp); 686 funsetown(&filp->f_sigio); 687 kfree(filp); 688 689 return (error); 690 } 691 692 static int 693 linux_file_ioctl(struct file *fp, u_long cmd, void *data, struct ucred *cred, 694 struct thread *td) 695 { 696 struct linux_file *filp; 697 int error; 698 699 filp = (struct linux_file *)fp->f_data; 700 filp->f_flags = fp->f_flag; 701 error = 0; 702 703 switch (cmd) { 704 case FIONBIO: 705 break; 706 case FIOASYNC: 707 if (filp->f_op->fasync == NULL) 708 break; 709 error = filp->f_op->fasync(0, filp, fp->f_flag & FASYNC); 710 break; 711 case FIOSETOWN: 712 error = fsetown(*(int *)data, &filp->f_sigio); 713 if (error == 0) 714 error = filp->f_op->fasync(0, filp, 715 fp->f_flag & FASYNC); 716 break; 717 case FIOGETOWN: 718 *(int *)data = fgetown(&filp->f_sigio); 719 break; 720 default: 721 error = ENOTTY; 722 break; 723 } 724 return (error); 725 } 726 727 static int 728 linux_file_stat(struct file *fp, struct stat *sb, struct ucred *active_cred, 729 struct thread *td) 730 { 731 732 return (EOPNOTSUPP); 733 } 734 735 static int 736 linux_file_fill_kinfo(struct file *fp, struct kinfo_file *kif, 737 struct filedesc *fdp) 738 { 739 740 return (0); 741 } 742 743 struct fileops linuxfileops = { 744 .fo_read = linux_file_read, 745 .fo_write = invfo_rdwr, 746 .fo_truncate = invfo_truncate, 747 .fo_kqfilter = invfo_kqfilter, 748 .fo_stat = linux_file_stat, 749 .fo_fill_kinfo = linux_file_fill_kinfo, 750 .fo_poll = linux_file_poll, 751 .fo_close = linux_file_close, 752 .fo_ioctl = linux_file_ioctl, 753 .fo_chmod = invfo_chmod, 754 .fo_chown = invfo_chown, 755 .fo_sendfile = invfo_sendfile, 756 }; 757 758 /* 759 * Hash of vmmap addresses. This is infrequently accessed and does not 760 * need to be particularly large. This is done because we must store the 761 * caller's idea of the map size to properly unmap. 762 */ 763 struct vmmap { 764 LIST_ENTRY(vmmap) vm_next; 765 void *vm_addr; 766 unsigned long vm_size; 767 }; 768 769 struct vmmaphd { 770 struct vmmap *lh_first; 771 }; 772 #define VMMAP_HASH_SIZE 64 773 #define VMMAP_HASH_MASK (VMMAP_HASH_SIZE - 1) 774 #define VM_HASH(addr) ((uintptr_t)(addr) >> PAGE_SHIFT) & VMMAP_HASH_MASK 775 static struct vmmaphd vmmaphead[VMMAP_HASH_SIZE]; 776 static struct mtx vmmaplock; 777 778 static void 779 vmmap_add(void *addr, unsigned long size) 780 { 781 struct vmmap *vmmap; 782 783 vmmap = kmalloc(sizeof(*vmmap), GFP_KERNEL); 784 mtx_lock(&vmmaplock); 785 vmmap->vm_size = size; 786 vmmap->vm_addr = addr; 787 LIST_INSERT_HEAD(&vmmaphead[VM_HASH(addr)], vmmap, vm_next); 788 mtx_unlock(&vmmaplock); 789 } 790 791 static struct vmmap * 792 vmmap_remove(void *addr) 793 { 794 struct vmmap *vmmap; 795 796 mtx_lock(&vmmaplock); 797 LIST_FOREACH(vmmap, &vmmaphead[VM_HASH(addr)], vm_next) 798 if (vmmap->vm_addr == addr) 799 break; 800 if (vmmap) 801 LIST_REMOVE(vmmap, vm_next); 802 mtx_unlock(&vmmaplock); 803 804 return (vmmap); 805 } 806 807 #if defined(__i386__) || defined(__amd64__) 808 void * 809 _ioremap_attr(vm_paddr_t phys_addr, unsigned long size, int attr) 810 { 811 void *addr; 812 813 addr = pmap_mapdev_attr(phys_addr, size, attr); 814 if (addr == NULL) 815 return (NULL); 816 vmmap_add(addr, size); 817 818 return (addr); 819 } 820 #endif 821 822 void 823 iounmap(void *addr) 824 { 825 struct vmmap *vmmap; 826 827 vmmap = vmmap_remove(addr); 828 if (vmmap == NULL) 829 return; 830 #if defined(__i386__) || defined(__amd64__) 831 pmap_unmapdev((vm_offset_t)addr, vmmap->vm_size); 832 #endif 833 kfree(vmmap); 834 } 835 836 837 void * 838 vmap(struct page **pages, unsigned int count, unsigned long flags, int prot) 839 { 840 vm_offset_t off; 841 size_t size; 842 843 size = count * PAGE_SIZE; 844 off = kva_alloc(size); 845 if (off == 0) 846 return (NULL); 847 vmmap_add((void *)off, size); 848 pmap_qenter(off, pages, count); 849 850 return ((void *)off); 851 } 852 853 void 854 vunmap(void *addr) 855 { 856 struct vmmap *vmmap; 857 858 vmmap = vmmap_remove(addr); 859 if (vmmap == NULL) 860 return; 861 pmap_qremove((vm_offset_t)addr, vmmap->vm_size / PAGE_SIZE); 862 kva_free((vm_offset_t)addr, vmmap->vm_size); 863 kfree(vmmap); 864 } 865 866 char * 867 kvasprintf(gfp_t gfp, const char *fmt, va_list ap) 868 { 869 unsigned int len; 870 char *p; 871 va_list aq; 872 873 va_copy(aq, ap); 874 len = vsnprintf(NULL, 0, fmt, aq); 875 va_end(aq); 876 877 p = kmalloc(len + 1, gfp); 878 if (p != NULL) 879 vsnprintf(p, len + 1, fmt, ap); 880 881 return (p); 882 } 883 884 char * 885 kasprintf(gfp_t gfp, const char *fmt, ...) 886 { 887 va_list ap; 888 char *p; 889 890 va_start(ap, fmt); 891 p = kvasprintf(gfp, fmt, ap); 892 va_end(ap); 893 894 return (p); 895 } 896 897 static void 898 linux_timer_callback_wrapper(void *context) 899 { 900 struct timer_list *timer; 901 902 timer = context; 903 timer->function(timer->data); 904 } 905 906 void 907 mod_timer(struct timer_list *timer, unsigned long expires) 908 { 909 910 timer->expires = expires; 911 callout_reset(&timer->timer_callout, 912 linux_timer_jiffies_until(expires), 913 &linux_timer_callback_wrapper, timer); 914 } 915 916 void 917 add_timer(struct timer_list *timer) 918 { 919 920 callout_reset(&timer->timer_callout, 921 linux_timer_jiffies_until(timer->expires), 922 &linux_timer_callback_wrapper, timer); 923 } 924 925 static void 926 linux_timer_init(void *arg) 927 { 928 929 /* 930 * Compute an internal HZ value which can divide 2**32 to 931 * avoid timer rounding problems when the tick value wraps 932 * around 2**32: 933 */ 934 linux_timer_hz_mask = 1; 935 while (linux_timer_hz_mask < (unsigned long)hz) 936 linux_timer_hz_mask *= 2; 937 linux_timer_hz_mask--; 938 } 939 SYSINIT(linux_timer, SI_SUB_DRIVERS, SI_ORDER_FIRST, linux_timer_init, NULL); 940 941 void 942 linux_complete_common(struct completion *c, int all) 943 { 944 int wakeup_swapper; 945 946 sleepq_lock(c); 947 c->done++; 948 if (all) 949 wakeup_swapper = sleepq_broadcast(c, SLEEPQ_SLEEP, 0, 0); 950 else 951 wakeup_swapper = sleepq_signal(c, SLEEPQ_SLEEP, 0, 0); 952 sleepq_release(c); 953 if (wakeup_swapper) 954 kick_proc0(); 955 } 956 957 /* 958 * Indefinite wait for done != 0 with or without signals. 959 */ 960 long 961 linux_wait_for_common(struct completion *c, int flags) 962 { 963 964 if (flags != 0) 965 flags = SLEEPQ_INTERRUPTIBLE | SLEEPQ_SLEEP; 966 else 967 flags = SLEEPQ_SLEEP; 968 for (;;) { 969 sleepq_lock(c); 970 if (c->done) 971 break; 972 sleepq_add(c, NULL, "completion", flags, 0); 973 if (flags & SLEEPQ_INTERRUPTIBLE) { 974 if (sleepq_wait_sig(c, 0) != 0) 975 return (-ERESTARTSYS); 976 } else 977 sleepq_wait(c, 0); 978 } 979 c->done--; 980 sleepq_release(c); 981 982 return (0); 983 } 984 985 /* 986 * Time limited wait for done != 0 with or without signals. 987 */ 988 long 989 linux_wait_for_timeout_common(struct completion *c, long timeout, int flags) 990 { 991 long end = jiffies + timeout; 992 993 if (flags != 0) 994 flags = SLEEPQ_INTERRUPTIBLE | SLEEPQ_SLEEP; 995 else 996 flags = SLEEPQ_SLEEP; 997 for (;;) { 998 int ret; 999 1000 sleepq_lock(c); 1001 if (c->done) 1002 break; 1003 sleepq_add(c, NULL, "completion", flags, 0); 1004 sleepq_set_timeout(c, linux_timer_jiffies_until(end)); 1005 if (flags & SLEEPQ_INTERRUPTIBLE) 1006 ret = sleepq_timedwait_sig(c, 0); 1007 else 1008 ret = sleepq_timedwait(c, 0); 1009 if (ret != 0) { 1010 /* check for timeout or signal */ 1011 if (ret == EWOULDBLOCK) 1012 return (0); 1013 else 1014 return (-ERESTARTSYS); 1015 } 1016 } 1017 c->done--; 1018 sleepq_release(c); 1019 1020 /* return how many jiffies are left */ 1021 return (linux_timer_jiffies_until(end)); 1022 } 1023 1024 int 1025 linux_try_wait_for_completion(struct completion *c) 1026 { 1027 int isdone; 1028 1029 isdone = 1; 1030 sleepq_lock(c); 1031 if (c->done) 1032 c->done--; 1033 else 1034 isdone = 0; 1035 sleepq_release(c); 1036 return (isdone); 1037 } 1038 1039 int 1040 linux_completion_done(struct completion *c) 1041 { 1042 int isdone; 1043 1044 isdone = 1; 1045 sleepq_lock(c); 1046 if (c->done == 0) 1047 isdone = 0; 1048 sleepq_release(c); 1049 return (isdone); 1050 } 1051 1052 void 1053 linux_delayed_work_fn(void *arg) 1054 { 1055 struct delayed_work *work; 1056 1057 work = arg; 1058 taskqueue_enqueue(work->work.taskqueue, &work->work.work_task); 1059 } 1060 1061 void 1062 linux_work_fn(void *context, int pending) 1063 { 1064 struct work_struct *work; 1065 1066 work = context; 1067 work->fn(work); 1068 } 1069 1070 void 1071 linux_flush_fn(void *context, int pending) 1072 { 1073 } 1074 1075 struct workqueue_struct * 1076 linux_create_workqueue_common(const char *name, int cpus) 1077 { 1078 struct workqueue_struct *wq; 1079 1080 wq = kmalloc(sizeof(*wq), M_WAITOK); 1081 wq->taskqueue = taskqueue_create(name, M_WAITOK, 1082 taskqueue_thread_enqueue, &wq->taskqueue); 1083 atomic_set(&wq->draining, 0); 1084 taskqueue_start_threads(&wq->taskqueue, cpus, PWAIT, "%s", name); 1085 1086 return (wq); 1087 } 1088 1089 void 1090 destroy_workqueue(struct workqueue_struct *wq) 1091 { 1092 taskqueue_free(wq->taskqueue); 1093 kfree(wq); 1094 } 1095 1096 static void 1097 linux_cdev_release(struct kobject *kobj) 1098 { 1099 struct linux_cdev *cdev; 1100 struct kobject *parent; 1101 1102 cdev = container_of(kobj, struct linux_cdev, kobj); 1103 parent = kobj->parent; 1104 if (cdev->cdev) 1105 destroy_dev(cdev->cdev); 1106 kfree(cdev); 1107 kobject_put(parent); 1108 } 1109 1110 static void 1111 linux_cdev_static_release(struct kobject *kobj) 1112 { 1113 struct linux_cdev *cdev; 1114 struct kobject *parent; 1115 1116 cdev = container_of(kobj, struct linux_cdev, kobj); 1117 parent = kobj->parent; 1118 if (cdev->cdev) 1119 destroy_dev(cdev->cdev); 1120 kobject_put(parent); 1121 } 1122 1123 const struct kobj_type linux_cdev_ktype = { 1124 .release = linux_cdev_release, 1125 }; 1126 1127 const struct kobj_type linux_cdev_static_ktype = { 1128 .release = linux_cdev_static_release, 1129 }; 1130 1131 static void 1132 linux_handle_ifnet_link_event(void *arg, struct ifnet *ifp, int linkstate) 1133 { 1134 struct notifier_block *nb; 1135 1136 nb = arg; 1137 if (linkstate == LINK_STATE_UP) 1138 nb->notifier_call(nb, NETDEV_UP, ifp); 1139 else 1140 nb->notifier_call(nb, NETDEV_DOWN, ifp); 1141 } 1142 1143 static void 1144 linux_handle_ifnet_arrival_event(void *arg, struct ifnet *ifp) 1145 { 1146 struct notifier_block *nb; 1147 1148 nb = arg; 1149 nb->notifier_call(nb, NETDEV_REGISTER, ifp); 1150 } 1151 1152 static void 1153 linux_handle_ifnet_departure_event(void *arg, struct ifnet *ifp) 1154 { 1155 struct notifier_block *nb; 1156 1157 nb = arg; 1158 nb->notifier_call(nb, NETDEV_UNREGISTER, ifp); 1159 } 1160 1161 static void 1162 linux_handle_iflladdr_event(void *arg, struct ifnet *ifp) 1163 { 1164 struct notifier_block *nb; 1165 1166 nb = arg; 1167 nb->notifier_call(nb, NETDEV_CHANGEADDR, ifp); 1168 } 1169 1170 static void 1171 linux_handle_ifaddr_event(void *arg, struct ifnet *ifp) 1172 { 1173 struct notifier_block *nb; 1174 1175 nb = arg; 1176 nb->notifier_call(nb, NETDEV_CHANGEIFADDR, ifp); 1177 } 1178 1179 int 1180 register_netdevice_notifier(struct notifier_block *nb) 1181 { 1182 1183 nb->tags[NETDEV_UP] = EVENTHANDLER_REGISTER( 1184 ifnet_link_event, linux_handle_ifnet_link_event, nb, 0); 1185 nb->tags[NETDEV_REGISTER] = EVENTHANDLER_REGISTER( 1186 ifnet_arrival_event, linux_handle_ifnet_arrival_event, nb, 0); 1187 nb->tags[NETDEV_UNREGISTER] = EVENTHANDLER_REGISTER( 1188 ifnet_departure_event, linux_handle_ifnet_departure_event, nb, 0); 1189 nb->tags[NETDEV_CHANGEADDR] = EVENTHANDLER_REGISTER( 1190 iflladdr_event, linux_handle_iflladdr_event, nb, 0); 1191 1192 return (0); 1193 } 1194 1195 int 1196 register_inetaddr_notifier(struct notifier_block *nb) 1197 { 1198 1199 nb->tags[NETDEV_CHANGEIFADDR] = EVENTHANDLER_REGISTER( 1200 ifaddr_event, linux_handle_ifaddr_event, nb, 0); 1201 return (0); 1202 } 1203 1204 int 1205 unregister_netdevice_notifier(struct notifier_block *nb) 1206 { 1207 1208 EVENTHANDLER_DEREGISTER(ifnet_link_event, 1209 nb->tags[NETDEV_UP]); 1210 EVENTHANDLER_DEREGISTER(ifnet_arrival_event, 1211 nb->tags[NETDEV_REGISTER]); 1212 EVENTHANDLER_DEREGISTER(ifnet_departure_event, 1213 nb->tags[NETDEV_UNREGISTER]); 1214 EVENTHANDLER_DEREGISTER(iflladdr_event, 1215 nb->tags[NETDEV_CHANGEADDR]); 1216 1217 return (0); 1218 } 1219 1220 int 1221 unregister_inetaddr_notifier(struct notifier_block *nb) 1222 { 1223 1224 EVENTHANDLER_DEREGISTER(ifaddr_event, 1225 nb->tags[NETDEV_CHANGEIFADDR]); 1226 1227 return (0); 1228 } 1229 1230 void 1231 linux_irq_handler(void *ent) 1232 { 1233 struct irq_ent *irqe; 1234 1235 irqe = ent; 1236 irqe->handler(irqe->irq, irqe->arg); 1237 } 1238 1239 static void 1240 linux_compat_init(void *arg) 1241 { 1242 struct sysctl_oid *rootoid; 1243 int i; 1244 1245 sx_init(&linux_global_rcu_lock, "LinuxGlobalRCU"); 1246 1247 rootoid = SYSCTL_ADD_ROOT_NODE(NULL, 1248 OID_AUTO, "sys", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "sys"); 1249 kobject_init(&linux_class_root, &linux_class_ktype); 1250 kobject_set_name(&linux_class_root, "class"); 1251 linux_class_root.oidp = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(rootoid), 1252 OID_AUTO, "class", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "class"); 1253 kobject_init(&linux_root_device.kobj, &linux_dev_ktype); 1254 kobject_set_name(&linux_root_device.kobj, "device"); 1255 linux_root_device.kobj.oidp = SYSCTL_ADD_NODE(NULL, 1256 SYSCTL_CHILDREN(rootoid), OID_AUTO, "device", CTLFLAG_RD, NULL, 1257 "device"); 1258 linux_root_device.bsddev = root_bus; 1259 linux_class_misc.name = "misc"; 1260 class_register(&linux_class_misc); 1261 INIT_LIST_HEAD(&pci_drivers); 1262 INIT_LIST_HEAD(&pci_devices); 1263 spin_lock_init(&pci_lock); 1264 mtx_init(&vmmaplock, "IO Map lock", NULL, MTX_DEF); 1265 for (i = 0; i < VMMAP_HASH_SIZE; i++) 1266 LIST_INIT(&vmmaphead[i]); 1267 } 1268 SYSINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_init, NULL); 1269 1270 static void 1271 linux_compat_uninit(void *arg) 1272 { 1273 linux_kobject_kfree_name(&linux_class_root); 1274 linux_kobject_kfree_name(&linux_root_device.kobj); 1275 linux_kobject_kfree_name(&linux_class_misc.kobj); 1276 1277 synchronize_rcu(); 1278 sx_destroy(&linux_global_rcu_lock); 1279 } 1280 SYSUNINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_uninit, NULL); 1281 1282 /* 1283 * NOTE: Linux frequently uses "unsigned long" for pointer to integer 1284 * conversion and vice versa, where in FreeBSD "uintptr_t" would be 1285 * used. Assert these types have the same size, else some parts of the 1286 * LinuxKPI may not work like expected: 1287 */ 1288 CTASSERT(sizeof(unsigned long) == sizeof(uintptr_t)); 1289