1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice unmodified, this list of conditions, and the following 13 * disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/malloc.h> 36 #include <sys/kernel.h> 37 #include <sys/sysctl.h> 38 #include <sys/proc.h> 39 #include <sys/sglist.h> 40 #include <sys/sleepqueue.h> 41 #include <sys/lock.h> 42 #include <sys/mutex.h> 43 #include <sys/bus.h> 44 #include <sys/fcntl.h> 45 #include <sys/file.h> 46 #include <sys/filio.h> 47 #include <sys/rwlock.h> 48 49 #include <vm/vm.h> 50 #include <vm/pmap.h> 51 52 #include <machine/stdarg.h> 53 54 #if defined(__i386__) || defined(__amd64__) 55 #include <machine/md_var.h> 56 #endif 57 58 #include <linux/kobject.h> 59 #include <linux/device.h> 60 #include <linux/slab.h> 61 #include <linux/module.h> 62 #include <linux/moduleparam.h> 63 #include <linux/cdev.h> 64 #include <linux/file.h> 65 #include <linux/sysfs.h> 66 #include <linux/mm.h> 67 #include <linux/io.h> 68 #include <linux/vmalloc.h> 69 #include <linux/netdevice.h> 70 #include <linux/timer.h> 71 #include <linux/interrupt.h> 72 #include <linux/uaccess.h> 73 #include <linux/kernel.h> 74 #include <linux/list.h> 75 #include <linux/compat.h> 76 #include <linux/poll.h> 77 78 #include <vm/vm_pager.h> 79 80 SYSCTL_NODE(_compat, OID_AUTO, linuxkpi, CTLFLAG_RW, 0, "LinuxKPI parameters"); 81 82 MALLOC_DEFINE(M_KMALLOC, "linux", "Linux kmalloc compat"); 83 84 #include <linux/rbtree.h> 85 /* Undo Linux compat changes. */ 86 #undef RB_ROOT 87 #undef file 88 #undef cdev 89 #define RB_ROOT(head) (head)->rbh_root 90 91 static struct vm_area_struct *linux_cdev_handle_find(void *handle); 92 93 struct kobject linux_class_root; 94 struct device linux_root_device; 95 struct class linux_class_misc; 96 struct list_head pci_drivers; 97 struct list_head pci_devices; 98 spinlock_t pci_lock; 99 100 unsigned long linux_timer_hz_mask; 101 102 int 103 panic_cmp(struct rb_node *one, struct rb_node *two) 104 { 105 panic("no cmp"); 106 } 107 108 RB_GENERATE(linux_root, rb_node, __entry, panic_cmp); 109 110 int 111 kobject_set_name_vargs(struct kobject *kobj, const char *fmt, va_list args) 112 { 113 va_list tmp_va; 114 int len; 115 char *old; 116 char *name; 117 char dummy; 118 119 old = kobj->name; 120 121 if (old && fmt == NULL) 122 return (0); 123 124 /* compute length of string */ 125 va_copy(tmp_va, args); 126 len = vsnprintf(&dummy, 0, fmt, tmp_va); 127 va_end(tmp_va); 128 129 /* account for zero termination */ 130 len++; 131 132 /* check for error */ 133 if (len < 1) 134 return (-EINVAL); 135 136 /* allocate memory for string */ 137 name = kzalloc(len, GFP_KERNEL); 138 if (name == NULL) 139 return (-ENOMEM); 140 vsnprintf(name, len, fmt, args); 141 kobj->name = name; 142 143 /* free old string */ 144 kfree(old); 145 146 /* filter new string */ 147 for (; *name != '\0'; name++) 148 if (*name == '/') 149 *name = '!'; 150 return (0); 151 } 152 153 int 154 kobject_set_name(struct kobject *kobj, const char *fmt, ...) 155 { 156 va_list args; 157 int error; 158 159 va_start(args, fmt); 160 error = kobject_set_name_vargs(kobj, fmt, args); 161 va_end(args); 162 163 return (error); 164 } 165 166 static int 167 kobject_add_complete(struct kobject *kobj, struct kobject *parent) 168 { 169 const struct kobj_type *t; 170 int error; 171 172 kobj->parent = parent; 173 error = sysfs_create_dir(kobj); 174 if (error == 0 && kobj->ktype && kobj->ktype->default_attrs) { 175 struct attribute **attr; 176 t = kobj->ktype; 177 178 for (attr = t->default_attrs; *attr != NULL; attr++) { 179 error = sysfs_create_file(kobj, *attr); 180 if (error) 181 break; 182 } 183 if (error) 184 sysfs_remove_dir(kobj); 185 186 } 187 return (error); 188 } 189 190 int 191 kobject_add(struct kobject *kobj, struct kobject *parent, const char *fmt, ...) 192 { 193 va_list args; 194 int error; 195 196 va_start(args, fmt); 197 error = kobject_set_name_vargs(kobj, fmt, args); 198 va_end(args); 199 if (error) 200 return (error); 201 202 return kobject_add_complete(kobj, parent); 203 } 204 205 void 206 linux_kobject_release(struct kref *kref) 207 { 208 struct kobject *kobj; 209 char *name; 210 211 kobj = container_of(kref, struct kobject, kref); 212 sysfs_remove_dir(kobj); 213 name = kobj->name; 214 if (kobj->ktype && kobj->ktype->release) 215 kobj->ktype->release(kobj); 216 kfree(name); 217 } 218 219 static void 220 linux_kobject_kfree(struct kobject *kobj) 221 { 222 kfree(kobj); 223 } 224 225 static void 226 linux_kobject_kfree_name(struct kobject *kobj) 227 { 228 if (kobj) { 229 kfree(kobj->name); 230 } 231 } 232 233 const struct kobj_type linux_kfree_type = { 234 .release = linux_kobject_kfree 235 }; 236 237 static void 238 linux_device_release(struct device *dev) 239 { 240 pr_debug("linux_device_release: %s\n", dev_name(dev)); 241 kfree(dev); 242 } 243 244 static ssize_t 245 linux_class_show(struct kobject *kobj, struct attribute *attr, char *buf) 246 { 247 struct class_attribute *dattr; 248 ssize_t error; 249 250 dattr = container_of(attr, struct class_attribute, attr); 251 error = -EIO; 252 if (dattr->show) 253 error = dattr->show(container_of(kobj, struct class, kobj), 254 dattr, buf); 255 return (error); 256 } 257 258 static ssize_t 259 linux_class_store(struct kobject *kobj, struct attribute *attr, const char *buf, 260 size_t count) 261 { 262 struct class_attribute *dattr; 263 ssize_t error; 264 265 dattr = container_of(attr, struct class_attribute, attr); 266 error = -EIO; 267 if (dattr->store) 268 error = dattr->store(container_of(kobj, struct class, kobj), 269 dattr, buf, count); 270 return (error); 271 } 272 273 static void 274 linux_class_release(struct kobject *kobj) 275 { 276 struct class *class; 277 278 class = container_of(kobj, struct class, kobj); 279 if (class->class_release) 280 class->class_release(class); 281 } 282 283 static const struct sysfs_ops linux_class_sysfs = { 284 .show = linux_class_show, 285 .store = linux_class_store, 286 }; 287 288 const struct kobj_type linux_class_ktype = { 289 .release = linux_class_release, 290 .sysfs_ops = &linux_class_sysfs 291 }; 292 293 static void 294 linux_dev_release(struct kobject *kobj) 295 { 296 struct device *dev; 297 298 dev = container_of(kobj, struct device, kobj); 299 /* This is the precedence defined by linux. */ 300 if (dev->release) 301 dev->release(dev); 302 else if (dev->class && dev->class->dev_release) 303 dev->class->dev_release(dev); 304 } 305 306 static ssize_t 307 linux_dev_show(struct kobject *kobj, struct attribute *attr, char *buf) 308 { 309 struct device_attribute *dattr; 310 ssize_t error; 311 312 dattr = container_of(attr, struct device_attribute, attr); 313 error = -EIO; 314 if (dattr->show) 315 error = dattr->show(container_of(kobj, struct device, kobj), 316 dattr, buf); 317 return (error); 318 } 319 320 static ssize_t 321 linux_dev_store(struct kobject *kobj, struct attribute *attr, const char *buf, 322 size_t count) 323 { 324 struct device_attribute *dattr; 325 ssize_t error; 326 327 dattr = container_of(attr, struct device_attribute, attr); 328 error = -EIO; 329 if (dattr->store) 330 error = dattr->store(container_of(kobj, struct device, kobj), 331 dattr, buf, count); 332 return (error); 333 } 334 335 static const struct sysfs_ops linux_dev_sysfs = { 336 .show = linux_dev_show, 337 .store = linux_dev_store, 338 }; 339 340 const struct kobj_type linux_dev_ktype = { 341 .release = linux_dev_release, 342 .sysfs_ops = &linux_dev_sysfs 343 }; 344 345 struct device * 346 device_create(struct class *class, struct device *parent, dev_t devt, 347 void *drvdata, const char *fmt, ...) 348 { 349 struct device *dev; 350 va_list args; 351 352 dev = kzalloc(sizeof(*dev), M_WAITOK); 353 dev->parent = parent; 354 dev->class = class; 355 dev->devt = devt; 356 dev->driver_data = drvdata; 357 dev->release = linux_device_release; 358 va_start(args, fmt); 359 kobject_set_name_vargs(&dev->kobj, fmt, args); 360 va_end(args); 361 device_register(dev); 362 363 return (dev); 364 } 365 366 int 367 kobject_init_and_add(struct kobject *kobj, const struct kobj_type *ktype, 368 struct kobject *parent, const char *fmt, ...) 369 { 370 va_list args; 371 int error; 372 373 kobject_init(kobj, ktype); 374 kobj->ktype = ktype; 375 kobj->parent = parent; 376 kobj->name = NULL; 377 378 va_start(args, fmt); 379 error = kobject_set_name_vargs(kobj, fmt, args); 380 va_end(args); 381 if (error) 382 return (error); 383 return kobject_add_complete(kobj, parent); 384 } 385 386 static void 387 linux_file_dtor(void *cdp) 388 { 389 struct linux_file *filp; 390 391 linux_set_current(curthread); 392 filp = cdp; 393 filp->f_op->release(filp->f_vnode, filp); 394 vdrop(filp->f_vnode); 395 kfree(filp); 396 } 397 398 static int 399 linux_cdev_pager_populate(vm_object_t vm_obj, vm_pindex_t pidx, int fault_type, 400 vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last) 401 { 402 struct vm_area_struct *vmap; 403 struct vm_fault vmf; 404 int err; 405 406 linux_set_current(curthread); 407 408 /* get VM area structure */ 409 vmap = linux_cdev_handle_find(vm_obj->handle); 410 MPASS(vmap != NULL); 411 MPASS(vmap->vm_private_data == vm_obj->handle); 412 413 /* fill out VM fault structure */ 414 vmf.virtual_address = (void *)((uintptr_t)pidx << PAGE_SHIFT); 415 vmf.flags = (fault_type & VM_PROT_WRITE) ? FAULT_FLAG_WRITE : 0; 416 vmf.pgoff = 0; 417 vmf.page = NULL; 418 419 VM_OBJECT_WUNLOCK(vm_obj); 420 421 down_write(&vmap->vm_mm->mmap_sem); 422 if (unlikely(vmap->vm_ops == NULL)) { 423 err = VM_FAULT_SIGBUS; 424 } else { 425 vmap->vm_pfn_count = 0; 426 vmap->vm_pfn_pcount = &vmap->vm_pfn_count; 427 vmap->vm_obj = vm_obj; 428 429 err = vmap->vm_ops->fault(vmap, &vmf); 430 431 while (vmap->vm_pfn_count == 0 && err == VM_FAULT_NOPAGE) { 432 kern_yield(0); 433 err = vmap->vm_ops->fault(vmap, &vmf); 434 } 435 } 436 437 /* translate return code */ 438 switch (err) { 439 case VM_FAULT_OOM: 440 err = VM_PAGER_AGAIN; 441 break; 442 case VM_FAULT_SIGBUS: 443 err = VM_PAGER_BAD; 444 break; 445 case VM_FAULT_NOPAGE: 446 /* 447 * By contract the fault handler will return having 448 * busied all the pages itself. If pidx is already 449 * found in the object, it will simply xbusy the first 450 * page and return with vm_pfn_count set to 1. 451 */ 452 *first = vmap->vm_pfn_first; 453 *last = *first + vmap->vm_pfn_count - 1; 454 err = VM_PAGER_OK; 455 break; 456 default: 457 err = VM_PAGER_ERROR; 458 break; 459 } 460 up_write(&vmap->vm_mm->mmap_sem); 461 VM_OBJECT_WLOCK(vm_obj); 462 return (err); 463 } 464 465 static struct rwlock linux_vma_lock; 466 static TAILQ_HEAD(, vm_area_struct) linux_vma_head = 467 TAILQ_HEAD_INITIALIZER(linux_vma_head); 468 469 static struct vm_area_struct * 470 linux_cdev_handle_insert(void *handle, struct vm_area_struct *vmap) 471 { 472 struct vm_area_struct *ptr; 473 474 rw_wlock(&linux_vma_lock); 475 TAILQ_FOREACH(ptr, &linux_vma_head, vm_entry) { 476 if (ptr->vm_private_data == handle) { 477 rw_wunlock(&linux_vma_lock); 478 kfree(vmap); 479 return (NULL); 480 } 481 } 482 TAILQ_INSERT_TAIL(&linux_vma_head, vmap, vm_entry); 483 rw_wunlock(&linux_vma_lock); 484 return (vmap); 485 } 486 487 static void 488 linux_cdev_handle_remove(struct vm_area_struct *vmap) 489 { 490 if (vmap == NULL) 491 return; 492 493 rw_wlock(&linux_vma_lock); 494 TAILQ_REMOVE(&linux_vma_head, vmap, vm_entry); 495 rw_wunlock(&linux_vma_lock); 496 kfree(vmap); 497 } 498 499 static struct vm_area_struct * 500 linux_cdev_handle_find(void *handle) 501 { 502 struct vm_area_struct *vmap; 503 504 rw_rlock(&linux_vma_lock); 505 TAILQ_FOREACH(vmap, &linux_vma_head, vm_entry) { 506 if (vmap->vm_private_data == handle) 507 break; 508 } 509 rw_runlock(&linux_vma_lock); 510 return (vmap); 511 } 512 513 static int 514 linux_cdev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 515 vm_ooffset_t foff, struct ucred *cred, u_short *color) 516 { 517 const struct vm_operations_struct *vm_ops; 518 struct vm_area_struct *vmap; 519 520 vmap = linux_cdev_handle_find(handle); 521 MPASS(vmap != NULL); 522 523 *color = 0; 524 525 down_write(&vmap->vm_mm->mmap_sem); 526 vm_ops = vmap->vm_ops; 527 if (likely(vm_ops != NULL)) 528 vm_ops->open(vmap); 529 up_write(&vmap->vm_mm->mmap_sem); 530 531 return (0); 532 } 533 534 static void 535 linux_cdev_pager_dtor(void *handle) 536 { 537 const struct vm_operations_struct *vm_ops; 538 struct vm_area_struct *vmap; 539 540 vmap = linux_cdev_handle_find(handle); 541 MPASS(vmap != NULL); 542 543 down_write(&vmap->vm_mm->mmap_sem); 544 vm_ops = vmap->vm_ops; 545 if (likely(vm_ops != NULL)) 546 vm_ops->close(vmap); 547 up_write(&vmap->vm_mm->mmap_sem); 548 549 linux_cdev_handle_remove(vmap); 550 } 551 552 static struct cdev_pager_ops linux_cdev_pager_ops = { 553 .cdev_pg_populate = linux_cdev_pager_populate, 554 .cdev_pg_ctor = linux_cdev_pager_ctor, 555 .cdev_pg_dtor = linux_cdev_pager_dtor 556 }; 557 558 static int 559 linux_dev_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 560 { 561 struct linux_cdev *ldev; 562 struct linux_file *filp; 563 struct file *file; 564 int error; 565 566 file = td->td_fpop; 567 ldev = dev->si_drv1; 568 if (ldev == NULL) 569 return (ENODEV); 570 filp = kzalloc(sizeof(*filp), GFP_KERNEL); 571 filp->f_dentry = &filp->f_dentry_store; 572 filp->f_op = ldev->ops; 573 filp->f_flags = file->f_flag; 574 vhold(file->f_vnode); 575 filp->f_vnode = file->f_vnode; 576 linux_set_current(td); 577 if (filp->f_op->open) { 578 error = -filp->f_op->open(file->f_vnode, filp); 579 if (error) { 580 kfree(filp); 581 goto done; 582 } 583 } 584 error = devfs_set_cdevpriv(filp, linux_file_dtor); 585 if (error) { 586 filp->f_op->release(file->f_vnode, filp); 587 kfree(filp); 588 } 589 done: 590 return (error); 591 } 592 593 static int 594 linux_dev_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 595 { 596 struct linux_file *filp; 597 struct file *file; 598 int error; 599 600 file = td->td_fpop; 601 if (dev->si_drv1 == NULL) 602 return (0); 603 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 604 return (error); 605 filp->f_flags = file->f_flag; 606 devfs_clear_cdevpriv(); 607 608 609 return (0); 610 } 611 612 #define LINUX_IOCTL_MIN_PTR 0x10000UL 613 #define LINUX_IOCTL_MAX_PTR (LINUX_IOCTL_MIN_PTR + IOCPARM_MAX) 614 615 static inline int 616 linux_remap_address(void **uaddr, size_t len) 617 { 618 uintptr_t uaddr_val = (uintptr_t)(*uaddr); 619 620 if (unlikely(uaddr_val >= LINUX_IOCTL_MIN_PTR && 621 uaddr_val < LINUX_IOCTL_MAX_PTR)) { 622 struct task_struct *pts = current; 623 if (pts == NULL) { 624 *uaddr = NULL; 625 return (1); 626 } 627 628 /* compute data offset */ 629 uaddr_val -= LINUX_IOCTL_MIN_PTR; 630 631 /* check that length is within bounds */ 632 if ((len > IOCPARM_MAX) || 633 (uaddr_val + len) > pts->bsd_ioctl_len) { 634 *uaddr = NULL; 635 return (1); 636 } 637 638 /* re-add kernel buffer address */ 639 uaddr_val += (uintptr_t)pts->bsd_ioctl_data; 640 641 /* update address location */ 642 *uaddr = (void *)uaddr_val; 643 return (1); 644 } 645 return (0); 646 } 647 648 int 649 linux_copyin(const void *uaddr, void *kaddr, size_t len) 650 { 651 if (linux_remap_address(__DECONST(void **, &uaddr), len)) { 652 if (uaddr == NULL) 653 return (-EFAULT); 654 memcpy(kaddr, uaddr, len); 655 return (0); 656 } 657 return (-copyin(uaddr, kaddr, len)); 658 } 659 660 int 661 linux_copyout(const void *kaddr, void *uaddr, size_t len) 662 { 663 if (linux_remap_address(&uaddr, len)) { 664 if (uaddr == NULL) 665 return (-EFAULT); 666 memcpy(uaddr, kaddr, len); 667 return (0); 668 } 669 return (-copyout(kaddr, uaddr, len)); 670 } 671 672 size_t 673 linux_clear_user(void *_uaddr, size_t _len) 674 { 675 uint8_t *uaddr = _uaddr; 676 size_t len = _len; 677 678 /* make sure uaddr is aligned before going into the fast loop */ 679 while (((uintptr_t)uaddr & 7) != 0 && len > 7) { 680 if (subyte(uaddr, 0)) 681 return (_len); 682 uaddr++; 683 len--; 684 } 685 686 /* zero 8 bytes at a time */ 687 while (len > 7) { 688 #ifdef __LP64__ 689 if (suword64(uaddr, 0)) 690 return (_len); 691 #else 692 if (suword32(uaddr, 0)) 693 return (_len); 694 if (suword32(uaddr + 4, 0)) 695 return (_len); 696 #endif 697 uaddr += 8; 698 len -= 8; 699 } 700 701 /* zero fill end, if any */ 702 while (len > 0) { 703 if (subyte(uaddr, 0)) 704 return (_len); 705 uaddr++; 706 len--; 707 } 708 return (0); 709 } 710 711 int 712 linux_access_ok(int rw, const void *uaddr, size_t len) 713 { 714 uintptr_t saddr; 715 uintptr_t eaddr; 716 717 /* get start and end address */ 718 saddr = (uintptr_t)uaddr; 719 eaddr = (uintptr_t)uaddr + len; 720 721 /* verify addresses are valid for userspace */ 722 return ((saddr == eaddr) || 723 (eaddr > saddr && eaddr <= VM_MAXUSER_ADDRESS)); 724 } 725 726 static int 727 linux_dev_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, 728 struct thread *td) 729 { 730 struct linux_file *filp; 731 struct file *file; 732 unsigned size; 733 int error; 734 735 file = td->td_fpop; 736 if (dev->si_drv1 == NULL) 737 return (ENXIO); 738 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 739 return (error); 740 filp->f_flags = file->f_flag; 741 742 linux_set_current(td); 743 size = IOCPARM_LEN(cmd); 744 /* refer to logic in sys_ioctl() */ 745 if (size > 0) { 746 /* 747 * Setup hint for linux_copyin() and linux_copyout(). 748 * 749 * Background: Linux code expects a user-space address 750 * while FreeBSD supplies a kernel-space address. 751 */ 752 current->bsd_ioctl_data = data; 753 current->bsd_ioctl_len = size; 754 data = (void *)LINUX_IOCTL_MIN_PTR; 755 } else { 756 /* fetch user-space pointer */ 757 data = *(void **)data; 758 } 759 if (filp->f_op->unlocked_ioctl) 760 error = -filp->f_op->unlocked_ioctl(filp, cmd, (u_long)data); 761 else 762 error = ENOTTY; 763 if (size > 0) { 764 current->bsd_ioctl_data = NULL; 765 current->bsd_ioctl_len = 0; 766 } 767 768 return (error); 769 } 770 771 static int 772 linux_dev_read(struct cdev *dev, struct uio *uio, int ioflag) 773 { 774 struct linux_file *filp; 775 struct thread *td; 776 struct file *file; 777 ssize_t bytes; 778 int error; 779 780 td = curthread; 781 file = td->td_fpop; 782 if (dev->si_drv1 == NULL) 783 return (ENXIO); 784 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 785 return (error); 786 filp->f_flags = file->f_flag; 787 /* XXX no support for I/O vectors currently */ 788 if (uio->uio_iovcnt != 1) 789 return (EOPNOTSUPP); 790 linux_set_current(td); 791 if (filp->f_op->read) { 792 bytes = filp->f_op->read(filp, uio->uio_iov->iov_base, 793 uio->uio_iov->iov_len, &uio->uio_offset); 794 if (bytes >= 0) { 795 uio->uio_iov->iov_base = 796 ((uint8_t *)uio->uio_iov->iov_base) + bytes; 797 uio->uio_iov->iov_len -= bytes; 798 uio->uio_resid -= bytes; 799 } else 800 error = -bytes; 801 } else 802 error = ENXIO; 803 804 return (error); 805 } 806 807 static int 808 linux_dev_write(struct cdev *dev, struct uio *uio, int ioflag) 809 { 810 struct linux_file *filp; 811 struct thread *td; 812 struct file *file; 813 ssize_t bytes; 814 int error; 815 816 td = curthread; 817 file = td->td_fpop; 818 if (dev->si_drv1 == NULL) 819 return (ENXIO); 820 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 821 return (error); 822 filp->f_flags = file->f_flag; 823 /* XXX no support for I/O vectors currently */ 824 if (uio->uio_iovcnt != 1) 825 return (EOPNOTSUPP); 826 linux_set_current(td); 827 if (filp->f_op->write) { 828 bytes = filp->f_op->write(filp, uio->uio_iov->iov_base, 829 uio->uio_iov->iov_len, &uio->uio_offset); 830 if (bytes >= 0) { 831 uio->uio_iov->iov_base = 832 ((uint8_t *)uio->uio_iov->iov_base) + bytes; 833 uio->uio_iov->iov_len -= bytes; 834 uio->uio_resid -= bytes; 835 } else 836 error = -bytes; 837 } else 838 error = ENXIO; 839 840 return (error); 841 } 842 843 static int 844 linux_dev_poll(struct cdev *dev, int events, struct thread *td) 845 { 846 struct linux_file *filp; 847 struct file *file; 848 int revents; 849 850 if (dev->si_drv1 == NULL) 851 goto error; 852 if (devfs_get_cdevpriv((void **)&filp) != 0) 853 goto error; 854 855 file = td->td_fpop; 856 filp->f_flags = file->f_flag; 857 linux_set_current(td); 858 if (filp->f_op->poll) 859 revents = filp->f_op->poll(filp, NULL) & events; 860 else 861 revents = 0; 862 863 return (revents); 864 error: 865 return (events & (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM)); 866 } 867 868 static int 869 linux_dev_mmap_single(struct cdev *dev, vm_ooffset_t *offset, 870 vm_size_t size, struct vm_object **object, int nprot) 871 { 872 struct vm_area_struct *vmap; 873 struct linux_file *filp; 874 struct thread *td; 875 struct file *file; 876 vm_memattr_t attr; 877 int error; 878 879 td = curthread; 880 file = td->td_fpop; 881 if (dev->si_drv1 == NULL) 882 return (ENODEV); 883 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 884 return (error); 885 filp->f_flags = file->f_flag; 886 887 if (filp->f_op->mmap == NULL) 888 return (ENODEV); 889 890 linux_set_current(td); 891 892 vmap = kzalloc(sizeof(*vmap), GFP_KERNEL); 893 vmap->vm_start = 0; 894 vmap->vm_end = size; 895 vmap->vm_pgoff = *offset / PAGE_SIZE; 896 vmap->vm_pfn = 0; 897 vmap->vm_flags = vmap->vm_page_prot = nprot; 898 vmap->vm_ops = NULL; 899 vmap->vm_file = filp; 900 vmap->vm_mm = current->mm; 901 902 if (unlikely(down_write_killable(&vmap->vm_mm->mmap_sem))) { 903 error = EINTR; 904 } else { 905 error = -filp->f_op->mmap(filp, vmap); 906 up_write(&vmap->vm_mm->mmap_sem); 907 } 908 909 if (error != 0) { 910 kfree(vmap); 911 return (error); 912 } 913 914 attr = pgprot2cachemode(vmap->vm_page_prot); 915 916 if (vmap->vm_ops != NULL) { 917 void *vm_private_data; 918 919 if (vmap->vm_ops->fault == NULL || 920 vmap->vm_ops->open == NULL || 921 vmap->vm_ops->close == NULL || 922 vmap->vm_private_data == NULL) { 923 kfree(vmap); 924 return (EINVAL); 925 } 926 927 vm_private_data = vmap->vm_private_data; 928 929 vmap = linux_cdev_handle_insert(vm_private_data, vmap); 930 931 *object = cdev_pager_allocate(vm_private_data, OBJT_MGTDEVICE, 932 &linux_cdev_pager_ops, size, nprot, *offset, curthread->td_ucred); 933 934 if (*object == NULL) { 935 linux_cdev_handle_remove(vmap); 936 return (EINVAL); 937 } 938 } else { 939 struct sglist *sg; 940 941 sg = sglist_alloc(1, M_WAITOK); 942 sglist_append_phys(sg, (vm_paddr_t)vmap->vm_pfn << PAGE_SHIFT, vmap->vm_len); 943 944 *object = vm_pager_allocate(OBJT_SG, sg, vmap->vm_len, 945 nprot, 0, curthread->td_ucred); 946 947 kfree(vmap); 948 949 if (*object == NULL) { 950 sglist_free(sg); 951 return (EINVAL); 952 } 953 } 954 955 if (attr != VM_MEMATTR_DEFAULT) { 956 VM_OBJECT_WLOCK(*object); 957 vm_object_set_memattr(*object, attr); 958 VM_OBJECT_WUNLOCK(*object); 959 } 960 *offset = 0; 961 return (0); 962 } 963 964 struct cdevsw linuxcdevsw = { 965 .d_version = D_VERSION, 966 .d_flags = D_TRACKCLOSE, 967 .d_open = linux_dev_open, 968 .d_close = linux_dev_close, 969 .d_read = linux_dev_read, 970 .d_write = linux_dev_write, 971 .d_ioctl = linux_dev_ioctl, 972 .d_mmap_single = linux_dev_mmap_single, 973 .d_poll = linux_dev_poll, 974 }; 975 976 static int 977 linux_file_read(struct file *file, struct uio *uio, struct ucred *active_cred, 978 int flags, struct thread *td) 979 { 980 struct linux_file *filp; 981 ssize_t bytes; 982 int error; 983 984 error = 0; 985 filp = (struct linux_file *)file->f_data; 986 filp->f_flags = file->f_flag; 987 /* XXX no support for I/O vectors currently */ 988 if (uio->uio_iovcnt != 1) 989 return (EOPNOTSUPP); 990 linux_set_current(td); 991 if (filp->f_op->read) { 992 bytes = filp->f_op->read(filp, uio->uio_iov->iov_base, 993 uio->uio_iov->iov_len, &uio->uio_offset); 994 if (bytes >= 0) { 995 uio->uio_iov->iov_base = 996 ((uint8_t *)uio->uio_iov->iov_base) + bytes; 997 uio->uio_iov->iov_len -= bytes; 998 uio->uio_resid -= bytes; 999 } else 1000 error = -bytes; 1001 } else 1002 error = ENXIO; 1003 1004 return (error); 1005 } 1006 1007 static int 1008 linux_file_poll(struct file *file, int events, struct ucred *active_cred, 1009 struct thread *td) 1010 { 1011 struct linux_file *filp; 1012 int revents; 1013 1014 filp = (struct linux_file *)file->f_data; 1015 filp->f_flags = file->f_flag; 1016 linux_set_current(td); 1017 if (filp->f_op->poll) 1018 revents = filp->f_op->poll(filp, NULL) & events; 1019 else 1020 revents = 0; 1021 1022 return (revents); 1023 } 1024 1025 static int 1026 linux_file_close(struct file *file, struct thread *td) 1027 { 1028 struct linux_file *filp; 1029 int error; 1030 1031 filp = (struct linux_file *)file->f_data; 1032 filp->f_flags = file->f_flag; 1033 linux_set_current(td); 1034 error = -filp->f_op->release(NULL, filp); 1035 funsetown(&filp->f_sigio); 1036 kfree(filp); 1037 1038 return (error); 1039 } 1040 1041 static int 1042 linux_file_ioctl(struct file *fp, u_long cmd, void *data, struct ucred *cred, 1043 struct thread *td) 1044 { 1045 struct linux_file *filp; 1046 int error; 1047 1048 filp = (struct linux_file *)fp->f_data; 1049 filp->f_flags = fp->f_flag; 1050 error = 0; 1051 1052 linux_set_current(td); 1053 switch (cmd) { 1054 case FIONBIO: 1055 break; 1056 case FIOASYNC: 1057 if (filp->f_op->fasync == NULL) 1058 break; 1059 error = filp->f_op->fasync(0, filp, fp->f_flag & FASYNC); 1060 break; 1061 case FIOSETOWN: 1062 error = fsetown(*(int *)data, &filp->f_sigio); 1063 if (error == 0) 1064 error = filp->f_op->fasync(0, filp, 1065 fp->f_flag & FASYNC); 1066 break; 1067 case FIOGETOWN: 1068 *(int *)data = fgetown(&filp->f_sigio); 1069 break; 1070 default: 1071 error = ENOTTY; 1072 break; 1073 } 1074 return (error); 1075 } 1076 1077 static int 1078 linux_file_stat(struct file *fp, struct stat *sb, struct ucred *active_cred, 1079 struct thread *td) 1080 { 1081 1082 return (EOPNOTSUPP); 1083 } 1084 1085 static int 1086 linux_file_fill_kinfo(struct file *fp, struct kinfo_file *kif, 1087 struct filedesc *fdp) 1088 { 1089 1090 return (0); 1091 } 1092 1093 struct fileops linuxfileops = { 1094 .fo_read = linux_file_read, 1095 .fo_write = invfo_rdwr, 1096 .fo_truncate = invfo_truncate, 1097 .fo_kqfilter = invfo_kqfilter, 1098 .fo_stat = linux_file_stat, 1099 .fo_fill_kinfo = linux_file_fill_kinfo, 1100 .fo_poll = linux_file_poll, 1101 .fo_close = linux_file_close, 1102 .fo_ioctl = linux_file_ioctl, 1103 .fo_chmod = invfo_chmod, 1104 .fo_chown = invfo_chown, 1105 .fo_sendfile = invfo_sendfile, 1106 }; 1107 1108 /* 1109 * Hash of vmmap addresses. This is infrequently accessed and does not 1110 * need to be particularly large. This is done because we must store the 1111 * caller's idea of the map size to properly unmap. 1112 */ 1113 struct vmmap { 1114 LIST_ENTRY(vmmap) vm_next; 1115 void *vm_addr; 1116 unsigned long vm_size; 1117 }; 1118 1119 struct vmmaphd { 1120 struct vmmap *lh_first; 1121 }; 1122 #define VMMAP_HASH_SIZE 64 1123 #define VMMAP_HASH_MASK (VMMAP_HASH_SIZE - 1) 1124 #define VM_HASH(addr) ((uintptr_t)(addr) >> PAGE_SHIFT) & VMMAP_HASH_MASK 1125 static struct vmmaphd vmmaphead[VMMAP_HASH_SIZE]; 1126 static struct mtx vmmaplock; 1127 1128 static void 1129 vmmap_add(void *addr, unsigned long size) 1130 { 1131 struct vmmap *vmmap; 1132 1133 vmmap = kmalloc(sizeof(*vmmap), GFP_KERNEL); 1134 mtx_lock(&vmmaplock); 1135 vmmap->vm_size = size; 1136 vmmap->vm_addr = addr; 1137 LIST_INSERT_HEAD(&vmmaphead[VM_HASH(addr)], vmmap, vm_next); 1138 mtx_unlock(&vmmaplock); 1139 } 1140 1141 static struct vmmap * 1142 vmmap_remove(void *addr) 1143 { 1144 struct vmmap *vmmap; 1145 1146 mtx_lock(&vmmaplock); 1147 LIST_FOREACH(vmmap, &vmmaphead[VM_HASH(addr)], vm_next) 1148 if (vmmap->vm_addr == addr) 1149 break; 1150 if (vmmap) 1151 LIST_REMOVE(vmmap, vm_next); 1152 mtx_unlock(&vmmaplock); 1153 1154 return (vmmap); 1155 } 1156 1157 #if defined(__i386__) || defined(__amd64__) 1158 void * 1159 _ioremap_attr(vm_paddr_t phys_addr, unsigned long size, int attr) 1160 { 1161 void *addr; 1162 1163 addr = pmap_mapdev_attr(phys_addr, size, attr); 1164 if (addr == NULL) 1165 return (NULL); 1166 vmmap_add(addr, size); 1167 1168 return (addr); 1169 } 1170 #endif 1171 1172 void 1173 iounmap(void *addr) 1174 { 1175 struct vmmap *vmmap; 1176 1177 vmmap = vmmap_remove(addr); 1178 if (vmmap == NULL) 1179 return; 1180 #if defined(__i386__) || defined(__amd64__) 1181 pmap_unmapdev((vm_offset_t)addr, vmmap->vm_size); 1182 #endif 1183 kfree(vmmap); 1184 } 1185 1186 1187 void * 1188 vmap(struct page **pages, unsigned int count, unsigned long flags, int prot) 1189 { 1190 vm_offset_t off; 1191 size_t size; 1192 1193 size = count * PAGE_SIZE; 1194 off = kva_alloc(size); 1195 if (off == 0) 1196 return (NULL); 1197 vmmap_add((void *)off, size); 1198 pmap_qenter(off, pages, count); 1199 1200 return ((void *)off); 1201 } 1202 1203 void 1204 vunmap(void *addr) 1205 { 1206 struct vmmap *vmmap; 1207 1208 vmmap = vmmap_remove(addr); 1209 if (vmmap == NULL) 1210 return; 1211 pmap_qremove((vm_offset_t)addr, vmmap->vm_size / PAGE_SIZE); 1212 kva_free((vm_offset_t)addr, vmmap->vm_size); 1213 kfree(vmmap); 1214 } 1215 1216 char * 1217 kvasprintf(gfp_t gfp, const char *fmt, va_list ap) 1218 { 1219 unsigned int len; 1220 char *p; 1221 va_list aq; 1222 1223 va_copy(aq, ap); 1224 len = vsnprintf(NULL, 0, fmt, aq); 1225 va_end(aq); 1226 1227 p = kmalloc(len + 1, gfp); 1228 if (p != NULL) 1229 vsnprintf(p, len + 1, fmt, ap); 1230 1231 return (p); 1232 } 1233 1234 char * 1235 kasprintf(gfp_t gfp, const char *fmt, ...) 1236 { 1237 va_list ap; 1238 char *p; 1239 1240 va_start(ap, fmt); 1241 p = kvasprintf(gfp, fmt, ap); 1242 va_end(ap); 1243 1244 return (p); 1245 } 1246 1247 static void 1248 linux_timer_callback_wrapper(void *context) 1249 { 1250 struct timer_list *timer; 1251 1252 linux_set_current(curthread); 1253 1254 timer = context; 1255 timer->function(timer->data); 1256 } 1257 1258 void 1259 mod_timer(struct timer_list *timer, unsigned long expires) 1260 { 1261 1262 timer->expires = expires; 1263 callout_reset(&timer->timer_callout, 1264 linux_timer_jiffies_until(expires), 1265 &linux_timer_callback_wrapper, timer); 1266 } 1267 1268 void 1269 add_timer(struct timer_list *timer) 1270 { 1271 1272 callout_reset(&timer->timer_callout, 1273 linux_timer_jiffies_until(timer->expires), 1274 &linux_timer_callback_wrapper, timer); 1275 } 1276 1277 void 1278 add_timer_on(struct timer_list *timer, int cpu) 1279 { 1280 1281 callout_reset_on(&timer->timer_callout, 1282 linux_timer_jiffies_until(timer->expires), 1283 &linux_timer_callback_wrapper, timer, cpu); 1284 } 1285 1286 static void 1287 linux_timer_init(void *arg) 1288 { 1289 1290 /* 1291 * Compute an internal HZ value which can divide 2**32 to 1292 * avoid timer rounding problems when the tick value wraps 1293 * around 2**32: 1294 */ 1295 linux_timer_hz_mask = 1; 1296 while (linux_timer_hz_mask < (unsigned long)hz) 1297 linux_timer_hz_mask *= 2; 1298 linux_timer_hz_mask--; 1299 } 1300 SYSINIT(linux_timer, SI_SUB_DRIVERS, SI_ORDER_FIRST, linux_timer_init, NULL); 1301 1302 void 1303 linux_complete_common(struct completion *c, int all) 1304 { 1305 int wakeup_swapper; 1306 1307 sleepq_lock(c); 1308 c->done++; 1309 if (all) 1310 wakeup_swapper = sleepq_broadcast(c, SLEEPQ_SLEEP, 0, 0); 1311 else 1312 wakeup_swapper = sleepq_signal(c, SLEEPQ_SLEEP, 0, 0); 1313 sleepq_release(c); 1314 if (wakeup_swapper) 1315 kick_proc0(); 1316 } 1317 1318 /* 1319 * Indefinite wait for done != 0 with or without signals. 1320 */ 1321 long 1322 linux_wait_for_common(struct completion *c, int flags) 1323 { 1324 long error; 1325 1326 if (SCHEDULER_STOPPED()) 1327 return (0); 1328 1329 DROP_GIANT(); 1330 1331 if (flags != 0) 1332 flags = SLEEPQ_INTERRUPTIBLE | SLEEPQ_SLEEP; 1333 else 1334 flags = SLEEPQ_SLEEP; 1335 error = 0; 1336 for (;;) { 1337 sleepq_lock(c); 1338 if (c->done) 1339 break; 1340 sleepq_add(c, NULL, "completion", flags, 0); 1341 if (flags & SLEEPQ_INTERRUPTIBLE) { 1342 if (sleepq_wait_sig(c, 0) != 0) { 1343 error = -ERESTARTSYS; 1344 goto intr; 1345 } 1346 } else 1347 sleepq_wait(c, 0); 1348 } 1349 c->done--; 1350 sleepq_release(c); 1351 1352 intr: 1353 PICKUP_GIANT(); 1354 1355 return (error); 1356 } 1357 1358 /* 1359 * Time limited wait for done != 0 with or without signals. 1360 */ 1361 long 1362 linux_wait_for_timeout_common(struct completion *c, long timeout, int flags) 1363 { 1364 long end = jiffies + timeout, error; 1365 int ret; 1366 1367 if (SCHEDULER_STOPPED()) 1368 return (0); 1369 1370 DROP_GIANT(); 1371 1372 if (flags != 0) 1373 flags = SLEEPQ_INTERRUPTIBLE | SLEEPQ_SLEEP; 1374 else 1375 flags = SLEEPQ_SLEEP; 1376 1377 error = 0; 1378 ret = 0; 1379 for (;;) { 1380 sleepq_lock(c); 1381 if (c->done) 1382 break; 1383 sleepq_add(c, NULL, "completion", flags, 0); 1384 sleepq_set_timeout(c, linux_timer_jiffies_until(end)); 1385 if (flags & SLEEPQ_INTERRUPTIBLE) 1386 ret = sleepq_timedwait_sig(c, 0); 1387 else 1388 ret = sleepq_timedwait(c, 0); 1389 if (ret != 0) { 1390 /* check for timeout or signal */ 1391 if (ret == EWOULDBLOCK) 1392 error = 0; 1393 else 1394 error = -ERESTARTSYS; 1395 goto intr; 1396 } 1397 } 1398 c->done--; 1399 sleepq_release(c); 1400 1401 intr: 1402 PICKUP_GIANT(); 1403 1404 /* return how many jiffies are left */ 1405 return (ret != 0 ? error : linux_timer_jiffies_until(end)); 1406 } 1407 1408 int 1409 linux_try_wait_for_completion(struct completion *c) 1410 { 1411 int isdone; 1412 1413 isdone = 1; 1414 sleepq_lock(c); 1415 if (c->done) 1416 c->done--; 1417 else 1418 isdone = 0; 1419 sleepq_release(c); 1420 return (isdone); 1421 } 1422 1423 int 1424 linux_completion_done(struct completion *c) 1425 { 1426 int isdone; 1427 1428 isdone = 1; 1429 sleepq_lock(c); 1430 if (c->done == 0) 1431 isdone = 0; 1432 sleepq_release(c); 1433 return (isdone); 1434 } 1435 1436 static void 1437 linux_cdev_release(struct kobject *kobj) 1438 { 1439 struct linux_cdev *cdev; 1440 struct kobject *parent; 1441 1442 cdev = container_of(kobj, struct linux_cdev, kobj); 1443 parent = kobj->parent; 1444 if (cdev->cdev) 1445 destroy_dev(cdev->cdev); 1446 kfree(cdev); 1447 kobject_put(parent); 1448 } 1449 1450 static void 1451 linux_cdev_static_release(struct kobject *kobj) 1452 { 1453 struct linux_cdev *cdev; 1454 struct kobject *parent; 1455 1456 cdev = container_of(kobj, struct linux_cdev, kobj); 1457 parent = kobj->parent; 1458 if (cdev->cdev) 1459 destroy_dev(cdev->cdev); 1460 kobject_put(parent); 1461 } 1462 1463 const struct kobj_type linux_cdev_ktype = { 1464 .release = linux_cdev_release, 1465 }; 1466 1467 const struct kobj_type linux_cdev_static_ktype = { 1468 .release = linux_cdev_static_release, 1469 }; 1470 1471 static void 1472 linux_handle_ifnet_link_event(void *arg, struct ifnet *ifp, int linkstate) 1473 { 1474 struct notifier_block *nb; 1475 1476 nb = arg; 1477 if (linkstate == LINK_STATE_UP) 1478 nb->notifier_call(nb, NETDEV_UP, ifp); 1479 else 1480 nb->notifier_call(nb, NETDEV_DOWN, ifp); 1481 } 1482 1483 static void 1484 linux_handle_ifnet_arrival_event(void *arg, struct ifnet *ifp) 1485 { 1486 struct notifier_block *nb; 1487 1488 nb = arg; 1489 nb->notifier_call(nb, NETDEV_REGISTER, ifp); 1490 } 1491 1492 static void 1493 linux_handle_ifnet_departure_event(void *arg, struct ifnet *ifp) 1494 { 1495 struct notifier_block *nb; 1496 1497 nb = arg; 1498 nb->notifier_call(nb, NETDEV_UNREGISTER, ifp); 1499 } 1500 1501 static void 1502 linux_handle_iflladdr_event(void *arg, struct ifnet *ifp) 1503 { 1504 struct notifier_block *nb; 1505 1506 nb = arg; 1507 nb->notifier_call(nb, NETDEV_CHANGEADDR, ifp); 1508 } 1509 1510 static void 1511 linux_handle_ifaddr_event(void *arg, struct ifnet *ifp) 1512 { 1513 struct notifier_block *nb; 1514 1515 nb = arg; 1516 nb->notifier_call(nb, NETDEV_CHANGEIFADDR, ifp); 1517 } 1518 1519 int 1520 register_netdevice_notifier(struct notifier_block *nb) 1521 { 1522 1523 nb->tags[NETDEV_UP] = EVENTHANDLER_REGISTER( 1524 ifnet_link_event, linux_handle_ifnet_link_event, nb, 0); 1525 nb->tags[NETDEV_REGISTER] = EVENTHANDLER_REGISTER( 1526 ifnet_arrival_event, linux_handle_ifnet_arrival_event, nb, 0); 1527 nb->tags[NETDEV_UNREGISTER] = EVENTHANDLER_REGISTER( 1528 ifnet_departure_event, linux_handle_ifnet_departure_event, nb, 0); 1529 nb->tags[NETDEV_CHANGEADDR] = EVENTHANDLER_REGISTER( 1530 iflladdr_event, linux_handle_iflladdr_event, nb, 0); 1531 1532 return (0); 1533 } 1534 1535 int 1536 register_inetaddr_notifier(struct notifier_block *nb) 1537 { 1538 1539 nb->tags[NETDEV_CHANGEIFADDR] = EVENTHANDLER_REGISTER( 1540 ifaddr_event, linux_handle_ifaddr_event, nb, 0); 1541 return (0); 1542 } 1543 1544 int 1545 unregister_netdevice_notifier(struct notifier_block *nb) 1546 { 1547 1548 EVENTHANDLER_DEREGISTER(ifnet_link_event, 1549 nb->tags[NETDEV_UP]); 1550 EVENTHANDLER_DEREGISTER(ifnet_arrival_event, 1551 nb->tags[NETDEV_REGISTER]); 1552 EVENTHANDLER_DEREGISTER(ifnet_departure_event, 1553 nb->tags[NETDEV_UNREGISTER]); 1554 EVENTHANDLER_DEREGISTER(iflladdr_event, 1555 nb->tags[NETDEV_CHANGEADDR]); 1556 1557 return (0); 1558 } 1559 1560 int 1561 unregister_inetaddr_notifier(struct notifier_block *nb) 1562 { 1563 1564 EVENTHANDLER_DEREGISTER(ifaddr_event, 1565 nb->tags[NETDEV_CHANGEIFADDR]); 1566 1567 return (0); 1568 } 1569 1570 struct list_sort_thunk { 1571 int (*cmp)(void *, struct list_head *, struct list_head *); 1572 void *priv; 1573 }; 1574 1575 static inline int 1576 linux_le_cmp(void *priv, const void *d1, const void *d2) 1577 { 1578 struct list_head *le1, *le2; 1579 struct list_sort_thunk *thunk; 1580 1581 thunk = priv; 1582 le1 = *(__DECONST(struct list_head **, d1)); 1583 le2 = *(__DECONST(struct list_head **, d2)); 1584 return ((thunk->cmp)(thunk->priv, le1, le2)); 1585 } 1586 1587 void 1588 list_sort(void *priv, struct list_head *head, int (*cmp)(void *priv, 1589 struct list_head *a, struct list_head *b)) 1590 { 1591 struct list_sort_thunk thunk; 1592 struct list_head **ar, *le; 1593 size_t count, i; 1594 1595 count = 0; 1596 list_for_each(le, head) 1597 count++; 1598 ar = malloc(sizeof(struct list_head *) * count, M_KMALLOC, M_WAITOK); 1599 i = 0; 1600 list_for_each(le, head) 1601 ar[i++] = le; 1602 thunk.cmp = cmp; 1603 thunk.priv = priv; 1604 qsort_r(ar, count, sizeof(struct list_head *), &thunk, linux_le_cmp); 1605 INIT_LIST_HEAD(head); 1606 for (i = 0; i < count; i++) 1607 list_add_tail(ar[i], head); 1608 free(ar, M_KMALLOC); 1609 } 1610 1611 void 1612 linux_irq_handler(void *ent) 1613 { 1614 struct irq_ent *irqe; 1615 1616 linux_set_current(curthread); 1617 1618 irqe = ent; 1619 irqe->handler(irqe->irq, irqe->arg); 1620 } 1621 1622 struct linux_cdev * 1623 linux_find_cdev(const char *name, unsigned major, unsigned minor) 1624 { 1625 int unit = MKDEV(major, minor); 1626 struct cdev *cdev; 1627 1628 dev_lock(); 1629 LIST_FOREACH(cdev, &linuxcdevsw.d_devs, si_list) { 1630 struct linux_cdev *ldev = cdev->si_drv1; 1631 if (dev2unit(cdev) == unit && 1632 strcmp(kobject_name(&ldev->kobj), name) == 0) { 1633 break; 1634 } 1635 } 1636 dev_unlock(); 1637 1638 return (cdev != NULL ? cdev->si_drv1 : NULL); 1639 } 1640 1641 int 1642 __register_chrdev(unsigned int major, unsigned int baseminor, 1643 unsigned int count, const char *name, 1644 const struct file_operations *fops) 1645 { 1646 struct linux_cdev *cdev; 1647 int ret = 0; 1648 int i; 1649 1650 for (i = baseminor; i < baseminor + count; i++) { 1651 cdev = cdev_alloc(); 1652 cdev_init(cdev, fops); 1653 kobject_set_name(&cdev->kobj, name); 1654 1655 ret = cdev_add(cdev, makedev(major, i), 1); 1656 if (ret != 0) 1657 break; 1658 } 1659 return (ret); 1660 } 1661 1662 int 1663 __register_chrdev_p(unsigned int major, unsigned int baseminor, 1664 unsigned int count, const char *name, 1665 const struct file_operations *fops, uid_t uid, 1666 gid_t gid, int mode) 1667 { 1668 struct linux_cdev *cdev; 1669 int ret = 0; 1670 int i; 1671 1672 for (i = baseminor; i < baseminor + count; i++) { 1673 cdev = cdev_alloc(); 1674 cdev_init(cdev, fops); 1675 kobject_set_name(&cdev->kobj, name); 1676 1677 ret = cdev_add_ext(cdev, makedev(major, i), uid, gid, mode); 1678 if (ret != 0) 1679 break; 1680 } 1681 return (ret); 1682 } 1683 1684 void 1685 __unregister_chrdev(unsigned int major, unsigned int baseminor, 1686 unsigned int count, const char *name) 1687 { 1688 struct linux_cdev *cdevp; 1689 int i; 1690 1691 for (i = baseminor; i < baseminor + count; i++) { 1692 cdevp = linux_find_cdev(name, major, i); 1693 if (cdevp != NULL) 1694 cdev_del(cdevp); 1695 } 1696 } 1697 1698 #if defined(__i386__) || defined(__amd64__) 1699 bool linux_cpu_has_clflush; 1700 #endif 1701 1702 static void 1703 linux_compat_init(void *arg) 1704 { 1705 struct sysctl_oid *rootoid; 1706 int i; 1707 1708 #if defined(__i386__) || defined(__amd64__) 1709 linux_cpu_has_clflush = (cpu_feature & CPUID_CLFSH); 1710 #endif 1711 rw_init(&linux_vma_lock, "lkpi-vma-lock"); 1712 1713 rootoid = SYSCTL_ADD_ROOT_NODE(NULL, 1714 OID_AUTO, "sys", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "sys"); 1715 kobject_init(&linux_class_root, &linux_class_ktype); 1716 kobject_set_name(&linux_class_root, "class"); 1717 linux_class_root.oidp = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(rootoid), 1718 OID_AUTO, "class", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "class"); 1719 kobject_init(&linux_root_device.kobj, &linux_dev_ktype); 1720 kobject_set_name(&linux_root_device.kobj, "device"); 1721 linux_root_device.kobj.oidp = SYSCTL_ADD_NODE(NULL, 1722 SYSCTL_CHILDREN(rootoid), OID_AUTO, "device", CTLFLAG_RD, NULL, 1723 "device"); 1724 linux_root_device.bsddev = root_bus; 1725 linux_class_misc.name = "misc"; 1726 class_register(&linux_class_misc); 1727 INIT_LIST_HEAD(&pci_drivers); 1728 INIT_LIST_HEAD(&pci_devices); 1729 spin_lock_init(&pci_lock); 1730 mtx_init(&vmmaplock, "IO Map lock", NULL, MTX_DEF); 1731 for (i = 0; i < VMMAP_HASH_SIZE; i++) 1732 LIST_INIT(&vmmaphead[i]); 1733 } 1734 SYSINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_init, NULL); 1735 1736 static void 1737 linux_compat_uninit(void *arg) 1738 { 1739 linux_kobject_kfree_name(&linux_class_root); 1740 linux_kobject_kfree_name(&linux_root_device.kobj); 1741 linux_kobject_kfree_name(&linux_class_misc.kobj); 1742 1743 rw_destroy(&linux_vma_lock); 1744 } 1745 SYSUNINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_uninit, NULL); 1746 1747 /* 1748 * NOTE: Linux frequently uses "unsigned long" for pointer to integer 1749 * conversion and vice versa, where in FreeBSD "uintptr_t" would be 1750 * used. Assert these types have the same size, else some parts of the 1751 * LinuxKPI may not work like expected: 1752 */ 1753 CTASSERT(sizeof(unsigned long) == sizeof(uintptr_t)); 1754