1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2017 Mellanox Technologies, Ltd. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice unmodified, this list of conditions, and the following 13 * disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/malloc.h> 36 #include <sys/kernel.h> 37 #include <sys/sysctl.h> 38 #include <sys/proc.h> 39 #include <sys/sglist.h> 40 #include <sys/sleepqueue.h> 41 #include <sys/lock.h> 42 #include <sys/mutex.h> 43 #include <sys/bus.h> 44 #include <sys/fcntl.h> 45 #include <sys/file.h> 46 #include <sys/filio.h> 47 #include <sys/rwlock.h> 48 49 #include <vm/vm.h> 50 #include <vm/pmap.h> 51 #include <vm/vm_object.h> 52 #include <vm/vm_page.h> 53 #include <vm/vm_pager.h> 54 55 #include <machine/stdarg.h> 56 57 #if defined(__i386__) || defined(__amd64__) 58 #include <machine/md_var.h> 59 #endif 60 61 #include <linux/kobject.h> 62 #include <linux/device.h> 63 #include <linux/slab.h> 64 #include <linux/module.h> 65 #include <linux/moduleparam.h> 66 #include <linux/cdev.h> 67 #include <linux/file.h> 68 #include <linux/sysfs.h> 69 #include <linux/mm.h> 70 #include <linux/io.h> 71 #include <linux/vmalloc.h> 72 #include <linux/netdevice.h> 73 #include <linux/timer.h> 74 #include <linux/interrupt.h> 75 #include <linux/uaccess.h> 76 #include <linux/kernel.h> 77 #include <linux/list.h> 78 #include <linux/compat.h> 79 #include <linux/poll.h> 80 #include <linux/smp.h> 81 82 #if defined(__i386__) || defined(__amd64__) 83 #include <asm/smp.h> 84 #endif 85 86 SYSCTL_NODE(_compat, OID_AUTO, linuxkpi, CTLFLAG_RW, 0, "LinuxKPI parameters"); 87 88 MALLOC_DEFINE(M_KMALLOC, "linux", "Linux kmalloc compat"); 89 90 #include <linux/rbtree.h> 91 /* Undo Linux compat changes. */ 92 #undef RB_ROOT 93 #undef file 94 #undef cdev 95 #define RB_ROOT(head) (head)->rbh_root 96 97 static struct vm_area_struct *linux_cdev_handle_find(void *handle); 98 99 struct kobject linux_class_root; 100 struct device linux_root_device; 101 struct class linux_class_misc; 102 struct list_head pci_drivers; 103 struct list_head pci_devices; 104 spinlock_t pci_lock; 105 106 unsigned long linux_timer_hz_mask; 107 108 int 109 panic_cmp(struct rb_node *one, struct rb_node *two) 110 { 111 panic("no cmp"); 112 } 113 114 RB_GENERATE(linux_root, rb_node, __entry, panic_cmp); 115 116 int 117 kobject_set_name_vargs(struct kobject *kobj, const char *fmt, va_list args) 118 { 119 va_list tmp_va; 120 int len; 121 char *old; 122 char *name; 123 char dummy; 124 125 old = kobj->name; 126 127 if (old && fmt == NULL) 128 return (0); 129 130 /* compute length of string */ 131 va_copy(tmp_va, args); 132 len = vsnprintf(&dummy, 0, fmt, tmp_va); 133 va_end(tmp_va); 134 135 /* account for zero termination */ 136 len++; 137 138 /* check for error */ 139 if (len < 1) 140 return (-EINVAL); 141 142 /* allocate memory for string */ 143 name = kzalloc(len, GFP_KERNEL); 144 if (name == NULL) 145 return (-ENOMEM); 146 vsnprintf(name, len, fmt, args); 147 kobj->name = name; 148 149 /* free old string */ 150 kfree(old); 151 152 /* filter new string */ 153 for (; *name != '\0'; name++) 154 if (*name == '/') 155 *name = '!'; 156 return (0); 157 } 158 159 int 160 kobject_set_name(struct kobject *kobj, const char *fmt, ...) 161 { 162 va_list args; 163 int error; 164 165 va_start(args, fmt); 166 error = kobject_set_name_vargs(kobj, fmt, args); 167 va_end(args); 168 169 return (error); 170 } 171 172 static int 173 kobject_add_complete(struct kobject *kobj, struct kobject *parent) 174 { 175 const struct kobj_type *t; 176 int error; 177 178 kobj->parent = parent; 179 error = sysfs_create_dir(kobj); 180 if (error == 0 && kobj->ktype && kobj->ktype->default_attrs) { 181 struct attribute **attr; 182 t = kobj->ktype; 183 184 for (attr = t->default_attrs; *attr != NULL; attr++) { 185 error = sysfs_create_file(kobj, *attr); 186 if (error) 187 break; 188 } 189 if (error) 190 sysfs_remove_dir(kobj); 191 192 } 193 return (error); 194 } 195 196 int 197 kobject_add(struct kobject *kobj, struct kobject *parent, const char *fmt, ...) 198 { 199 va_list args; 200 int error; 201 202 va_start(args, fmt); 203 error = kobject_set_name_vargs(kobj, fmt, args); 204 va_end(args); 205 if (error) 206 return (error); 207 208 return kobject_add_complete(kobj, parent); 209 } 210 211 void 212 linux_kobject_release(struct kref *kref) 213 { 214 struct kobject *kobj; 215 char *name; 216 217 kobj = container_of(kref, struct kobject, kref); 218 sysfs_remove_dir(kobj); 219 name = kobj->name; 220 if (kobj->ktype && kobj->ktype->release) 221 kobj->ktype->release(kobj); 222 kfree(name); 223 } 224 225 static void 226 linux_kobject_kfree(struct kobject *kobj) 227 { 228 kfree(kobj); 229 } 230 231 static void 232 linux_kobject_kfree_name(struct kobject *kobj) 233 { 234 if (kobj) { 235 kfree(kobj->name); 236 } 237 } 238 239 const struct kobj_type linux_kfree_type = { 240 .release = linux_kobject_kfree 241 }; 242 243 static void 244 linux_device_release(struct device *dev) 245 { 246 pr_debug("linux_device_release: %s\n", dev_name(dev)); 247 kfree(dev); 248 } 249 250 static ssize_t 251 linux_class_show(struct kobject *kobj, struct attribute *attr, char *buf) 252 { 253 struct class_attribute *dattr; 254 ssize_t error; 255 256 dattr = container_of(attr, struct class_attribute, attr); 257 error = -EIO; 258 if (dattr->show) 259 error = dattr->show(container_of(kobj, struct class, kobj), 260 dattr, buf); 261 return (error); 262 } 263 264 static ssize_t 265 linux_class_store(struct kobject *kobj, struct attribute *attr, const char *buf, 266 size_t count) 267 { 268 struct class_attribute *dattr; 269 ssize_t error; 270 271 dattr = container_of(attr, struct class_attribute, attr); 272 error = -EIO; 273 if (dattr->store) 274 error = dattr->store(container_of(kobj, struct class, kobj), 275 dattr, buf, count); 276 return (error); 277 } 278 279 static void 280 linux_class_release(struct kobject *kobj) 281 { 282 struct class *class; 283 284 class = container_of(kobj, struct class, kobj); 285 if (class->class_release) 286 class->class_release(class); 287 } 288 289 static const struct sysfs_ops linux_class_sysfs = { 290 .show = linux_class_show, 291 .store = linux_class_store, 292 }; 293 294 const struct kobj_type linux_class_ktype = { 295 .release = linux_class_release, 296 .sysfs_ops = &linux_class_sysfs 297 }; 298 299 static void 300 linux_dev_release(struct kobject *kobj) 301 { 302 struct device *dev; 303 304 dev = container_of(kobj, struct device, kobj); 305 /* This is the precedence defined by linux. */ 306 if (dev->release) 307 dev->release(dev); 308 else if (dev->class && dev->class->dev_release) 309 dev->class->dev_release(dev); 310 } 311 312 static ssize_t 313 linux_dev_show(struct kobject *kobj, struct attribute *attr, char *buf) 314 { 315 struct device_attribute *dattr; 316 ssize_t error; 317 318 dattr = container_of(attr, struct device_attribute, attr); 319 error = -EIO; 320 if (dattr->show) 321 error = dattr->show(container_of(kobj, struct device, kobj), 322 dattr, buf); 323 return (error); 324 } 325 326 static ssize_t 327 linux_dev_store(struct kobject *kobj, struct attribute *attr, const char *buf, 328 size_t count) 329 { 330 struct device_attribute *dattr; 331 ssize_t error; 332 333 dattr = container_of(attr, struct device_attribute, attr); 334 error = -EIO; 335 if (dattr->store) 336 error = dattr->store(container_of(kobj, struct device, kobj), 337 dattr, buf, count); 338 return (error); 339 } 340 341 static const struct sysfs_ops linux_dev_sysfs = { 342 .show = linux_dev_show, 343 .store = linux_dev_store, 344 }; 345 346 const struct kobj_type linux_dev_ktype = { 347 .release = linux_dev_release, 348 .sysfs_ops = &linux_dev_sysfs 349 }; 350 351 struct device * 352 device_create(struct class *class, struct device *parent, dev_t devt, 353 void *drvdata, const char *fmt, ...) 354 { 355 struct device *dev; 356 va_list args; 357 358 dev = kzalloc(sizeof(*dev), M_WAITOK); 359 dev->parent = parent; 360 dev->class = class; 361 dev->devt = devt; 362 dev->driver_data = drvdata; 363 dev->release = linux_device_release; 364 va_start(args, fmt); 365 kobject_set_name_vargs(&dev->kobj, fmt, args); 366 va_end(args); 367 device_register(dev); 368 369 return (dev); 370 } 371 372 int 373 kobject_init_and_add(struct kobject *kobj, const struct kobj_type *ktype, 374 struct kobject *parent, const char *fmt, ...) 375 { 376 va_list args; 377 int error; 378 379 kobject_init(kobj, ktype); 380 kobj->ktype = ktype; 381 kobj->parent = parent; 382 kobj->name = NULL; 383 384 va_start(args, fmt); 385 error = kobject_set_name_vargs(kobj, fmt, args); 386 va_end(args); 387 if (error) 388 return (error); 389 return kobject_add_complete(kobj, parent); 390 } 391 392 static void 393 linux_file_dtor(void *cdp) 394 { 395 struct linux_file *filp; 396 397 linux_set_current(curthread); 398 filp = cdp; 399 filp->f_op->release(filp->f_vnode, filp); 400 vdrop(filp->f_vnode); 401 kfree(filp); 402 } 403 404 static int 405 linux_cdev_pager_populate(vm_object_t vm_obj, vm_pindex_t pidx, int fault_type, 406 vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last) 407 { 408 struct vm_area_struct *vmap; 409 struct vm_fault vmf; 410 int err; 411 412 linux_set_current(curthread); 413 414 /* get VM area structure */ 415 vmap = linux_cdev_handle_find(vm_obj->handle); 416 MPASS(vmap != NULL); 417 MPASS(vmap->vm_private_data == vm_obj->handle); 418 419 /* fill out VM fault structure */ 420 vmf.virtual_address = (void *)((uintptr_t)pidx << PAGE_SHIFT); 421 vmf.flags = (fault_type & VM_PROT_WRITE) ? FAULT_FLAG_WRITE : 0; 422 vmf.pgoff = 0; 423 vmf.page = NULL; 424 425 VM_OBJECT_WUNLOCK(vm_obj); 426 427 down_write(&vmap->vm_mm->mmap_sem); 428 if (unlikely(vmap->vm_ops == NULL)) { 429 err = VM_FAULT_SIGBUS; 430 } else { 431 vmap->vm_pfn_count = 0; 432 vmap->vm_pfn_pcount = &vmap->vm_pfn_count; 433 vmap->vm_obj = vm_obj; 434 435 err = vmap->vm_ops->fault(vmap, &vmf); 436 437 while (vmap->vm_pfn_count == 0 && err == VM_FAULT_NOPAGE) { 438 kern_yield(PRI_USER); 439 err = vmap->vm_ops->fault(vmap, &vmf); 440 } 441 } 442 443 /* translate return code */ 444 switch (err) { 445 case VM_FAULT_OOM: 446 err = VM_PAGER_AGAIN; 447 break; 448 case VM_FAULT_SIGBUS: 449 err = VM_PAGER_BAD; 450 break; 451 case VM_FAULT_NOPAGE: 452 /* 453 * By contract the fault handler will return having 454 * busied all the pages itself. If pidx is already 455 * found in the object, it will simply xbusy the first 456 * page and return with vm_pfn_count set to 1. 457 */ 458 *first = vmap->vm_pfn_first; 459 *last = *first + vmap->vm_pfn_count - 1; 460 err = VM_PAGER_OK; 461 break; 462 default: 463 err = VM_PAGER_ERROR; 464 break; 465 } 466 up_write(&vmap->vm_mm->mmap_sem); 467 VM_OBJECT_WLOCK(vm_obj); 468 return (err); 469 } 470 471 static struct rwlock linux_vma_lock; 472 static TAILQ_HEAD(, vm_area_struct) linux_vma_head = 473 TAILQ_HEAD_INITIALIZER(linux_vma_head); 474 475 static struct vm_area_struct * 476 linux_cdev_handle_insert(void *handle, struct vm_area_struct *vmap) 477 { 478 struct vm_area_struct *ptr; 479 480 rw_wlock(&linux_vma_lock); 481 TAILQ_FOREACH(ptr, &linux_vma_head, vm_entry) { 482 if (ptr->vm_private_data == handle) { 483 rw_wunlock(&linux_vma_lock); 484 kfree(vmap); 485 return (NULL); 486 } 487 } 488 /* 489 * The same VM object might be shared by multiple processes 490 * and the mm_struct is usually freed when a process exits. 491 * 492 * The atomic reference below makes sure the mm_struct is 493 * available as long as the vmap is in the linux_vma_head. 494 */ 495 if (atomic_inc_not_zero(&vmap->vm_mm->mm_users) == 0) 496 panic("linuxkpi: mm_users is zero\n"); 497 498 TAILQ_INSERT_TAIL(&linux_vma_head, vmap, vm_entry); 499 rw_wunlock(&linux_vma_lock); 500 return (vmap); 501 } 502 503 static void 504 linux_cdev_handle_remove(struct vm_area_struct *vmap) 505 { 506 if (vmap == NULL) 507 return; 508 509 rw_wlock(&linux_vma_lock); 510 TAILQ_REMOVE(&linux_vma_head, vmap, vm_entry); 511 rw_wunlock(&linux_vma_lock); 512 513 /* Drop reference on mm_struct */ 514 mmput(vmap->vm_mm); 515 kfree(vmap); 516 } 517 518 static struct vm_area_struct * 519 linux_cdev_handle_find(void *handle) 520 { 521 struct vm_area_struct *vmap; 522 523 rw_rlock(&linux_vma_lock); 524 TAILQ_FOREACH(vmap, &linux_vma_head, vm_entry) { 525 if (vmap->vm_private_data == handle) 526 break; 527 } 528 rw_runlock(&linux_vma_lock); 529 return (vmap); 530 } 531 532 static int 533 linux_cdev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 534 vm_ooffset_t foff, struct ucred *cred, u_short *color) 535 { 536 const struct vm_operations_struct *vm_ops; 537 struct vm_area_struct *vmap; 538 539 vmap = linux_cdev_handle_find(handle); 540 MPASS(vmap != NULL); 541 542 *color = 0; 543 544 down_write(&vmap->vm_mm->mmap_sem); 545 vm_ops = vmap->vm_ops; 546 if (likely(vm_ops != NULL)) 547 vm_ops->open(vmap); 548 up_write(&vmap->vm_mm->mmap_sem); 549 550 return (0); 551 } 552 553 static void 554 linux_cdev_pager_dtor(void *handle) 555 { 556 const struct vm_operations_struct *vm_ops; 557 struct vm_area_struct *vmap; 558 559 vmap = linux_cdev_handle_find(handle); 560 MPASS(vmap != NULL); 561 562 down_write(&vmap->vm_mm->mmap_sem); 563 vm_ops = vmap->vm_ops; 564 if (likely(vm_ops != NULL)) 565 vm_ops->close(vmap); 566 up_write(&vmap->vm_mm->mmap_sem); 567 568 linux_cdev_handle_remove(vmap); 569 } 570 571 static struct cdev_pager_ops linux_cdev_pager_ops = { 572 .cdev_pg_populate = linux_cdev_pager_populate, 573 .cdev_pg_ctor = linux_cdev_pager_ctor, 574 .cdev_pg_dtor = linux_cdev_pager_dtor 575 }; 576 577 static int 578 linux_dev_open(struct cdev *dev, int oflags, int devtype, struct thread *td) 579 { 580 struct linux_cdev *ldev; 581 struct linux_file *filp; 582 struct file *file; 583 int error; 584 585 file = td->td_fpop; 586 ldev = dev->si_drv1; 587 if (ldev == NULL) 588 return (ENODEV); 589 filp = kzalloc(sizeof(*filp), GFP_KERNEL); 590 filp->f_dentry = &filp->f_dentry_store; 591 filp->f_op = ldev->ops; 592 filp->f_flags = file->f_flag; 593 vhold(file->f_vnode); 594 filp->f_vnode = file->f_vnode; 595 linux_set_current(td); 596 if (filp->f_op->open) { 597 error = -filp->f_op->open(file->f_vnode, filp); 598 if (error) { 599 kfree(filp); 600 goto done; 601 } 602 } 603 error = devfs_set_cdevpriv(filp, linux_file_dtor); 604 if (error) { 605 filp->f_op->release(file->f_vnode, filp); 606 kfree(filp); 607 } 608 done: 609 return (error); 610 } 611 612 static int 613 linux_dev_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 614 { 615 struct linux_file *filp; 616 struct file *file; 617 int error; 618 619 file = td->td_fpop; 620 if (dev->si_drv1 == NULL) 621 return (0); 622 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 623 return (error); 624 filp->f_flags = file->f_flag; 625 devfs_clear_cdevpriv(); 626 627 628 return (0); 629 } 630 631 #define LINUX_IOCTL_MIN_PTR 0x10000UL 632 #define LINUX_IOCTL_MAX_PTR (LINUX_IOCTL_MIN_PTR + IOCPARM_MAX) 633 634 static inline int 635 linux_remap_address(void **uaddr, size_t len) 636 { 637 uintptr_t uaddr_val = (uintptr_t)(*uaddr); 638 639 if (unlikely(uaddr_val >= LINUX_IOCTL_MIN_PTR && 640 uaddr_val < LINUX_IOCTL_MAX_PTR)) { 641 struct task_struct *pts = current; 642 if (pts == NULL) { 643 *uaddr = NULL; 644 return (1); 645 } 646 647 /* compute data offset */ 648 uaddr_val -= LINUX_IOCTL_MIN_PTR; 649 650 /* check that length is within bounds */ 651 if ((len > IOCPARM_MAX) || 652 (uaddr_val + len) > pts->bsd_ioctl_len) { 653 *uaddr = NULL; 654 return (1); 655 } 656 657 /* re-add kernel buffer address */ 658 uaddr_val += (uintptr_t)pts->bsd_ioctl_data; 659 660 /* update address location */ 661 *uaddr = (void *)uaddr_val; 662 return (1); 663 } 664 return (0); 665 } 666 667 int 668 linux_copyin(const void *uaddr, void *kaddr, size_t len) 669 { 670 if (linux_remap_address(__DECONST(void **, &uaddr), len)) { 671 if (uaddr == NULL) 672 return (-EFAULT); 673 memcpy(kaddr, uaddr, len); 674 return (0); 675 } 676 return (-copyin(uaddr, kaddr, len)); 677 } 678 679 int 680 linux_copyout(const void *kaddr, void *uaddr, size_t len) 681 { 682 if (linux_remap_address(&uaddr, len)) { 683 if (uaddr == NULL) 684 return (-EFAULT); 685 memcpy(uaddr, kaddr, len); 686 return (0); 687 } 688 return (-copyout(kaddr, uaddr, len)); 689 } 690 691 size_t 692 linux_clear_user(void *_uaddr, size_t _len) 693 { 694 uint8_t *uaddr = _uaddr; 695 size_t len = _len; 696 697 /* make sure uaddr is aligned before going into the fast loop */ 698 while (((uintptr_t)uaddr & 7) != 0 && len > 7) { 699 if (subyte(uaddr, 0)) 700 return (_len); 701 uaddr++; 702 len--; 703 } 704 705 /* zero 8 bytes at a time */ 706 while (len > 7) { 707 #ifdef __LP64__ 708 if (suword64(uaddr, 0)) 709 return (_len); 710 #else 711 if (suword32(uaddr, 0)) 712 return (_len); 713 if (suword32(uaddr + 4, 0)) 714 return (_len); 715 #endif 716 uaddr += 8; 717 len -= 8; 718 } 719 720 /* zero fill end, if any */ 721 while (len > 0) { 722 if (subyte(uaddr, 0)) 723 return (_len); 724 uaddr++; 725 len--; 726 } 727 return (0); 728 } 729 730 int 731 linux_access_ok(int rw, const void *uaddr, size_t len) 732 { 733 uintptr_t saddr; 734 uintptr_t eaddr; 735 736 /* get start and end address */ 737 saddr = (uintptr_t)uaddr; 738 eaddr = (uintptr_t)uaddr + len; 739 740 /* verify addresses are valid for userspace */ 741 return ((saddr == eaddr) || 742 (eaddr > saddr && eaddr <= VM_MAXUSER_ADDRESS)); 743 } 744 745 static int 746 linux_dev_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag, 747 struct thread *td) 748 { 749 struct linux_file *filp; 750 struct file *file; 751 unsigned size; 752 int error; 753 754 file = td->td_fpop; 755 if (dev->si_drv1 == NULL) 756 return (ENXIO); 757 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 758 return (error); 759 filp->f_flags = file->f_flag; 760 761 linux_set_current(td); 762 size = IOCPARM_LEN(cmd); 763 /* refer to logic in sys_ioctl() */ 764 if (size > 0) { 765 /* 766 * Setup hint for linux_copyin() and linux_copyout(). 767 * 768 * Background: Linux code expects a user-space address 769 * while FreeBSD supplies a kernel-space address. 770 */ 771 current->bsd_ioctl_data = data; 772 current->bsd_ioctl_len = size; 773 data = (void *)LINUX_IOCTL_MIN_PTR; 774 } else { 775 /* fetch user-space pointer */ 776 data = *(void **)data; 777 } 778 if (filp->f_op->unlocked_ioctl) 779 error = -filp->f_op->unlocked_ioctl(filp, cmd, (u_long)data); 780 else 781 error = ENOTTY; 782 if (size > 0) { 783 current->bsd_ioctl_data = NULL; 784 current->bsd_ioctl_len = 0; 785 } 786 787 return (error); 788 } 789 790 static int 791 linux_dev_read(struct cdev *dev, struct uio *uio, int ioflag) 792 { 793 struct linux_file *filp; 794 struct thread *td; 795 struct file *file; 796 ssize_t bytes; 797 int error; 798 799 td = curthread; 800 file = td->td_fpop; 801 if (dev->si_drv1 == NULL) 802 return (ENXIO); 803 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 804 return (error); 805 filp->f_flags = file->f_flag; 806 /* XXX no support for I/O vectors currently */ 807 if (uio->uio_iovcnt != 1) 808 return (EOPNOTSUPP); 809 linux_set_current(td); 810 if (filp->f_op->read) { 811 bytes = filp->f_op->read(filp, uio->uio_iov->iov_base, 812 uio->uio_iov->iov_len, &uio->uio_offset); 813 if (bytes >= 0) { 814 uio->uio_iov->iov_base = 815 ((uint8_t *)uio->uio_iov->iov_base) + bytes; 816 uio->uio_iov->iov_len -= bytes; 817 uio->uio_resid -= bytes; 818 } else 819 error = -bytes; 820 } else 821 error = ENXIO; 822 823 return (error); 824 } 825 826 static int 827 linux_dev_write(struct cdev *dev, struct uio *uio, int ioflag) 828 { 829 struct linux_file *filp; 830 struct thread *td; 831 struct file *file; 832 ssize_t bytes; 833 int error; 834 835 td = curthread; 836 file = td->td_fpop; 837 if (dev->si_drv1 == NULL) 838 return (ENXIO); 839 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 840 return (error); 841 filp->f_flags = file->f_flag; 842 /* XXX no support for I/O vectors currently */ 843 if (uio->uio_iovcnt != 1) 844 return (EOPNOTSUPP); 845 linux_set_current(td); 846 if (filp->f_op->write) { 847 bytes = filp->f_op->write(filp, uio->uio_iov->iov_base, 848 uio->uio_iov->iov_len, &uio->uio_offset); 849 if (bytes >= 0) { 850 uio->uio_iov->iov_base = 851 ((uint8_t *)uio->uio_iov->iov_base) + bytes; 852 uio->uio_iov->iov_len -= bytes; 853 uio->uio_resid -= bytes; 854 } else 855 error = -bytes; 856 } else 857 error = ENXIO; 858 859 return (error); 860 } 861 862 static int 863 linux_dev_poll(struct cdev *dev, int events, struct thread *td) 864 { 865 struct linux_file *filp; 866 struct file *file; 867 int revents; 868 869 if (dev->si_drv1 == NULL) 870 goto error; 871 if (devfs_get_cdevpriv((void **)&filp) != 0) 872 goto error; 873 874 file = td->td_fpop; 875 filp->f_flags = file->f_flag; 876 linux_set_current(td); 877 if (filp->f_op->poll) 878 revents = filp->f_op->poll(filp, NULL) & events; 879 else 880 revents = 0; 881 882 return (revents); 883 error: 884 return (events & (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM)); 885 } 886 887 static int 888 linux_dev_mmap_single(struct cdev *dev, vm_ooffset_t *offset, 889 vm_size_t size, struct vm_object **object, int nprot) 890 { 891 struct vm_area_struct *vmap; 892 struct linux_file *filp; 893 struct thread *td; 894 struct file *file; 895 vm_memattr_t attr; 896 int error; 897 898 td = curthread; 899 file = td->td_fpop; 900 if (dev->si_drv1 == NULL) 901 return (ENODEV); 902 if ((error = devfs_get_cdevpriv((void **)&filp)) != 0) 903 return (error); 904 filp->f_flags = file->f_flag; 905 906 if (filp->f_op->mmap == NULL) 907 return (ENODEV); 908 909 linux_set_current(td); 910 911 vmap = kzalloc(sizeof(*vmap), GFP_KERNEL); 912 vmap->vm_start = 0; 913 vmap->vm_end = size; 914 vmap->vm_pgoff = *offset / PAGE_SIZE; 915 vmap->vm_pfn = 0; 916 vmap->vm_flags = vmap->vm_page_prot = nprot; 917 vmap->vm_ops = NULL; 918 vmap->vm_file = filp; 919 vmap->vm_mm = current->mm; 920 921 if (unlikely(down_write_killable(&vmap->vm_mm->mmap_sem))) { 922 error = EINTR; 923 } else { 924 error = -filp->f_op->mmap(filp, vmap); 925 up_write(&vmap->vm_mm->mmap_sem); 926 } 927 928 if (error != 0) { 929 kfree(vmap); 930 return (error); 931 } 932 933 attr = pgprot2cachemode(vmap->vm_page_prot); 934 935 if (vmap->vm_ops != NULL) { 936 void *vm_private_data; 937 938 if (vmap->vm_ops->fault == NULL || 939 vmap->vm_ops->open == NULL || 940 vmap->vm_ops->close == NULL || 941 vmap->vm_private_data == NULL) { 942 kfree(vmap); 943 return (EINVAL); 944 } 945 946 vm_private_data = vmap->vm_private_data; 947 948 vmap = linux_cdev_handle_insert(vm_private_data, vmap); 949 950 *object = cdev_pager_allocate(vm_private_data, OBJT_MGTDEVICE, 951 &linux_cdev_pager_ops, size, nprot, *offset, curthread->td_ucred); 952 953 if (*object == NULL) { 954 linux_cdev_handle_remove(vmap); 955 return (EINVAL); 956 } 957 } else { 958 struct sglist *sg; 959 960 sg = sglist_alloc(1, M_WAITOK); 961 sglist_append_phys(sg, (vm_paddr_t)vmap->vm_pfn << PAGE_SHIFT, vmap->vm_len); 962 963 *object = vm_pager_allocate(OBJT_SG, sg, vmap->vm_len, 964 nprot, 0, curthread->td_ucred); 965 966 kfree(vmap); 967 968 if (*object == NULL) { 969 sglist_free(sg); 970 return (EINVAL); 971 } 972 } 973 974 if (attr != VM_MEMATTR_DEFAULT) { 975 VM_OBJECT_WLOCK(*object); 976 vm_object_set_memattr(*object, attr); 977 VM_OBJECT_WUNLOCK(*object); 978 } 979 *offset = 0; 980 return (0); 981 } 982 983 struct cdevsw linuxcdevsw = { 984 .d_version = D_VERSION, 985 .d_flags = D_TRACKCLOSE, 986 .d_open = linux_dev_open, 987 .d_close = linux_dev_close, 988 .d_read = linux_dev_read, 989 .d_write = linux_dev_write, 990 .d_ioctl = linux_dev_ioctl, 991 .d_mmap_single = linux_dev_mmap_single, 992 .d_poll = linux_dev_poll, 993 }; 994 995 static int 996 linux_file_read(struct file *file, struct uio *uio, struct ucred *active_cred, 997 int flags, struct thread *td) 998 { 999 struct linux_file *filp; 1000 ssize_t bytes; 1001 int error; 1002 1003 error = 0; 1004 filp = (struct linux_file *)file->f_data; 1005 filp->f_flags = file->f_flag; 1006 /* XXX no support for I/O vectors currently */ 1007 if (uio->uio_iovcnt != 1) 1008 return (EOPNOTSUPP); 1009 linux_set_current(td); 1010 if (filp->f_op->read) { 1011 bytes = filp->f_op->read(filp, uio->uio_iov->iov_base, 1012 uio->uio_iov->iov_len, &uio->uio_offset); 1013 if (bytes >= 0) { 1014 uio->uio_iov->iov_base = 1015 ((uint8_t *)uio->uio_iov->iov_base) + bytes; 1016 uio->uio_iov->iov_len -= bytes; 1017 uio->uio_resid -= bytes; 1018 } else 1019 error = -bytes; 1020 } else 1021 error = ENXIO; 1022 1023 return (error); 1024 } 1025 1026 static int 1027 linux_file_poll(struct file *file, int events, struct ucred *active_cred, 1028 struct thread *td) 1029 { 1030 struct linux_file *filp; 1031 int revents; 1032 1033 filp = (struct linux_file *)file->f_data; 1034 filp->f_flags = file->f_flag; 1035 linux_set_current(td); 1036 if (filp->f_op->poll) 1037 revents = filp->f_op->poll(filp, NULL) & events; 1038 else 1039 revents = 0; 1040 1041 return (revents); 1042 } 1043 1044 static int 1045 linux_file_close(struct file *file, struct thread *td) 1046 { 1047 struct linux_file *filp; 1048 int error; 1049 1050 filp = (struct linux_file *)file->f_data; 1051 filp->f_flags = file->f_flag; 1052 linux_set_current(td); 1053 error = -filp->f_op->release(NULL, filp); 1054 funsetown(&filp->f_sigio); 1055 kfree(filp); 1056 1057 return (error); 1058 } 1059 1060 static int 1061 linux_file_ioctl(struct file *fp, u_long cmd, void *data, struct ucred *cred, 1062 struct thread *td) 1063 { 1064 struct linux_file *filp; 1065 int error; 1066 1067 filp = (struct linux_file *)fp->f_data; 1068 filp->f_flags = fp->f_flag; 1069 error = 0; 1070 1071 linux_set_current(td); 1072 switch (cmd) { 1073 case FIONBIO: 1074 break; 1075 case FIOASYNC: 1076 if (filp->f_op->fasync == NULL) 1077 break; 1078 error = filp->f_op->fasync(0, filp, fp->f_flag & FASYNC); 1079 break; 1080 case FIOSETOWN: 1081 error = fsetown(*(int *)data, &filp->f_sigio); 1082 if (error == 0) 1083 error = filp->f_op->fasync(0, filp, 1084 fp->f_flag & FASYNC); 1085 break; 1086 case FIOGETOWN: 1087 *(int *)data = fgetown(&filp->f_sigio); 1088 break; 1089 default: 1090 error = ENOTTY; 1091 break; 1092 } 1093 return (error); 1094 } 1095 1096 static int 1097 linux_file_stat(struct file *fp, struct stat *sb, struct ucred *active_cred, 1098 struct thread *td) 1099 { 1100 1101 return (EOPNOTSUPP); 1102 } 1103 1104 static int 1105 linux_file_fill_kinfo(struct file *fp, struct kinfo_file *kif, 1106 struct filedesc *fdp) 1107 { 1108 1109 return (0); 1110 } 1111 1112 struct fileops linuxfileops = { 1113 .fo_read = linux_file_read, 1114 .fo_write = invfo_rdwr, 1115 .fo_truncate = invfo_truncate, 1116 .fo_kqfilter = invfo_kqfilter, 1117 .fo_stat = linux_file_stat, 1118 .fo_fill_kinfo = linux_file_fill_kinfo, 1119 .fo_poll = linux_file_poll, 1120 .fo_close = linux_file_close, 1121 .fo_ioctl = linux_file_ioctl, 1122 .fo_chmod = invfo_chmod, 1123 .fo_chown = invfo_chown, 1124 .fo_sendfile = invfo_sendfile, 1125 }; 1126 1127 /* 1128 * Hash of vmmap addresses. This is infrequently accessed and does not 1129 * need to be particularly large. This is done because we must store the 1130 * caller's idea of the map size to properly unmap. 1131 */ 1132 struct vmmap { 1133 LIST_ENTRY(vmmap) vm_next; 1134 void *vm_addr; 1135 unsigned long vm_size; 1136 }; 1137 1138 struct vmmaphd { 1139 struct vmmap *lh_first; 1140 }; 1141 #define VMMAP_HASH_SIZE 64 1142 #define VMMAP_HASH_MASK (VMMAP_HASH_SIZE - 1) 1143 #define VM_HASH(addr) ((uintptr_t)(addr) >> PAGE_SHIFT) & VMMAP_HASH_MASK 1144 static struct vmmaphd vmmaphead[VMMAP_HASH_SIZE]; 1145 static struct mtx vmmaplock; 1146 1147 static void 1148 vmmap_add(void *addr, unsigned long size) 1149 { 1150 struct vmmap *vmmap; 1151 1152 vmmap = kmalloc(sizeof(*vmmap), GFP_KERNEL); 1153 mtx_lock(&vmmaplock); 1154 vmmap->vm_size = size; 1155 vmmap->vm_addr = addr; 1156 LIST_INSERT_HEAD(&vmmaphead[VM_HASH(addr)], vmmap, vm_next); 1157 mtx_unlock(&vmmaplock); 1158 } 1159 1160 static struct vmmap * 1161 vmmap_remove(void *addr) 1162 { 1163 struct vmmap *vmmap; 1164 1165 mtx_lock(&vmmaplock); 1166 LIST_FOREACH(vmmap, &vmmaphead[VM_HASH(addr)], vm_next) 1167 if (vmmap->vm_addr == addr) 1168 break; 1169 if (vmmap) 1170 LIST_REMOVE(vmmap, vm_next); 1171 mtx_unlock(&vmmaplock); 1172 1173 return (vmmap); 1174 } 1175 1176 #if defined(__i386__) || defined(__amd64__) 1177 void * 1178 _ioremap_attr(vm_paddr_t phys_addr, unsigned long size, int attr) 1179 { 1180 void *addr; 1181 1182 addr = pmap_mapdev_attr(phys_addr, size, attr); 1183 if (addr == NULL) 1184 return (NULL); 1185 vmmap_add(addr, size); 1186 1187 return (addr); 1188 } 1189 #endif 1190 1191 void 1192 iounmap(void *addr) 1193 { 1194 struct vmmap *vmmap; 1195 1196 vmmap = vmmap_remove(addr); 1197 if (vmmap == NULL) 1198 return; 1199 #if defined(__i386__) || defined(__amd64__) 1200 pmap_unmapdev((vm_offset_t)addr, vmmap->vm_size); 1201 #endif 1202 kfree(vmmap); 1203 } 1204 1205 1206 void * 1207 vmap(struct page **pages, unsigned int count, unsigned long flags, int prot) 1208 { 1209 vm_offset_t off; 1210 size_t size; 1211 1212 size = count * PAGE_SIZE; 1213 off = kva_alloc(size); 1214 if (off == 0) 1215 return (NULL); 1216 vmmap_add((void *)off, size); 1217 pmap_qenter(off, pages, count); 1218 1219 return ((void *)off); 1220 } 1221 1222 void 1223 vunmap(void *addr) 1224 { 1225 struct vmmap *vmmap; 1226 1227 vmmap = vmmap_remove(addr); 1228 if (vmmap == NULL) 1229 return; 1230 pmap_qremove((vm_offset_t)addr, vmmap->vm_size / PAGE_SIZE); 1231 kva_free((vm_offset_t)addr, vmmap->vm_size); 1232 kfree(vmmap); 1233 } 1234 1235 char * 1236 kvasprintf(gfp_t gfp, const char *fmt, va_list ap) 1237 { 1238 unsigned int len; 1239 char *p; 1240 va_list aq; 1241 1242 va_copy(aq, ap); 1243 len = vsnprintf(NULL, 0, fmt, aq); 1244 va_end(aq); 1245 1246 p = kmalloc(len + 1, gfp); 1247 if (p != NULL) 1248 vsnprintf(p, len + 1, fmt, ap); 1249 1250 return (p); 1251 } 1252 1253 char * 1254 kasprintf(gfp_t gfp, const char *fmt, ...) 1255 { 1256 va_list ap; 1257 char *p; 1258 1259 va_start(ap, fmt); 1260 p = kvasprintf(gfp, fmt, ap); 1261 va_end(ap); 1262 1263 return (p); 1264 } 1265 1266 static void 1267 linux_timer_callback_wrapper(void *context) 1268 { 1269 struct timer_list *timer; 1270 1271 linux_set_current(curthread); 1272 1273 timer = context; 1274 timer->function(timer->data); 1275 } 1276 1277 void 1278 mod_timer(struct timer_list *timer, unsigned long expires) 1279 { 1280 1281 timer->expires = expires; 1282 callout_reset(&timer->timer_callout, 1283 linux_timer_jiffies_until(expires), 1284 &linux_timer_callback_wrapper, timer); 1285 } 1286 1287 void 1288 add_timer(struct timer_list *timer) 1289 { 1290 1291 callout_reset(&timer->timer_callout, 1292 linux_timer_jiffies_until(timer->expires), 1293 &linux_timer_callback_wrapper, timer); 1294 } 1295 1296 void 1297 add_timer_on(struct timer_list *timer, int cpu) 1298 { 1299 1300 callout_reset_on(&timer->timer_callout, 1301 linux_timer_jiffies_until(timer->expires), 1302 &linux_timer_callback_wrapper, timer, cpu); 1303 } 1304 1305 static void 1306 linux_timer_init(void *arg) 1307 { 1308 1309 /* 1310 * Compute an internal HZ value which can divide 2**32 to 1311 * avoid timer rounding problems when the tick value wraps 1312 * around 2**32: 1313 */ 1314 linux_timer_hz_mask = 1; 1315 while (linux_timer_hz_mask < (unsigned long)hz) 1316 linux_timer_hz_mask *= 2; 1317 linux_timer_hz_mask--; 1318 } 1319 SYSINIT(linux_timer, SI_SUB_DRIVERS, SI_ORDER_FIRST, linux_timer_init, NULL); 1320 1321 void 1322 linux_complete_common(struct completion *c, int all) 1323 { 1324 int wakeup_swapper; 1325 1326 sleepq_lock(c); 1327 c->done++; 1328 if (all) 1329 wakeup_swapper = sleepq_broadcast(c, SLEEPQ_SLEEP, 0, 0); 1330 else 1331 wakeup_swapper = sleepq_signal(c, SLEEPQ_SLEEP, 0, 0); 1332 sleepq_release(c); 1333 if (wakeup_swapper) 1334 kick_proc0(); 1335 } 1336 1337 /* 1338 * Indefinite wait for done != 0 with or without signals. 1339 */ 1340 long 1341 linux_wait_for_common(struct completion *c, int flags) 1342 { 1343 long error; 1344 1345 if (SCHEDULER_STOPPED()) 1346 return (0); 1347 1348 DROP_GIANT(); 1349 1350 if (flags != 0) 1351 flags = SLEEPQ_INTERRUPTIBLE | SLEEPQ_SLEEP; 1352 else 1353 flags = SLEEPQ_SLEEP; 1354 error = 0; 1355 for (;;) { 1356 sleepq_lock(c); 1357 if (c->done) 1358 break; 1359 sleepq_add(c, NULL, "completion", flags, 0); 1360 if (flags & SLEEPQ_INTERRUPTIBLE) { 1361 if (sleepq_wait_sig(c, 0) != 0) { 1362 error = -ERESTARTSYS; 1363 goto intr; 1364 } 1365 } else 1366 sleepq_wait(c, 0); 1367 } 1368 c->done--; 1369 sleepq_release(c); 1370 1371 intr: 1372 PICKUP_GIANT(); 1373 1374 return (error); 1375 } 1376 1377 /* 1378 * Time limited wait for done != 0 with or without signals. 1379 */ 1380 long 1381 linux_wait_for_timeout_common(struct completion *c, long timeout, int flags) 1382 { 1383 long end = jiffies + timeout, error; 1384 int ret; 1385 1386 if (SCHEDULER_STOPPED()) 1387 return (0); 1388 1389 DROP_GIANT(); 1390 1391 if (flags != 0) 1392 flags = SLEEPQ_INTERRUPTIBLE | SLEEPQ_SLEEP; 1393 else 1394 flags = SLEEPQ_SLEEP; 1395 1396 error = 0; 1397 ret = 0; 1398 for (;;) { 1399 sleepq_lock(c); 1400 if (c->done) 1401 break; 1402 sleepq_add(c, NULL, "completion", flags, 0); 1403 sleepq_set_timeout(c, linux_timer_jiffies_until(end)); 1404 if (flags & SLEEPQ_INTERRUPTIBLE) 1405 ret = sleepq_timedwait_sig(c, 0); 1406 else 1407 ret = sleepq_timedwait(c, 0); 1408 if (ret != 0) { 1409 /* check for timeout or signal */ 1410 if (ret == EWOULDBLOCK) 1411 error = 0; 1412 else 1413 error = -ERESTARTSYS; 1414 goto intr; 1415 } 1416 } 1417 c->done--; 1418 sleepq_release(c); 1419 1420 intr: 1421 PICKUP_GIANT(); 1422 1423 /* return how many jiffies are left */ 1424 return (ret != 0 ? error : linux_timer_jiffies_until(end)); 1425 } 1426 1427 int 1428 linux_try_wait_for_completion(struct completion *c) 1429 { 1430 int isdone; 1431 1432 isdone = 1; 1433 sleepq_lock(c); 1434 if (c->done) 1435 c->done--; 1436 else 1437 isdone = 0; 1438 sleepq_release(c); 1439 return (isdone); 1440 } 1441 1442 int 1443 linux_completion_done(struct completion *c) 1444 { 1445 int isdone; 1446 1447 isdone = 1; 1448 sleepq_lock(c); 1449 if (c->done == 0) 1450 isdone = 0; 1451 sleepq_release(c); 1452 return (isdone); 1453 } 1454 1455 static void 1456 linux_cdev_release(struct kobject *kobj) 1457 { 1458 struct linux_cdev *cdev; 1459 struct kobject *parent; 1460 1461 cdev = container_of(kobj, struct linux_cdev, kobj); 1462 parent = kobj->parent; 1463 if (cdev->cdev) 1464 destroy_dev(cdev->cdev); 1465 kfree(cdev); 1466 kobject_put(parent); 1467 } 1468 1469 static void 1470 linux_cdev_static_release(struct kobject *kobj) 1471 { 1472 struct linux_cdev *cdev; 1473 struct kobject *parent; 1474 1475 cdev = container_of(kobj, struct linux_cdev, kobj); 1476 parent = kobj->parent; 1477 if (cdev->cdev) 1478 destroy_dev(cdev->cdev); 1479 kobject_put(parent); 1480 } 1481 1482 const struct kobj_type linux_cdev_ktype = { 1483 .release = linux_cdev_release, 1484 }; 1485 1486 const struct kobj_type linux_cdev_static_ktype = { 1487 .release = linux_cdev_static_release, 1488 }; 1489 1490 static void 1491 linux_handle_ifnet_link_event(void *arg, struct ifnet *ifp, int linkstate) 1492 { 1493 struct notifier_block *nb; 1494 1495 nb = arg; 1496 if (linkstate == LINK_STATE_UP) 1497 nb->notifier_call(nb, NETDEV_UP, ifp); 1498 else 1499 nb->notifier_call(nb, NETDEV_DOWN, ifp); 1500 } 1501 1502 static void 1503 linux_handle_ifnet_arrival_event(void *arg, struct ifnet *ifp) 1504 { 1505 struct notifier_block *nb; 1506 1507 nb = arg; 1508 nb->notifier_call(nb, NETDEV_REGISTER, ifp); 1509 } 1510 1511 static void 1512 linux_handle_ifnet_departure_event(void *arg, struct ifnet *ifp) 1513 { 1514 struct notifier_block *nb; 1515 1516 nb = arg; 1517 nb->notifier_call(nb, NETDEV_UNREGISTER, ifp); 1518 } 1519 1520 static void 1521 linux_handle_iflladdr_event(void *arg, struct ifnet *ifp) 1522 { 1523 struct notifier_block *nb; 1524 1525 nb = arg; 1526 nb->notifier_call(nb, NETDEV_CHANGEADDR, ifp); 1527 } 1528 1529 static void 1530 linux_handle_ifaddr_event(void *arg, struct ifnet *ifp) 1531 { 1532 struct notifier_block *nb; 1533 1534 nb = arg; 1535 nb->notifier_call(nb, NETDEV_CHANGEIFADDR, ifp); 1536 } 1537 1538 int 1539 register_netdevice_notifier(struct notifier_block *nb) 1540 { 1541 1542 nb->tags[NETDEV_UP] = EVENTHANDLER_REGISTER( 1543 ifnet_link_event, linux_handle_ifnet_link_event, nb, 0); 1544 nb->tags[NETDEV_REGISTER] = EVENTHANDLER_REGISTER( 1545 ifnet_arrival_event, linux_handle_ifnet_arrival_event, nb, 0); 1546 nb->tags[NETDEV_UNREGISTER] = EVENTHANDLER_REGISTER( 1547 ifnet_departure_event, linux_handle_ifnet_departure_event, nb, 0); 1548 nb->tags[NETDEV_CHANGEADDR] = EVENTHANDLER_REGISTER( 1549 iflladdr_event, linux_handle_iflladdr_event, nb, 0); 1550 1551 return (0); 1552 } 1553 1554 int 1555 register_inetaddr_notifier(struct notifier_block *nb) 1556 { 1557 1558 nb->tags[NETDEV_CHANGEIFADDR] = EVENTHANDLER_REGISTER( 1559 ifaddr_event, linux_handle_ifaddr_event, nb, 0); 1560 return (0); 1561 } 1562 1563 int 1564 unregister_netdevice_notifier(struct notifier_block *nb) 1565 { 1566 1567 EVENTHANDLER_DEREGISTER(ifnet_link_event, 1568 nb->tags[NETDEV_UP]); 1569 EVENTHANDLER_DEREGISTER(ifnet_arrival_event, 1570 nb->tags[NETDEV_REGISTER]); 1571 EVENTHANDLER_DEREGISTER(ifnet_departure_event, 1572 nb->tags[NETDEV_UNREGISTER]); 1573 EVENTHANDLER_DEREGISTER(iflladdr_event, 1574 nb->tags[NETDEV_CHANGEADDR]); 1575 1576 return (0); 1577 } 1578 1579 int 1580 unregister_inetaddr_notifier(struct notifier_block *nb) 1581 { 1582 1583 EVENTHANDLER_DEREGISTER(ifaddr_event, 1584 nb->tags[NETDEV_CHANGEIFADDR]); 1585 1586 return (0); 1587 } 1588 1589 struct list_sort_thunk { 1590 int (*cmp)(void *, struct list_head *, struct list_head *); 1591 void *priv; 1592 }; 1593 1594 static inline int 1595 linux_le_cmp(void *priv, const void *d1, const void *d2) 1596 { 1597 struct list_head *le1, *le2; 1598 struct list_sort_thunk *thunk; 1599 1600 thunk = priv; 1601 le1 = *(__DECONST(struct list_head **, d1)); 1602 le2 = *(__DECONST(struct list_head **, d2)); 1603 return ((thunk->cmp)(thunk->priv, le1, le2)); 1604 } 1605 1606 void 1607 list_sort(void *priv, struct list_head *head, int (*cmp)(void *priv, 1608 struct list_head *a, struct list_head *b)) 1609 { 1610 struct list_sort_thunk thunk; 1611 struct list_head **ar, *le; 1612 size_t count, i; 1613 1614 count = 0; 1615 list_for_each(le, head) 1616 count++; 1617 ar = malloc(sizeof(struct list_head *) * count, M_KMALLOC, M_WAITOK); 1618 i = 0; 1619 list_for_each(le, head) 1620 ar[i++] = le; 1621 thunk.cmp = cmp; 1622 thunk.priv = priv; 1623 qsort_r(ar, count, sizeof(struct list_head *), &thunk, linux_le_cmp); 1624 INIT_LIST_HEAD(head); 1625 for (i = 0; i < count; i++) 1626 list_add_tail(ar[i], head); 1627 free(ar, M_KMALLOC); 1628 } 1629 1630 void 1631 linux_irq_handler(void *ent) 1632 { 1633 struct irq_ent *irqe; 1634 1635 linux_set_current(curthread); 1636 1637 irqe = ent; 1638 irqe->handler(irqe->irq, irqe->arg); 1639 } 1640 1641 #if defined(__i386__) || defined(__amd64__) 1642 int 1643 linux_wbinvd_on_all_cpus(void) 1644 { 1645 1646 pmap_invalidate_cache(); 1647 return (0); 1648 } 1649 #endif 1650 1651 int 1652 linux_on_each_cpu(void callback(void *), void *data) 1653 { 1654 1655 smp_rendezvous(smp_no_rendezvous_barrier, callback, 1656 smp_no_rendezvous_barrier, data); 1657 return (0); 1658 } 1659 1660 struct linux_cdev * 1661 linux_find_cdev(const char *name, unsigned major, unsigned minor) 1662 { 1663 int unit = MKDEV(major, minor); 1664 struct cdev *cdev; 1665 1666 dev_lock(); 1667 LIST_FOREACH(cdev, &linuxcdevsw.d_devs, si_list) { 1668 struct linux_cdev *ldev = cdev->si_drv1; 1669 if (dev2unit(cdev) == unit && 1670 strcmp(kobject_name(&ldev->kobj), name) == 0) { 1671 break; 1672 } 1673 } 1674 dev_unlock(); 1675 1676 return (cdev != NULL ? cdev->si_drv1 : NULL); 1677 } 1678 1679 int 1680 __register_chrdev(unsigned int major, unsigned int baseminor, 1681 unsigned int count, const char *name, 1682 const struct file_operations *fops) 1683 { 1684 struct linux_cdev *cdev; 1685 int ret = 0; 1686 int i; 1687 1688 for (i = baseminor; i < baseminor + count; i++) { 1689 cdev = cdev_alloc(); 1690 cdev_init(cdev, fops); 1691 kobject_set_name(&cdev->kobj, name); 1692 1693 ret = cdev_add(cdev, makedev(major, i), 1); 1694 if (ret != 0) 1695 break; 1696 } 1697 return (ret); 1698 } 1699 1700 int 1701 __register_chrdev_p(unsigned int major, unsigned int baseminor, 1702 unsigned int count, const char *name, 1703 const struct file_operations *fops, uid_t uid, 1704 gid_t gid, int mode) 1705 { 1706 struct linux_cdev *cdev; 1707 int ret = 0; 1708 int i; 1709 1710 for (i = baseminor; i < baseminor + count; i++) { 1711 cdev = cdev_alloc(); 1712 cdev_init(cdev, fops); 1713 kobject_set_name(&cdev->kobj, name); 1714 1715 ret = cdev_add_ext(cdev, makedev(major, i), uid, gid, mode); 1716 if (ret != 0) 1717 break; 1718 } 1719 return (ret); 1720 } 1721 1722 void 1723 __unregister_chrdev(unsigned int major, unsigned int baseminor, 1724 unsigned int count, const char *name) 1725 { 1726 struct linux_cdev *cdevp; 1727 int i; 1728 1729 for (i = baseminor; i < baseminor + count; i++) { 1730 cdevp = linux_find_cdev(name, major, i); 1731 if (cdevp != NULL) 1732 cdev_del(cdevp); 1733 } 1734 } 1735 1736 #if defined(__i386__) || defined(__amd64__) 1737 bool linux_cpu_has_clflush; 1738 #endif 1739 1740 static void 1741 linux_compat_init(void *arg) 1742 { 1743 struct sysctl_oid *rootoid; 1744 int i; 1745 1746 #if defined(__i386__) || defined(__amd64__) 1747 linux_cpu_has_clflush = (cpu_feature & CPUID_CLFSH); 1748 #endif 1749 rw_init(&linux_vma_lock, "lkpi-vma-lock"); 1750 1751 rootoid = SYSCTL_ADD_ROOT_NODE(NULL, 1752 OID_AUTO, "sys", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "sys"); 1753 kobject_init(&linux_class_root, &linux_class_ktype); 1754 kobject_set_name(&linux_class_root, "class"); 1755 linux_class_root.oidp = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(rootoid), 1756 OID_AUTO, "class", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "class"); 1757 kobject_init(&linux_root_device.kobj, &linux_dev_ktype); 1758 kobject_set_name(&linux_root_device.kobj, "device"); 1759 linux_root_device.kobj.oidp = SYSCTL_ADD_NODE(NULL, 1760 SYSCTL_CHILDREN(rootoid), OID_AUTO, "device", CTLFLAG_RD, NULL, 1761 "device"); 1762 linux_root_device.bsddev = root_bus; 1763 linux_class_misc.name = "misc"; 1764 class_register(&linux_class_misc); 1765 INIT_LIST_HEAD(&pci_drivers); 1766 INIT_LIST_HEAD(&pci_devices); 1767 spin_lock_init(&pci_lock); 1768 mtx_init(&vmmaplock, "IO Map lock", NULL, MTX_DEF); 1769 for (i = 0; i < VMMAP_HASH_SIZE; i++) 1770 LIST_INIT(&vmmaphead[i]); 1771 } 1772 SYSINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_init, NULL); 1773 1774 static void 1775 linux_compat_uninit(void *arg) 1776 { 1777 linux_kobject_kfree_name(&linux_class_root); 1778 linux_kobject_kfree_name(&linux_root_device.kobj); 1779 linux_kobject_kfree_name(&linux_class_misc.kobj); 1780 1781 rw_destroy(&linux_vma_lock); 1782 } 1783 SYSUNINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_uninit, NULL); 1784 1785 /* 1786 * NOTE: Linux frequently uses "unsigned long" for pointer to integer 1787 * conversion and vice versa, where in FreeBSD "uintptr_t" would be 1788 * used. Assert these types have the same size, else some parts of the 1789 * LinuxKPI may not work like expected: 1790 */ 1791 CTASSERT(sizeof(unsigned long) == sizeof(uintptr_t)); 1792