1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2021 Mellanox Technologies, Ltd. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice unmodified, this list of conditions, and the following 13 * disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 #include "opt_global.h" 32 #include "opt_stack.h" 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/malloc.h> 37 #include <sys/kernel.h> 38 #include <sys/sysctl.h> 39 #include <sys/proc.h> 40 #include <sys/sglist.h> 41 #include <sys/sleepqueue.h> 42 #include <sys/refcount.h> 43 #include <sys/lock.h> 44 #include <sys/mutex.h> 45 #include <sys/bus.h> 46 #include <sys/eventhandler.h> 47 #include <sys/fcntl.h> 48 #include <sys/file.h> 49 #include <sys/filio.h> 50 #include <sys/rwlock.h> 51 #include <sys/mman.h> 52 #include <sys/stack.h> 53 #include <sys/stdarg.h> 54 #include <sys/syscall.h> 55 #include <sys/sysent.h> 56 #include <sys/time.h> 57 #include <sys/user.h> 58 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #include <vm/vm_object.h> 62 #include <vm/vm_page.h> 63 #include <vm/vm_pager.h> 64 #include <vm/vm_radix.h> 65 66 #if defined(__i386__) || defined(__amd64__) 67 #include <machine/cputypes.h> 68 #include <machine/md_var.h> 69 #endif 70 71 #include <linux/kobject.h> 72 #include <linux/cpu.h> 73 #include <linux/device.h> 74 #include <linux/slab.h> 75 #include <linux/module.h> 76 #include <linux/moduleparam.h> 77 #include <linux/cdev.h> 78 #include <linux/file.h> 79 #include <linux/fs.h> 80 #include <linux/sysfs.h> 81 #include <linux/mm.h> 82 #include <linux/io.h> 83 #include <linux/vmalloc.h> 84 #include <linux/netdevice.h> 85 #include <linux/timer.h> 86 #include <linux/interrupt.h> 87 #include <linux/uaccess.h> 88 #include <linux/utsname.h> 89 #include <linux/list.h> 90 #include <linux/kthread.h> 91 #include <linux/kernel.h> 92 #include <linux/compat.h> 93 #include <linux/io-mapping.h> 94 #include <linux/poll.h> 95 #include <linux/smp.h> 96 #include <linux/wait_bit.h> 97 #include <linux/rcupdate.h> 98 #include <linux/interval_tree.h> 99 #include <linux/interval_tree_generic.h> 100 #include <linux/printk.h> 101 #include <linux/seq_file.h> 102 #include <linux/uuid.h> 103 104 #if defined(__i386__) || defined(__amd64__) 105 #include <asm/smp.h> 106 #include <asm/processor.h> 107 #endif 108 109 #include <xen/xen.h> 110 #ifdef XENHVM 111 #undef xen_pv_domain 112 #undef xen_initial_domain 113 /* xen/xen-os.h redefines __must_check */ 114 #undef __must_check 115 #include <xen/xen-os.h> 116 #endif 117 118 SYSCTL_NODE(_compat, OID_AUTO, linuxkpi, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 119 "LinuxKPI parameters"); 120 121 int linuxkpi_debug; 122 SYSCTL_INT(_compat_linuxkpi, OID_AUTO, debug, CTLFLAG_RWTUN, 123 &linuxkpi_debug, 0, "Set to enable pr_debug() prints. Clear to disable."); 124 125 int linuxkpi_rcu_debug; 126 SYSCTL_INT(_compat_linuxkpi, OID_AUTO, rcu_debug, CTLFLAG_RWTUN, 127 &linuxkpi_rcu_debug, 0, "Set to enable RCU warning. Clear to disable."); 128 129 int linuxkpi_warn_dump_stack = 0; 130 SYSCTL_INT(_compat_linuxkpi, OID_AUTO, warn_dump_stack, CTLFLAG_RWTUN, 131 &linuxkpi_warn_dump_stack, 0, 132 "Set to enable stack traces from WARN_ON(). Clear to disable."); 133 134 static struct timeval lkpi_net_lastlog; 135 static int lkpi_net_curpps; 136 static int lkpi_net_maxpps = 99; 137 SYSCTL_INT(_compat_linuxkpi, OID_AUTO, net_ratelimit, CTLFLAG_RWTUN, 138 &lkpi_net_maxpps, 0, "Limit number of LinuxKPI net messages per second."); 139 140 MALLOC_DEFINE(M_KMALLOC, "lkpikmalloc", "Linux kmalloc compat"); 141 142 #include <linux/rbtree.h> 143 /* Undo Linux compat changes. */ 144 #undef RB_ROOT 145 #undef file 146 #undef cdev 147 #define RB_ROOT(head) (head)->rbh_root 148 149 static void linux_destroy_dev(struct linux_cdev *); 150 static void linux_cdev_deref(struct linux_cdev *ldev); 151 static struct vm_area_struct *linux_cdev_handle_find(void *handle); 152 153 cpumask_t cpu_online_mask; 154 static cpumask_t **static_single_cpu_mask; 155 static cpumask_t *static_single_cpu_mask_lcs; 156 struct kobject linux_class_root; 157 struct device linux_root_device; 158 struct class linux_class_misc; 159 struct list_head pci_drivers; 160 struct list_head pci_devices; 161 spinlock_t pci_lock; 162 struct uts_namespace init_uts_ns; 163 164 unsigned long linux_timer_hz_mask; 165 166 wait_queue_head_t linux_bit_waitq; 167 wait_queue_head_t linux_var_waitq; 168 169 const guid_t guid_null; 170 171 enum system_states system_state = SYSTEM_RUNNING; 172 173 struct task_struct * 174 __lkpi_current(void) 175 { 176 struct thread *td; 177 178 td = curthread; 179 linux_set_current(td); 180 return ((struct task_struct *)td->td_lkpi_task); 181 } 182 183 int 184 panic_cmp(struct rb_node *one, struct rb_node *two) 185 { 186 panic("no cmp"); 187 } 188 189 RB_GENERATE(linux_root, rb_node, __entry, panic_cmp); 190 191 #define START(node) ((node)->start) 192 #define LAST(node) ((node)->last) 193 194 INTERVAL_TREE_DEFINE(struct interval_tree_node, rb, unsigned long,, START, 195 LAST,, lkpi_interval_tree) 196 197 static void 198 linux_device_release(struct device *dev) 199 { 200 pr_debug("linux_device_release: %s\n", dev_name(dev)); 201 kfree(dev); 202 } 203 204 static ssize_t 205 linux_class_show(struct kobject *kobj, struct attribute *attr, char *buf) 206 { 207 struct class_attribute *dattr; 208 ssize_t error; 209 210 dattr = container_of(attr, struct class_attribute, attr); 211 error = -EIO; 212 if (dattr->show) 213 error = dattr->show(container_of(kobj, struct class, kobj), 214 dattr, buf); 215 return (error); 216 } 217 218 static ssize_t 219 linux_class_store(struct kobject *kobj, struct attribute *attr, const char *buf, 220 size_t count) 221 { 222 struct class_attribute *dattr; 223 ssize_t error; 224 225 dattr = container_of(attr, struct class_attribute, attr); 226 error = -EIO; 227 if (dattr->store) 228 error = dattr->store(container_of(kobj, struct class, kobj), 229 dattr, buf, count); 230 return (error); 231 } 232 233 static void 234 linux_class_release(struct kobject *kobj) 235 { 236 struct class *class; 237 238 class = container_of(kobj, struct class, kobj); 239 if (class->class_release) 240 class->class_release(class); 241 } 242 243 static const struct sysfs_ops linux_class_sysfs = { 244 .show = linux_class_show, 245 .store = linux_class_store, 246 }; 247 248 const struct kobj_type linux_class_ktype = { 249 .release = linux_class_release, 250 .sysfs_ops = &linux_class_sysfs 251 }; 252 253 static void 254 linux_dev_release(struct kobject *kobj) 255 { 256 struct device *dev; 257 258 dev = container_of(kobj, struct device, kobj); 259 /* This is the precedence defined by linux. */ 260 if (dev->release) 261 dev->release(dev); 262 else if (dev->class && dev->class->dev_release) 263 dev->class->dev_release(dev); 264 } 265 266 static ssize_t 267 linux_dev_show(struct kobject *kobj, struct attribute *attr, char *buf) 268 { 269 struct device_attribute *dattr; 270 ssize_t error; 271 272 dattr = container_of(attr, struct device_attribute, attr); 273 error = -EIO; 274 if (dattr->show) 275 error = dattr->show(container_of(kobj, struct device, kobj), 276 dattr, buf); 277 return (error); 278 } 279 280 static ssize_t 281 linux_dev_store(struct kobject *kobj, struct attribute *attr, const char *buf, 282 size_t count) 283 { 284 struct device_attribute *dattr; 285 ssize_t error; 286 287 dattr = container_of(attr, struct device_attribute, attr); 288 error = -EIO; 289 if (dattr->store) 290 error = dattr->store(container_of(kobj, struct device, kobj), 291 dattr, buf, count); 292 return (error); 293 } 294 295 static const struct sysfs_ops linux_dev_sysfs = { 296 .show = linux_dev_show, 297 .store = linux_dev_store, 298 }; 299 300 const struct kobj_type linux_dev_ktype = { 301 .release = linux_dev_release, 302 .sysfs_ops = &linux_dev_sysfs 303 }; 304 305 struct device * 306 device_create(struct class *class, struct device *parent, dev_t devt, 307 void *drvdata, const char *fmt, ...) 308 { 309 struct device *dev; 310 va_list args; 311 312 dev = kzalloc(sizeof(*dev), M_WAITOK); 313 dev->parent = parent; 314 dev->class = class; 315 dev->devt = devt; 316 dev->driver_data = drvdata; 317 dev->release = linux_device_release; 318 va_start(args, fmt); 319 kobject_set_name_vargs(&dev->kobj, fmt, args); 320 va_end(args); 321 device_register(dev); 322 323 return (dev); 324 } 325 326 struct device * 327 device_create_groups_vargs(struct class *class, struct device *parent, 328 dev_t devt, void *drvdata, const struct attribute_group **groups, 329 const char *fmt, va_list args) 330 { 331 struct device *dev = NULL; 332 int retval = -ENODEV; 333 334 if (class == NULL || IS_ERR(class)) 335 goto error; 336 337 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 338 if (!dev) { 339 retval = -ENOMEM; 340 goto error; 341 } 342 343 dev->devt = devt; 344 dev->class = class; 345 dev->parent = parent; 346 dev->groups = groups; 347 dev->release = device_create_release; 348 /* device_initialize() needs the class and parent to be set */ 349 device_initialize(dev); 350 dev_set_drvdata(dev, drvdata); 351 352 retval = kobject_set_name_vargs(&dev->kobj, fmt, args); 353 if (retval) 354 goto error; 355 356 retval = device_add(dev); 357 if (retval) 358 goto error; 359 360 return dev; 361 362 error: 363 put_device(dev); 364 return ERR_PTR(retval); 365 } 366 367 struct class * 368 lkpi_class_create(const char *name) 369 { 370 struct class *class; 371 int error; 372 373 class = kzalloc(sizeof(*class), M_WAITOK); 374 class->name = name; 375 class->class_release = linux_class_kfree; 376 error = class_register(class); 377 if (error) { 378 kfree(class); 379 return (NULL); 380 } 381 382 return (class); 383 } 384 385 static void 386 linux_kq_lock(void *arg) 387 { 388 spinlock_t *s = arg; 389 390 spin_lock(s); 391 } 392 static void 393 linux_kq_unlock(void *arg) 394 { 395 spinlock_t *s = arg; 396 397 spin_unlock(s); 398 } 399 400 static void 401 linux_kq_assert_lock(void *arg, int what) 402 { 403 #ifdef INVARIANTS 404 spinlock_t *s = arg; 405 406 if (what == LA_LOCKED) 407 mtx_assert(s, MA_OWNED); 408 else 409 mtx_assert(s, MA_NOTOWNED); 410 #endif 411 } 412 413 static void 414 linux_file_kqfilter_poll(struct linux_file *, int); 415 416 struct linux_file * 417 linux_file_alloc(void) 418 { 419 struct linux_file *filp; 420 421 filp = kzalloc(sizeof(*filp), GFP_KERNEL); 422 423 /* set initial refcount */ 424 filp->f_count = 1; 425 426 /* setup fields needed by kqueue support */ 427 spin_lock_init(&filp->f_kqlock); 428 knlist_init(&filp->f_selinfo.si_note, &filp->f_kqlock, 429 linux_kq_lock, linux_kq_unlock, linux_kq_assert_lock); 430 431 return (filp); 432 } 433 434 void 435 linux_file_free(struct linux_file *filp) 436 { 437 if (filp->_file == NULL) { 438 if (filp->f_op != NULL && filp->f_op->release != NULL) 439 filp->f_op->release(filp->f_vnode, filp); 440 if (filp->f_shmem != NULL) 441 vm_object_deallocate(filp->f_shmem); 442 kfree_rcu(filp, rcu); 443 } else { 444 /* 445 * The close method of the character device or file 446 * will free the linux_file structure: 447 */ 448 _fdrop(filp->_file, curthread); 449 } 450 } 451 452 struct linux_cdev * 453 cdev_alloc(void) 454 { 455 struct linux_cdev *cdev; 456 457 cdev = kzalloc(sizeof(struct linux_cdev), M_WAITOK); 458 kobject_init(&cdev->kobj, &linux_cdev_ktype); 459 cdev->refs = 1; 460 return (cdev); 461 } 462 463 static int 464 linux_cdev_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, 465 vm_page_t *mres) 466 { 467 struct vm_area_struct *vmap; 468 469 vmap = linux_cdev_handle_find(vm_obj->handle); 470 471 MPASS(vmap != NULL); 472 MPASS(vmap->vm_private_data == vm_obj->handle); 473 474 if (likely(vmap->vm_ops != NULL && offset < vmap->vm_len)) { 475 vm_paddr_t paddr = IDX_TO_OFF(vmap->vm_pfn) + offset; 476 vm_page_t page; 477 478 if (((*mres)->flags & PG_FICTITIOUS) != 0) { 479 /* 480 * If the passed in result page is a fake 481 * page, update it with the new physical 482 * address. 483 */ 484 page = *mres; 485 vm_page_updatefake(page, paddr, vm_obj->memattr); 486 } else { 487 /* 488 * Replace the passed in "mres" page with our 489 * own fake page and free up the all of the 490 * original pages. 491 */ 492 VM_OBJECT_WUNLOCK(vm_obj); 493 page = vm_page_getfake(paddr, vm_obj->memattr); 494 VM_OBJECT_WLOCK(vm_obj); 495 496 vm_page_replace(page, vm_obj, (*mres)->pindex, *mres); 497 *mres = page; 498 } 499 vm_page_valid(page); 500 return (VM_PAGER_OK); 501 } 502 return (VM_PAGER_FAIL); 503 } 504 505 static int 506 linux_cdev_pager_populate(vm_object_t vm_obj, vm_pindex_t pidx, int fault_type, 507 vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last) 508 { 509 struct vm_area_struct *vmap; 510 int err; 511 512 /* get VM area structure */ 513 vmap = linux_cdev_handle_find(vm_obj->handle); 514 MPASS(vmap != NULL); 515 MPASS(vmap->vm_private_data == vm_obj->handle); 516 517 VM_OBJECT_WUNLOCK(vm_obj); 518 519 linux_set_current(curthread); 520 521 down_write(&vmap->vm_mm->mmap_sem); 522 if (unlikely(vmap->vm_ops == NULL)) { 523 err = VM_FAULT_SIGBUS; 524 } else { 525 struct vm_fault vmf; 526 527 /* fill out VM fault structure */ 528 vmf.virtual_address = (void *)(uintptr_t)IDX_TO_OFF(pidx); 529 vmf.flags = (fault_type & VM_PROT_WRITE) ? FAULT_FLAG_WRITE : 0; 530 vmf.pgoff = 0; 531 vmf.page = NULL; 532 vmf.vma = vmap; 533 534 vmap->vm_pfn_count = 0; 535 vmap->vm_pfn_pcount = &vmap->vm_pfn_count; 536 vmap->vm_obj = vm_obj; 537 538 err = vmap->vm_ops->fault(&vmf); 539 540 while (vmap->vm_pfn_count == 0 && err == VM_FAULT_NOPAGE) { 541 kern_yield(PRI_USER); 542 err = vmap->vm_ops->fault(&vmf); 543 } 544 } 545 546 /* translate return code */ 547 switch (err) { 548 case VM_FAULT_OOM: 549 err = VM_PAGER_AGAIN; 550 break; 551 case VM_FAULT_SIGBUS: 552 err = VM_PAGER_BAD; 553 break; 554 case VM_FAULT_NOPAGE: 555 /* 556 * By contract the fault handler will return having 557 * busied all the pages itself. If pidx is already 558 * found in the object, it will simply xbusy the first 559 * page and return with vm_pfn_count set to 1. 560 */ 561 *first = vmap->vm_pfn_first; 562 *last = *first + vmap->vm_pfn_count - 1; 563 err = VM_PAGER_OK; 564 break; 565 default: 566 err = VM_PAGER_ERROR; 567 break; 568 } 569 up_write(&vmap->vm_mm->mmap_sem); 570 VM_OBJECT_WLOCK(vm_obj); 571 return (err); 572 } 573 574 static struct rwlock linux_vma_lock; 575 static TAILQ_HEAD(, vm_area_struct) linux_vma_head = 576 TAILQ_HEAD_INITIALIZER(linux_vma_head); 577 578 static void 579 linux_cdev_handle_free(struct vm_area_struct *vmap) 580 { 581 /* Drop reference on vm_file */ 582 if (vmap->vm_file != NULL) 583 fput(vmap->vm_file); 584 585 /* Drop reference on mm_struct */ 586 mmput(vmap->vm_mm); 587 588 kfree(vmap); 589 } 590 591 static void 592 linux_cdev_handle_remove(struct vm_area_struct *vmap) 593 { 594 rw_wlock(&linux_vma_lock); 595 TAILQ_REMOVE(&linux_vma_head, vmap, vm_entry); 596 rw_wunlock(&linux_vma_lock); 597 } 598 599 static struct vm_area_struct * 600 linux_cdev_handle_find(void *handle) 601 { 602 struct vm_area_struct *vmap; 603 604 rw_rlock(&linux_vma_lock); 605 TAILQ_FOREACH(vmap, &linux_vma_head, vm_entry) { 606 if (vmap->vm_private_data == handle) 607 break; 608 } 609 rw_runlock(&linux_vma_lock); 610 return (vmap); 611 } 612 613 static int 614 linux_cdev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 615 vm_ooffset_t foff, struct ucred *cred, u_short *color) 616 { 617 618 MPASS(linux_cdev_handle_find(handle) != NULL); 619 *color = 0; 620 return (0); 621 } 622 623 static void 624 linux_cdev_pager_dtor(void *handle) 625 { 626 const struct vm_operations_struct *vm_ops; 627 struct vm_area_struct *vmap; 628 629 vmap = linux_cdev_handle_find(handle); 630 MPASS(vmap != NULL); 631 632 /* 633 * Remove handle before calling close operation to prevent 634 * other threads from reusing the handle pointer. 635 */ 636 linux_cdev_handle_remove(vmap); 637 638 down_write(&vmap->vm_mm->mmap_sem); 639 vm_ops = vmap->vm_ops; 640 if (likely(vm_ops != NULL)) 641 vm_ops->close(vmap); 642 up_write(&vmap->vm_mm->mmap_sem); 643 644 linux_cdev_handle_free(vmap); 645 } 646 647 static struct cdev_pager_ops linux_cdev_pager_ops[2] = { 648 { 649 /* OBJT_MGTDEVICE */ 650 .cdev_pg_populate = linux_cdev_pager_populate, 651 .cdev_pg_ctor = linux_cdev_pager_ctor, 652 .cdev_pg_dtor = linux_cdev_pager_dtor 653 }, 654 { 655 /* OBJT_DEVICE */ 656 .cdev_pg_fault = linux_cdev_pager_fault, 657 .cdev_pg_ctor = linux_cdev_pager_ctor, 658 .cdev_pg_dtor = linux_cdev_pager_dtor 659 }, 660 }; 661 662 int 663 zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, 664 unsigned long size) 665 { 666 struct pctrie_iter pages; 667 vm_object_t obj; 668 vm_page_t m; 669 670 obj = vma->vm_obj; 671 if (obj == NULL || (obj->flags & OBJ_UNMANAGED) != 0) 672 return (-ENOTSUP); 673 VM_OBJECT_RLOCK(obj); 674 vm_page_iter_limit_init(&pages, obj, OFF_TO_IDX(address + size)); 675 VM_RADIX_FOREACH_FROM(m, &pages, OFF_TO_IDX(address)) 676 pmap_remove_all(m); 677 VM_OBJECT_RUNLOCK(obj); 678 return (0); 679 } 680 681 void 682 vma_set_file(struct vm_area_struct *vma, struct linux_file *file) 683 { 684 struct linux_file *tmp; 685 686 /* Changing an anonymous vma with this is illegal */ 687 get_file(file); 688 tmp = vma->vm_file; 689 vma->vm_file = file; 690 fput(tmp); 691 } 692 693 static struct file_operations dummy_ldev_ops = { 694 /* XXXKIB */ 695 }; 696 697 static struct linux_cdev dummy_ldev = { 698 .ops = &dummy_ldev_ops, 699 }; 700 701 #define LDEV_SI_DTR 0x0001 702 #define LDEV_SI_REF 0x0002 703 704 static void 705 linux_get_fop(struct linux_file *filp, const struct file_operations **fop, 706 struct linux_cdev **dev) 707 { 708 struct linux_cdev *ldev; 709 u_int siref; 710 711 ldev = filp->f_cdev; 712 *fop = filp->f_op; 713 if (ldev != NULL) { 714 if (ldev->kobj.ktype == &linux_cdev_static_ktype) { 715 refcount_acquire(&ldev->refs); 716 } else { 717 for (siref = ldev->siref;;) { 718 if ((siref & LDEV_SI_DTR) != 0) { 719 ldev = &dummy_ldev; 720 *fop = ldev->ops; 721 siref = ldev->siref; 722 MPASS((ldev->siref & LDEV_SI_DTR) == 0); 723 } else if (atomic_fcmpset_int(&ldev->siref, 724 &siref, siref + LDEV_SI_REF)) { 725 break; 726 } 727 } 728 } 729 } 730 *dev = ldev; 731 } 732 733 static void 734 linux_drop_fop(struct linux_cdev *ldev) 735 { 736 737 if (ldev == NULL) 738 return; 739 if (ldev->kobj.ktype == &linux_cdev_static_ktype) { 740 linux_cdev_deref(ldev); 741 } else { 742 MPASS(ldev->kobj.ktype == &linux_cdev_ktype); 743 MPASS((ldev->siref & ~LDEV_SI_DTR) != 0); 744 atomic_subtract_int(&ldev->siref, LDEV_SI_REF); 745 } 746 } 747 748 #define OPW(fp,td,code) ({ \ 749 struct file *__fpop; \ 750 __typeof(code) __retval; \ 751 \ 752 __fpop = (td)->td_fpop; \ 753 (td)->td_fpop = (fp); \ 754 __retval = (code); \ 755 (td)->td_fpop = __fpop; \ 756 __retval; \ 757 }) 758 759 static int 760 linux_dev_fdopen(struct cdev *dev, int fflags, struct thread *td, 761 struct file *file) 762 { 763 struct linux_cdev *ldev; 764 struct linux_file *filp; 765 const struct file_operations *fop; 766 int error; 767 768 ldev = dev->si_drv1; 769 770 filp = linux_file_alloc(); 771 filp->f_dentry = &filp->f_dentry_store; 772 filp->f_op = ldev->ops; 773 filp->f_mode = file->f_flag; 774 filp->f_flags = file->f_flag; 775 filp->f_vnode = file->f_vnode; 776 filp->_file = file; 777 refcount_acquire(&ldev->refs); 778 filp->f_cdev = ldev; 779 780 linux_set_current(td); 781 linux_get_fop(filp, &fop, &ldev); 782 783 if (fop->open != NULL) { 784 error = -fop->open(file->f_vnode, filp); 785 if (error != 0) { 786 linux_drop_fop(ldev); 787 linux_cdev_deref(filp->f_cdev); 788 kfree(filp); 789 return (error); 790 } 791 } 792 793 /* hold on to the vnode - used for fstat() */ 794 vref(filp->f_vnode); 795 796 /* release the file from devfs */ 797 finit(file, filp->f_mode, DTYPE_DEV, filp, &linuxfileops); 798 linux_drop_fop(ldev); 799 return (ENXIO); 800 } 801 802 #define LINUX_IOCTL_MIN_PTR 0x10000UL 803 #define LINUX_IOCTL_MAX_PTR (LINUX_IOCTL_MIN_PTR + IOCPARM_MAX) 804 805 static inline int 806 linux_remap_address(void **uaddr, size_t len) 807 { 808 uintptr_t uaddr_val = (uintptr_t)(*uaddr); 809 810 if (unlikely(uaddr_val >= LINUX_IOCTL_MIN_PTR && 811 uaddr_val < LINUX_IOCTL_MAX_PTR)) { 812 struct task_struct *pts = current; 813 if (pts == NULL) { 814 *uaddr = NULL; 815 return (1); 816 } 817 818 /* compute data offset */ 819 uaddr_val -= LINUX_IOCTL_MIN_PTR; 820 821 /* check that length is within bounds */ 822 if ((len > IOCPARM_MAX) || 823 (uaddr_val + len) > pts->bsd_ioctl_len) { 824 *uaddr = NULL; 825 return (1); 826 } 827 828 /* re-add kernel buffer address */ 829 uaddr_val += (uintptr_t)pts->bsd_ioctl_data; 830 831 /* update address location */ 832 *uaddr = (void *)uaddr_val; 833 return (1); 834 } 835 return (0); 836 } 837 838 int 839 linux_copyin(const void *uaddr, void *kaddr, size_t len) 840 { 841 if (linux_remap_address(__DECONST(void **, &uaddr), len)) { 842 if (uaddr == NULL) 843 return (-EFAULT); 844 memcpy(kaddr, uaddr, len); 845 return (0); 846 } 847 return (-copyin(uaddr, kaddr, len)); 848 } 849 850 int 851 linux_copyout(const void *kaddr, void *uaddr, size_t len) 852 { 853 if (linux_remap_address(&uaddr, len)) { 854 if (uaddr == NULL) 855 return (-EFAULT); 856 memcpy(uaddr, kaddr, len); 857 return (0); 858 } 859 return (-copyout(kaddr, uaddr, len)); 860 } 861 862 size_t 863 linux_clear_user(void *_uaddr, size_t _len) 864 { 865 uint8_t *uaddr = _uaddr; 866 size_t len = _len; 867 868 /* make sure uaddr is aligned before going into the fast loop */ 869 while (((uintptr_t)uaddr & 7) != 0 && len > 7) { 870 if (subyte(uaddr, 0)) 871 return (_len); 872 uaddr++; 873 len--; 874 } 875 876 /* zero 8 bytes at a time */ 877 while (len > 7) { 878 #ifdef __LP64__ 879 if (suword64(uaddr, 0)) 880 return (_len); 881 #else 882 if (suword32(uaddr, 0)) 883 return (_len); 884 if (suword32(uaddr + 4, 0)) 885 return (_len); 886 #endif 887 uaddr += 8; 888 len -= 8; 889 } 890 891 /* zero fill end, if any */ 892 while (len > 0) { 893 if (subyte(uaddr, 0)) 894 return (_len); 895 uaddr++; 896 len--; 897 } 898 return (0); 899 } 900 901 int 902 linux_access_ok(const void *uaddr, size_t len) 903 { 904 uintptr_t saddr; 905 uintptr_t eaddr; 906 907 /* get start and end address */ 908 saddr = (uintptr_t)uaddr; 909 eaddr = (uintptr_t)uaddr + len; 910 911 /* verify addresses are valid for userspace */ 912 return ((saddr == eaddr) || 913 (eaddr > saddr && eaddr <= VM_MAXUSER_ADDRESS)); 914 } 915 916 /* 917 * This function should return either EINTR or ERESTART depending on 918 * the signal type sent to this thread: 919 */ 920 static int 921 linux_get_error(struct task_struct *task, int error) 922 { 923 /* check for signal type interrupt code */ 924 if (error == EINTR || error == ERESTARTSYS || error == ERESTART) { 925 error = -linux_schedule_get_interrupt_value(task); 926 if (error == 0) 927 error = EINTR; 928 } 929 return (error); 930 } 931 932 static int 933 linux_file_ioctl_sub(struct file *fp, struct linux_file *filp, 934 const struct file_operations *fop, u_long cmd, caddr_t data, 935 struct thread *td) 936 { 937 struct task_struct *task = current; 938 unsigned size; 939 int error; 940 bool direct; 941 942 size = IOCPARM_LEN(cmd); 943 /* refer to logic in sys_ioctl() */ 944 direct = false; 945 if (size > 0) { 946 /* 947 * Setup hint for linux_copyin() and linux_copyout(). 948 * 949 * Background: Linux kernel code expects to operate on 950 * userspace addresses, but FreeBSD's kern_ioctl() 951 * will generally provide a kernel address. For the 952 * native process ABI, where we know how to find the 953 * original address, we reach directly into the system 954 * call args to get it. Then, if the Linux driver 955 * copied out to that address, we copy the whole block 956 * back into the kernel buffer allocated by 957 * kern_ioctl() so that kern_ioctl() itself doesn't 958 * clobber the driver's data. 959 * 960 * Otherwise, fall back to the LINUX_IOCTL_MIN_PTR 961 * hack. 962 */ 963 task->bsd_ioctl_data = data; 964 task->bsd_ioctl_len = size; 965 if ((td->td_pflags & TDP_KTHREAD) == 0 && 966 SV_PROC_ABI(td->td_proc) == SV_ABI_FREEBSD && 967 td->td_sa.code == SYS_ioctl) { 968 direct = true; 969 data = (void *)(uintptr_t)td->td_sa.args[2]; 970 } else { 971 data = (void *)LINUX_IOCTL_MIN_PTR; 972 } 973 } else { 974 /* fetch user-space pointer */ 975 data = *(void **)data; 976 } 977 #ifdef COMPAT_FREEBSD32 978 if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) { 979 /* try the compat IOCTL handler first */ 980 if (fop->compat_ioctl != NULL) { 981 error = -OPW(fp, td, fop->compat_ioctl(filp, 982 cmd, (u_long)data)); 983 } else { 984 error = ENOTTY; 985 } 986 987 /* fallback to the regular IOCTL handler, if any */ 988 if (error == ENOTTY && fop->unlocked_ioctl != NULL) { 989 error = -OPW(fp, td, fop->unlocked_ioctl(filp, 990 cmd, (u_long)data)); 991 } 992 } else 993 #endif 994 { 995 if (fop->unlocked_ioctl != NULL) { 996 error = -OPW(fp, td, fop->unlocked_ioctl(filp, 997 cmd, (u_long)data)); 998 } else { 999 error = ENOTTY; 1000 } 1001 } 1002 if (error == 0 && size > 0 && (cmd & IOC_OUT) != 0 && direct) { 1003 void *xdata; 1004 int error1; 1005 1006 /* 1007 * Ensure that the copyout in sys_generic.c copies 1008 * over the data which is possibly modified by the 1009 * driver. A possible error from the copyin() is 1010 * ignored since it is formally possible for the memory 1011 * to become unaccessible in the meantime. Do the copying 1012 * through the intermediate buffer instead of copying 1013 * directly to bsd_ioctl_data, to ensure atomicity of 1014 * the change with respect to the error. 1015 */ 1016 xdata = malloc(size, M_TEMP, M_WAITOK); 1017 error1 = copyin(data, xdata, size); 1018 if (error1 == 0) 1019 memcpy(task->bsd_ioctl_data, xdata, size); 1020 free(xdata, M_TEMP); 1021 } 1022 if (size > 0) { 1023 task->bsd_ioctl_data = NULL; 1024 task->bsd_ioctl_len = 0; 1025 } 1026 if (error == EWOULDBLOCK) { 1027 /* update kqfilter status, if any */ 1028 linux_file_kqfilter_poll(filp, 1029 LINUX_KQ_FLAG_HAS_READ | LINUX_KQ_FLAG_HAS_WRITE); 1030 } else { 1031 error = linux_get_error(task, error); 1032 } 1033 return (error); 1034 } 1035 1036 #define LINUX_POLL_TABLE_NORMAL ((poll_table *)1) 1037 1038 /* 1039 * This function atomically updates the poll wakeup state and returns 1040 * the previous state at the time of update. 1041 */ 1042 static uint8_t 1043 linux_poll_wakeup_state(atomic_t *v, const uint8_t *pstate) 1044 { 1045 int c, old; 1046 1047 c = v->counter; 1048 1049 while ((old = atomic_cmpxchg(v, c, pstate[c])) != c) 1050 c = old; 1051 1052 return (c); 1053 } 1054 1055 static int 1056 linux_poll_wakeup_callback(wait_queue_t *wq, unsigned int wq_state, int flags, void *key) 1057 { 1058 static const uint8_t state[LINUX_FWQ_STATE_MAX] = { 1059 [LINUX_FWQ_STATE_INIT] = LINUX_FWQ_STATE_INIT, /* NOP */ 1060 [LINUX_FWQ_STATE_NOT_READY] = LINUX_FWQ_STATE_NOT_READY, /* NOP */ 1061 [LINUX_FWQ_STATE_QUEUED] = LINUX_FWQ_STATE_READY, 1062 [LINUX_FWQ_STATE_READY] = LINUX_FWQ_STATE_READY, /* NOP */ 1063 }; 1064 struct linux_file *filp = container_of(wq, struct linux_file, f_wait_queue.wq); 1065 1066 switch (linux_poll_wakeup_state(&filp->f_wait_queue.state, state)) { 1067 case LINUX_FWQ_STATE_QUEUED: 1068 linux_poll_wakeup(filp); 1069 return (1); 1070 default: 1071 return (0); 1072 } 1073 } 1074 1075 void 1076 linux_poll_wait(struct linux_file *filp, wait_queue_head_t *wqh, poll_table *p) 1077 { 1078 static const uint8_t state[LINUX_FWQ_STATE_MAX] = { 1079 [LINUX_FWQ_STATE_INIT] = LINUX_FWQ_STATE_NOT_READY, 1080 [LINUX_FWQ_STATE_NOT_READY] = LINUX_FWQ_STATE_NOT_READY, /* NOP */ 1081 [LINUX_FWQ_STATE_QUEUED] = LINUX_FWQ_STATE_QUEUED, /* NOP */ 1082 [LINUX_FWQ_STATE_READY] = LINUX_FWQ_STATE_QUEUED, 1083 }; 1084 1085 /* check if we are called inside the select system call */ 1086 if (p == LINUX_POLL_TABLE_NORMAL) 1087 selrecord(curthread, &filp->f_selinfo); 1088 1089 switch (linux_poll_wakeup_state(&filp->f_wait_queue.state, state)) { 1090 case LINUX_FWQ_STATE_INIT: 1091 /* NOTE: file handles can only belong to one wait-queue */ 1092 filp->f_wait_queue.wqh = wqh; 1093 filp->f_wait_queue.wq.func = &linux_poll_wakeup_callback; 1094 add_wait_queue(wqh, &filp->f_wait_queue.wq); 1095 atomic_set(&filp->f_wait_queue.state, LINUX_FWQ_STATE_QUEUED); 1096 break; 1097 default: 1098 break; 1099 } 1100 } 1101 1102 static void 1103 linux_poll_wait_dequeue(struct linux_file *filp) 1104 { 1105 static const uint8_t state[LINUX_FWQ_STATE_MAX] = { 1106 [LINUX_FWQ_STATE_INIT] = LINUX_FWQ_STATE_INIT, /* NOP */ 1107 [LINUX_FWQ_STATE_NOT_READY] = LINUX_FWQ_STATE_INIT, 1108 [LINUX_FWQ_STATE_QUEUED] = LINUX_FWQ_STATE_INIT, 1109 [LINUX_FWQ_STATE_READY] = LINUX_FWQ_STATE_INIT, 1110 }; 1111 1112 seldrain(&filp->f_selinfo); 1113 1114 switch (linux_poll_wakeup_state(&filp->f_wait_queue.state, state)) { 1115 case LINUX_FWQ_STATE_NOT_READY: 1116 case LINUX_FWQ_STATE_QUEUED: 1117 case LINUX_FWQ_STATE_READY: 1118 remove_wait_queue(filp->f_wait_queue.wqh, &filp->f_wait_queue.wq); 1119 break; 1120 default: 1121 break; 1122 } 1123 } 1124 1125 void 1126 linux_poll_wakeup(struct linux_file *filp) 1127 { 1128 /* this function should be NULL-safe */ 1129 if (filp == NULL) 1130 return; 1131 1132 selwakeup(&filp->f_selinfo); 1133 1134 spin_lock(&filp->f_kqlock); 1135 filp->f_kqflags |= LINUX_KQ_FLAG_NEED_READ | 1136 LINUX_KQ_FLAG_NEED_WRITE; 1137 1138 /* make sure the "knote" gets woken up */ 1139 KNOTE_LOCKED(&filp->f_selinfo.si_note, 1); 1140 spin_unlock(&filp->f_kqlock); 1141 } 1142 1143 static struct linux_file * 1144 __get_file_rcu(struct linux_file **f) 1145 { 1146 struct linux_file *file1, *file2; 1147 1148 file1 = READ_ONCE(*f); 1149 if (file1 == NULL) 1150 return (NULL); 1151 1152 if (!refcount_acquire_if_not_zero( 1153 file1->_file == NULL ? &file1->f_count : &file1->_file->f_count)) 1154 return (ERR_PTR(-EAGAIN)); 1155 1156 file2 = READ_ONCE(*f); 1157 if (file2 == file1) 1158 return (file2); 1159 1160 fput(file1); 1161 return (ERR_PTR(-EAGAIN)); 1162 } 1163 1164 struct linux_file * 1165 linux_get_file_rcu(struct linux_file **f) 1166 { 1167 struct linux_file *file1; 1168 1169 for (;;) { 1170 file1 = __get_file_rcu(f); 1171 if (file1 == NULL) 1172 return (NULL); 1173 1174 if (IS_ERR(file1)) 1175 continue; 1176 1177 return (file1); 1178 } 1179 } 1180 1181 struct linux_file * 1182 get_file_active(struct linux_file **f) 1183 { 1184 struct linux_file *file1; 1185 1186 rcu_read_lock(); 1187 file1 = __get_file_rcu(f); 1188 rcu_read_unlock(); 1189 if (IS_ERR(file1)) 1190 file1 = NULL; 1191 1192 return (file1); 1193 } 1194 1195 static void 1196 linux_file_kqfilter_detach(struct knote *kn) 1197 { 1198 struct linux_file *filp = kn->kn_hook; 1199 1200 spin_lock(&filp->f_kqlock); 1201 knlist_remove(&filp->f_selinfo.si_note, kn, 1); 1202 spin_unlock(&filp->f_kqlock); 1203 } 1204 1205 static int 1206 linux_file_kqfilter_read_event(struct knote *kn, long hint) 1207 { 1208 struct linux_file *filp = kn->kn_hook; 1209 1210 mtx_assert(&filp->f_kqlock, MA_OWNED); 1211 1212 return ((filp->f_kqflags & LINUX_KQ_FLAG_NEED_READ) ? 1 : 0); 1213 } 1214 1215 static int 1216 linux_file_kqfilter_write_event(struct knote *kn, long hint) 1217 { 1218 struct linux_file *filp = kn->kn_hook; 1219 1220 mtx_assert(&filp->f_kqlock, MA_OWNED); 1221 1222 return ((filp->f_kqflags & LINUX_KQ_FLAG_NEED_WRITE) ? 1 : 0); 1223 } 1224 1225 static const struct filterops linux_dev_kqfiltops_read = { 1226 .f_isfd = 1, 1227 .f_detach = linux_file_kqfilter_detach, 1228 .f_event = linux_file_kqfilter_read_event, 1229 .f_copy = knote_triv_copy, 1230 }; 1231 1232 static const struct filterops linux_dev_kqfiltops_write = { 1233 .f_isfd = 1, 1234 .f_detach = linux_file_kqfilter_detach, 1235 .f_event = linux_file_kqfilter_write_event, 1236 .f_copy = knote_triv_copy, 1237 }; 1238 1239 static void 1240 linux_file_kqfilter_poll(struct linux_file *filp, int kqflags) 1241 { 1242 struct thread *td; 1243 const struct file_operations *fop; 1244 struct linux_cdev *ldev; 1245 int temp; 1246 1247 if ((filp->f_kqflags & kqflags) == 0) 1248 return; 1249 1250 td = curthread; 1251 1252 linux_get_fop(filp, &fop, &ldev); 1253 /* get the latest polling state */ 1254 temp = OPW(filp->_file, td, fop->poll(filp, NULL)); 1255 linux_drop_fop(ldev); 1256 1257 spin_lock(&filp->f_kqlock); 1258 /* clear kqflags */ 1259 filp->f_kqflags &= ~(LINUX_KQ_FLAG_NEED_READ | 1260 LINUX_KQ_FLAG_NEED_WRITE); 1261 /* update kqflags */ 1262 if ((temp & (POLLIN | POLLOUT)) != 0) { 1263 if ((temp & POLLIN) != 0) 1264 filp->f_kqflags |= LINUX_KQ_FLAG_NEED_READ; 1265 if ((temp & POLLOUT) != 0) 1266 filp->f_kqflags |= LINUX_KQ_FLAG_NEED_WRITE; 1267 1268 /* make sure the "knote" gets woken up */ 1269 KNOTE_LOCKED(&filp->f_selinfo.si_note, 0); 1270 } 1271 spin_unlock(&filp->f_kqlock); 1272 } 1273 1274 static int 1275 linux_file_kqfilter(struct file *file, struct knote *kn) 1276 { 1277 struct linux_file *filp; 1278 struct thread *td; 1279 int error; 1280 1281 td = curthread; 1282 filp = (struct linux_file *)file->f_data; 1283 filp->f_flags = file->f_flag; 1284 if (filp->f_op->poll == NULL) 1285 return (EINVAL); 1286 1287 spin_lock(&filp->f_kqlock); 1288 switch (kn->kn_filter) { 1289 case EVFILT_READ: 1290 filp->f_kqflags |= LINUX_KQ_FLAG_HAS_READ; 1291 kn->kn_fop = &linux_dev_kqfiltops_read; 1292 kn->kn_hook = filp; 1293 knlist_add(&filp->f_selinfo.si_note, kn, 1); 1294 error = 0; 1295 break; 1296 case EVFILT_WRITE: 1297 filp->f_kqflags |= LINUX_KQ_FLAG_HAS_WRITE; 1298 kn->kn_fop = &linux_dev_kqfiltops_write; 1299 kn->kn_hook = filp; 1300 knlist_add(&filp->f_selinfo.si_note, kn, 1); 1301 error = 0; 1302 break; 1303 default: 1304 error = EINVAL; 1305 break; 1306 } 1307 spin_unlock(&filp->f_kqlock); 1308 1309 if (error == 0) { 1310 linux_set_current(td); 1311 1312 /* update kqfilter status, if any */ 1313 linux_file_kqfilter_poll(filp, 1314 LINUX_KQ_FLAG_HAS_READ | LINUX_KQ_FLAG_HAS_WRITE); 1315 } 1316 return (error); 1317 } 1318 1319 static int 1320 linux_file_mmap_single(struct file *fp, const struct file_operations *fop, 1321 vm_ooffset_t *offset, vm_size_t size, struct vm_object **object, 1322 int nprot, bool is_shared, struct thread *td) 1323 { 1324 struct task_struct *task; 1325 struct vm_area_struct *vmap; 1326 struct mm_struct *mm; 1327 struct linux_file *filp; 1328 vm_memattr_t attr; 1329 int error; 1330 1331 filp = (struct linux_file *)fp->f_data; 1332 filp->f_flags = fp->f_flag; 1333 1334 if (fop->mmap == NULL) 1335 return (EOPNOTSUPP); 1336 1337 linux_set_current(td); 1338 1339 /* 1340 * The same VM object might be shared by multiple processes 1341 * and the mm_struct is usually freed when a process exits. 1342 * 1343 * The atomic reference below makes sure the mm_struct is 1344 * available as long as the vmap is in the linux_vma_head. 1345 */ 1346 task = current; 1347 mm = task->mm; 1348 if (atomic_inc_not_zero(&mm->mm_users) == 0) 1349 return (EINVAL); 1350 1351 vmap = kzalloc(sizeof(*vmap), GFP_KERNEL); 1352 vmap->vm_start = 0; 1353 vmap->vm_end = size; 1354 vmap->vm_pgoff = *offset / PAGE_SIZE; 1355 vmap->vm_pfn = 0; 1356 vmap->vm_flags = vmap->vm_page_prot = (nprot & VM_PROT_ALL); 1357 if (is_shared) 1358 vmap->vm_flags |= VM_SHARED; 1359 vmap->vm_ops = NULL; 1360 vmap->vm_file = get_file(filp); 1361 vmap->vm_mm = mm; 1362 1363 if (unlikely(down_write_killable(&vmap->vm_mm->mmap_sem))) { 1364 error = linux_get_error(task, EINTR); 1365 } else { 1366 error = -OPW(fp, td, fop->mmap(filp, vmap)); 1367 error = linux_get_error(task, error); 1368 up_write(&vmap->vm_mm->mmap_sem); 1369 } 1370 1371 if (error != 0) { 1372 linux_cdev_handle_free(vmap); 1373 return (error); 1374 } 1375 1376 attr = pgprot2cachemode(vmap->vm_page_prot); 1377 1378 if (vmap->vm_ops != NULL) { 1379 struct vm_area_struct *ptr; 1380 void *vm_private_data; 1381 bool vm_no_fault; 1382 1383 if (vmap->vm_ops->open == NULL || 1384 vmap->vm_ops->close == NULL || 1385 vmap->vm_private_data == NULL) { 1386 /* free allocated VM area struct */ 1387 linux_cdev_handle_free(vmap); 1388 return (EINVAL); 1389 } 1390 1391 vm_private_data = vmap->vm_private_data; 1392 1393 rw_wlock(&linux_vma_lock); 1394 TAILQ_FOREACH(ptr, &linux_vma_head, vm_entry) { 1395 if (ptr->vm_private_data == vm_private_data) 1396 break; 1397 } 1398 /* check if there is an existing VM area struct */ 1399 if (ptr != NULL) { 1400 /* check if the VM area structure is invalid */ 1401 if (ptr->vm_ops == NULL || 1402 ptr->vm_ops->open == NULL || 1403 ptr->vm_ops->close == NULL) { 1404 error = ESTALE; 1405 vm_no_fault = 1; 1406 } else { 1407 error = EEXIST; 1408 vm_no_fault = (ptr->vm_ops->fault == NULL); 1409 } 1410 } else { 1411 /* insert VM area structure into list */ 1412 TAILQ_INSERT_TAIL(&linux_vma_head, vmap, vm_entry); 1413 error = 0; 1414 vm_no_fault = (vmap->vm_ops->fault == NULL); 1415 } 1416 rw_wunlock(&linux_vma_lock); 1417 1418 if (error != 0) { 1419 /* free allocated VM area struct */ 1420 linux_cdev_handle_free(vmap); 1421 /* check for stale VM area struct */ 1422 if (error != EEXIST) 1423 return (error); 1424 } 1425 1426 /* check if there is no fault handler */ 1427 if (vm_no_fault) { 1428 *object = cdev_pager_allocate(vm_private_data, OBJT_DEVICE, 1429 &linux_cdev_pager_ops[1], size, nprot, *offset, 1430 td->td_ucred); 1431 } else { 1432 *object = cdev_pager_allocate(vm_private_data, OBJT_MGTDEVICE, 1433 &linux_cdev_pager_ops[0], size, nprot, *offset, 1434 td->td_ucred); 1435 } 1436 1437 /* check if allocating the VM object failed */ 1438 if (*object == NULL) { 1439 if (error == 0) { 1440 /* remove VM area struct from list */ 1441 linux_cdev_handle_remove(vmap); 1442 /* free allocated VM area struct */ 1443 linux_cdev_handle_free(vmap); 1444 } 1445 return (EINVAL); 1446 } 1447 } else { 1448 struct sglist *sg; 1449 1450 sg = sglist_alloc(1, M_WAITOK); 1451 sglist_append_phys(sg, 1452 (vm_paddr_t)vmap->vm_pfn << PAGE_SHIFT, vmap->vm_len); 1453 1454 *object = vm_pager_allocate(OBJT_SG, sg, vmap->vm_len, 1455 nprot, 0, td->td_ucred); 1456 1457 linux_cdev_handle_free(vmap); 1458 1459 if (*object == NULL) { 1460 sglist_free(sg); 1461 return (EINVAL); 1462 } 1463 } 1464 1465 if (attr != VM_MEMATTR_DEFAULT) { 1466 VM_OBJECT_WLOCK(*object); 1467 vm_object_set_memattr(*object, attr); 1468 VM_OBJECT_WUNLOCK(*object); 1469 } 1470 *offset = 0; 1471 return (0); 1472 } 1473 1474 struct cdevsw linuxcdevsw = { 1475 .d_version = D_VERSION, 1476 .d_fdopen = linux_dev_fdopen, 1477 .d_name = "lkpidev", 1478 }; 1479 1480 static int 1481 linux_file_read(struct file *file, struct uio *uio, struct ucred *active_cred, 1482 int flags, struct thread *td) 1483 { 1484 struct linux_file *filp; 1485 const struct file_operations *fop; 1486 struct linux_cdev *ldev; 1487 ssize_t bytes; 1488 int error; 1489 1490 error = 0; 1491 filp = (struct linux_file *)file->f_data; 1492 filp->f_flags = file->f_flag; 1493 /* XXX no support for I/O vectors currently */ 1494 if (uio->uio_iovcnt != 1) 1495 return (EOPNOTSUPP); 1496 if (uio->uio_resid > DEVFS_IOSIZE_MAX) 1497 return (EINVAL); 1498 linux_set_current(td); 1499 linux_get_fop(filp, &fop, &ldev); 1500 if (fop->read != NULL) { 1501 bytes = OPW(file, td, fop->read(filp, 1502 uio->uio_iov->iov_base, 1503 uio->uio_iov->iov_len, &uio->uio_offset)); 1504 if (bytes >= 0) { 1505 uio->uio_iov->iov_base = 1506 ((uint8_t *)uio->uio_iov->iov_base) + bytes; 1507 uio->uio_iov->iov_len -= bytes; 1508 uio->uio_resid -= bytes; 1509 } else { 1510 error = linux_get_error(current, -bytes); 1511 } 1512 } else 1513 error = ENXIO; 1514 1515 /* update kqfilter status, if any */ 1516 linux_file_kqfilter_poll(filp, LINUX_KQ_FLAG_HAS_READ); 1517 linux_drop_fop(ldev); 1518 1519 return (error); 1520 } 1521 1522 static int 1523 linux_file_write(struct file *file, struct uio *uio, struct ucred *active_cred, 1524 int flags, struct thread *td) 1525 { 1526 struct linux_file *filp; 1527 const struct file_operations *fop; 1528 struct linux_cdev *ldev; 1529 ssize_t bytes; 1530 int error; 1531 1532 filp = (struct linux_file *)file->f_data; 1533 filp->f_flags = file->f_flag; 1534 /* XXX no support for I/O vectors currently */ 1535 if (uio->uio_iovcnt != 1) 1536 return (EOPNOTSUPP); 1537 if (uio->uio_resid > DEVFS_IOSIZE_MAX) 1538 return (EINVAL); 1539 linux_set_current(td); 1540 linux_get_fop(filp, &fop, &ldev); 1541 if (fop->write != NULL) { 1542 bytes = OPW(file, td, fop->write(filp, 1543 uio->uio_iov->iov_base, 1544 uio->uio_iov->iov_len, &uio->uio_offset)); 1545 if (bytes >= 0) { 1546 uio->uio_iov->iov_base = 1547 ((uint8_t *)uio->uio_iov->iov_base) + bytes; 1548 uio->uio_iov->iov_len -= bytes; 1549 uio->uio_resid -= bytes; 1550 error = 0; 1551 } else { 1552 error = linux_get_error(current, -bytes); 1553 } 1554 } else 1555 error = ENXIO; 1556 1557 /* update kqfilter status, if any */ 1558 linux_file_kqfilter_poll(filp, LINUX_KQ_FLAG_HAS_WRITE); 1559 1560 linux_drop_fop(ldev); 1561 1562 return (error); 1563 } 1564 1565 static int 1566 linux_file_poll(struct file *file, int events, struct ucred *active_cred, 1567 struct thread *td) 1568 { 1569 struct linux_file *filp; 1570 const struct file_operations *fop; 1571 struct linux_cdev *ldev; 1572 int revents; 1573 1574 filp = (struct linux_file *)file->f_data; 1575 filp->f_flags = file->f_flag; 1576 linux_set_current(td); 1577 linux_get_fop(filp, &fop, &ldev); 1578 if (fop->poll != NULL) { 1579 revents = OPW(file, td, fop->poll(filp, 1580 LINUX_POLL_TABLE_NORMAL)) & events; 1581 } else { 1582 revents = 0; 1583 } 1584 linux_drop_fop(ldev); 1585 return (revents); 1586 } 1587 1588 static int 1589 linux_file_close(struct file *file, struct thread *td) 1590 { 1591 struct linux_file *filp; 1592 int (*release)(struct inode *, struct linux_file *); 1593 const struct file_operations *fop; 1594 struct linux_cdev *ldev; 1595 int error; 1596 1597 filp = (struct linux_file *)file->f_data; 1598 1599 KASSERT(file_count(filp) == 0, 1600 ("File refcount(%d) is not zero", file_count(filp))); 1601 1602 if (td == NULL) 1603 td = curthread; 1604 1605 error = 0; 1606 filp->f_flags = file->f_flag; 1607 linux_set_current(td); 1608 linux_poll_wait_dequeue(filp); 1609 linux_get_fop(filp, &fop, &ldev); 1610 /* 1611 * Always use the real release function, if any, to avoid 1612 * leaking device resources: 1613 */ 1614 release = filp->f_op->release; 1615 if (release != NULL) 1616 error = -OPW(file, td, release(filp->f_vnode, filp)); 1617 funsetown(&filp->f_sigio); 1618 if (filp->f_vnode != NULL) 1619 vrele(filp->f_vnode); 1620 linux_drop_fop(ldev); 1621 ldev = filp->f_cdev; 1622 if (ldev != NULL) 1623 linux_cdev_deref(ldev); 1624 linux_synchronize_rcu(RCU_TYPE_REGULAR); 1625 kfree(filp); 1626 1627 return (error); 1628 } 1629 1630 static int 1631 linux_file_ioctl(struct file *fp, u_long cmd, void *data, struct ucred *cred, 1632 struct thread *td) 1633 { 1634 struct linux_file *filp; 1635 const struct file_operations *fop; 1636 struct linux_cdev *ldev; 1637 struct fiodgname_arg *fgn; 1638 const char *p; 1639 int error, i; 1640 1641 error = 0; 1642 filp = (struct linux_file *)fp->f_data; 1643 filp->f_flags = fp->f_flag; 1644 linux_get_fop(filp, &fop, &ldev); 1645 1646 linux_set_current(td); 1647 switch (cmd) { 1648 case FIONBIO: 1649 break; 1650 case FIOASYNC: 1651 if (fop->fasync == NULL) 1652 break; 1653 error = -OPW(fp, td, fop->fasync(0, filp, fp->f_flag & FASYNC)); 1654 break; 1655 case FIOSETOWN: 1656 error = fsetown(*(int *)data, &filp->f_sigio); 1657 if (error == 0) { 1658 if (fop->fasync == NULL) 1659 break; 1660 error = -OPW(fp, td, fop->fasync(0, filp, 1661 fp->f_flag & FASYNC)); 1662 } 1663 break; 1664 case FIOGETOWN: 1665 *(int *)data = fgetown(&filp->f_sigio); 1666 break; 1667 case FIODGNAME: 1668 #ifdef COMPAT_FREEBSD32 1669 case FIODGNAME_32: 1670 #endif 1671 if (filp->f_cdev == NULL || filp->f_cdev->cdev == NULL) { 1672 error = ENXIO; 1673 break; 1674 } 1675 fgn = data; 1676 p = devtoname(filp->f_cdev->cdev); 1677 i = strlen(p) + 1; 1678 if (i > fgn->len) { 1679 error = EINVAL; 1680 break; 1681 } 1682 error = copyout(p, fiodgname_buf_get_ptr(fgn, cmd), i); 1683 break; 1684 default: 1685 error = linux_file_ioctl_sub(fp, filp, fop, cmd, data, td); 1686 break; 1687 } 1688 linux_drop_fop(ldev); 1689 return (error); 1690 } 1691 1692 static int 1693 linux_file_mmap_sub(struct thread *td, vm_size_t objsize, vm_prot_t prot, 1694 vm_prot_t maxprot, int flags, struct file *fp, 1695 vm_ooffset_t *foff, const struct file_operations *fop, vm_object_t *objp) 1696 { 1697 /* 1698 * Character devices do not provide private mappings 1699 * of any kind: 1700 */ 1701 if ((maxprot & VM_PROT_WRITE) == 0 && 1702 (prot & VM_PROT_WRITE) != 0) 1703 return (EACCES); 1704 if ((flags & (MAP_PRIVATE | MAP_COPY)) != 0) 1705 return (EINVAL); 1706 1707 return (linux_file_mmap_single(fp, fop, foff, objsize, objp, 1708 (int)prot, (flags & MAP_SHARED) ? true : false, td)); 1709 } 1710 1711 static int 1712 linux_file_mmap(struct file *fp, vm_map_t map, vm_offset_t *addr, vm_size_t size, 1713 vm_prot_t prot, vm_prot_t cap_maxprot, int flags, vm_ooffset_t foff, 1714 struct thread *td) 1715 { 1716 struct linux_file *filp; 1717 const struct file_operations *fop; 1718 struct linux_cdev *ldev; 1719 struct mount *mp; 1720 struct vnode *vp; 1721 vm_object_t object; 1722 vm_prot_t maxprot; 1723 int error; 1724 1725 filp = (struct linux_file *)fp->f_data; 1726 1727 vp = filp->f_vnode; 1728 if (vp == NULL) 1729 return (EOPNOTSUPP); 1730 1731 /* 1732 * Ensure that file and memory protections are 1733 * compatible. 1734 */ 1735 mp = vp->v_mount; 1736 if (mp != NULL && (mp->mnt_flag & MNT_NOEXEC) != 0) { 1737 maxprot = VM_PROT_NONE; 1738 if ((prot & VM_PROT_EXECUTE) != 0) 1739 return (EACCES); 1740 } else 1741 maxprot = VM_PROT_EXECUTE; 1742 if ((fp->f_flag & FREAD) != 0) 1743 maxprot |= VM_PROT_READ; 1744 else if ((prot & VM_PROT_READ) != 0) 1745 return (EACCES); 1746 1747 /* 1748 * If we are sharing potential changes via MAP_SHARED and we 1749 * are trying to get write permission although we opened it 1750 * without asking for it, bail out. 1751 * 1752 * Note that most character devices always share mappings. 1753 * 1754 * Rely on linux_file_mmap_sub() to fail invalid MAP_PRIVATE 1755 * requests rather than doing it here. 1756 */ 1757 if ((flags & MAP_SHARED) != 0) { 1758 if ((fp->f_flag & FWRITE) != 0) 1759 maxprot |= VM_PROT_WRITE; 1760 else if ((prot & VM_PROT_WRITE) != 0) 1761 return (EACCES); 1762 } 1763 maxprot &= cap_maxprot; 1764 1765 linux_get_fop(filp, &fop, &ldev); 1766 error = linux_file_mmap_sub(td, size, prot, maxprot, flags, fp, 1767 &foff, fop, &object); 1768 if (error != 0) 1769 goto out; 1770 1771 error = vm_mmap_object(map, addr, size, prot, maxprot, flags, object, 1772 foff, FALSE, td); 1773 if (error != 0) 1774 vm_object_deallocate(object); 1775 out: 1776 linux_drop_fop(ldev); 1777 return (error); 1778 } 1779 1780 static int 1781 linux_file_stat(struct file *fp, struct stat *sb, struct ucred *active_cred) 1782 { 1783 struct linux_file *filp; 1784 struct vnode *vp; 1785 int error; 1786 1787 filp = (struct linux_file *)fp->f_data; 1788 if (filp->f_vnode == NULL) 1789 return (EOPNOTSUPP); 1790 1791 vp = filp->f_vnode; 1792 1793 vn_lock(vp, LK_SHARED | LK_RETRY); 1794 error = VOP_STAT(vp, sb, curthread->td_ucred, NOCRED); 1795 VOP_UNLOCK(vp); 1796 1797 return (error); 1798 } 1799 1800 static int 1801 linux_file_fill_kinfo(struct file *fp, struct kinfo_file *kif, 1802 struct filedesc *fdp) 1803 { 1804 struct linux_file *filp; 1805 struct vnode *vp; 1806 int error; 1807 1808 filp = fp->f_data; 1809 vp = filp->f_vnode; 1810 if (vp == NULL) { 1811 error = 0; 1812 kif->kf_type = KF_TYPE_DEV; 1813 } else { 1814 vref(vp); 1815 FILEDESC_SUNLOCK(fdp); 1816 error = vn_fill_kinfo_vnode(vp, kif); 1817 vrele(vp); 1818 kif->kf_type = KF_TYPE_VNODE; 1819 FILEDESC_SLOCK(fdp); 1820 } 1821 return (error); 1822 } 1823 1824 unsigned int 1825 linux_iminor(struct inode *inode) 1826 { 1827 struct linux_cdev *ldev; 1828 1829 if (inode == NULL || inode->v_rdev == NULL || 1830 inode->v_rdev->si_devsw != &linuxcdevsw) 1831 return (-1U); 1832 ldev = inode->v_rdev->si_drv1; 1833 if (ldev == NULL) 1834 return (-1U); 1835 1836 return (minor(ldev->dev)); 1837 } 1838 1839 static int 1840 linux_file_kcmp(struct file *fp1, struct file *fp2, struct thread *td) 1841 { 1842 struct linux_file *filp1, *filp2; 1843 1844 if (fp2->f_type != DTYPE_DEV) 1845 return (3); 1846 1847 filp1 = fp1->f_data; 1848 filp2 = fp2->f_data; 1849 return (kcmp_cmp((uintptr_t)filp1->f_cdev, (uintptr_t)filp2->f_cdev)); 1850 } 1851 1852 const struct fileops linuxfileops = { 1853 .fo_read = linux_file_read, 1854 .fo_write = linux_file_write, 1855 .fo_truncate = invfo_truncate, 1856 .fo_kqfilter = linux_file_kqfilter, 1857 .fo_stat = linux_file_stat, 1858 .fo_fill_kinfo = linux_file_fill_kinfo, 1859 .fo_poll = linux_file_poll, 1860 .fo_close = linux_file_close, 1861 .fo_ioctl = linux_file_ioctl, 1862 .fo_mmap = linux_file_mmap, 1863 .fo_chmod = invfo_chmod, 1864 .fo_chown = invfo_chown, 1865 .fo_sendfile = invfo_sendfile, 1866 .fo_cmp = linux_file_kcmp, 1867 .fo_flags = DFLAG_PASSABLE, 1868 }; 1869 1870 static char * 1871 devm_kvasprintf(struct device *dev, gfp_t gfp, const char *fmt, va_list ap) 1872 { 1873 unsigned int len; 1874 char *p; 1875 va_list aq; 1876 1877 va_copy(aq, ap); 1878 len = vsnprintf(NULL, 0, fmt, aq); 1879 va_end(aq); 1880 1881 if (dev != NULL) 1882 p = devm_kmalloc(dev, len + 1, gfp); 1883 else 1884 p = kmalloc(len + 1, gfp); 1885 if (p != NULL) 1886 vsnprintf(p, len + 1, fmt, ap); 1887 1888 return (p); 1889 } 1890 1891 char * 1892 kvasprintf(gfp_t gfp, const char *fmt, va_list ap) 1893 { 1894 1895 return (devm_kvasprintf(NULL, gfp, fmt, ap)); 1896 } 1897 1898 char * 1899 lkpi_devm_kasprintf(struct device *dev, gfp_t gfp, const char *fmt, ...) 1900 { 1901 va_list ap; 1902 char *p; 1903 1904 va_start(ap, fmt); 1905 p = devm_kvasprintf(dev, gfp, fmt, ap); 1906 va_end(ap); 1907 1908 return (p); 1909 } 1910 1911 char * 1912 kasprintf(gfp_t gfp, const char *fmt, ...) 1913 { 1914 va_list ap; 1915 char *p; 1916 1917 va_start(ap, fmt); 1918 p = kvasprintf(gfp, fmt, ap); 1919 va_end(ap); 1920 1921 return (p); 1922 } 1923 1924 int 1925 __lkpi_hexdump_printf(void *arg1 __unused, const char *fmt, ...) 1926 { 1927 va_list ap; 1928 int result; 1929 1930 va_start(ap, fmt); 1931 result = vprintf(fmt, ap); 1932 va_end(ap); 1933 return (result); 1934 } 1935 1936 int 1937 __lkpi_hexdump_sbuf_printf(void *arg1, const char *fmt, ...) 1938 { 1939 va_list ap; 1940 int result; 1941 1942 va_start(ap, fmt); 1943 result = sbuf_vprintf(arg1, fmt, ap); 1944 va_end(ap); 1945 return (result); 1946 } 1947 1948 void 1949 lkpi_hex_dump(int(*_fpf)(void *, const char *, ...), void *arg1, 1950 const char *level, const char *prefix_str, 1951 const int prefix_type, const int rowsize, const int groupsize, 1952 const void *buf, size_t len, const bool ascii, const bool trailing_newline) 1953 { 1954 typedef const struct { long long value; } __packed *print_64p_t; 1955 typedef const struct { uint32_t value; } __packed *print_32p_t; 1956 typedef const struct { uint16_t value; } __packed *print_16p_t; 1957 const void *buf_old = buf; 1958 int row, linelen, ret; 1959 1960 while (len > 0) { 1961 linelen = 0; 1962 if (level != NULL) { 1963 ret = _fpf(arg1, "%s", level); 1964 if (ret < 0) 1965 break; 1966 linelen += ret; 1967 } 1968 if (prefix_str != NULL) { 1969 ret = _fpf( 1970 arg1, "%s%s", linelen ? " " : "", prefix_str); 1971 if (ret < 0) 1972 break; 1973 linelen += ret; 1974 } 1975 1976 switch (prefix_type) { 1977 case DUMP_PREFIX_ADDRESS: 1978 ret = _fpf( 1979 arg1, "%s[%p]", linelen ? " " : "", buf); 1980 if (ret < 0) 1981 return; 1982 linelen += ret; 1983 break; 1984 case DUMP_PREFIX_OFFSET: 1985 ret = _fpf( 1986 arg1, "%s[%#tx]", linelen ? " " : "", 1987 ((const char *)buf - (const char *)buf_old)); 1988 if (ret < 0) 1989 return; 1990 linelen += ret; 1991 break; 1992 default: 1993 break; 1994 } 1995 for (row = 0; row != rowsize; row++) { 1996 if (groupsize == 8 && len > 7) { 1997 ret = _fpf( 1998 arg1, "%s%016llx", linelen ? " " : "", 1999 ((print_64p_t)buf)->value); 2000 if (ret < 0) 2001 return; 2002 linelen += ret; 2003 buf = (const uint8_t *)buf + 8; 2004 len -= 8; 2005 } else if (groupsize == 4 && len > 3) { 2006 ret = _fpf( 2007 arg1, "%s%08x", linelen ? " " : "", 2008 ((print_32p_t)buf)->value); 2009 if (ret < 0) 2010 return; 2011 linelen += ret; 2012 buf = (const uint8_t *)buf + 4; 2013 len -= 4; 2014 } else if (groupsize == 2 && len > 1) { 2015 ret = _fpf( 2016 arg1, "%s%04x", linelen ? " " : "", 2017 ((print_16p_t)buf)->value); 2018 if (ret < 0) 2019 return; 2020 linelen += ret; 2021 buf = (const uint8_t *)buf + 2; 2022 len -= 2; 2023 } else if (len > 0) { 2024 ret = _fpf( 2025 arg1, "%s%02x", linelen ? " " : "", 2026 *(const uint8_t *)buf); 2027 if (ret < 0) 2028 return; 2029 linelen += ret; 2030 buf = (const uint8_t *)buf + 1; 2031 len--; 2032 } else { 2033 break; 2034 } 2035 } 2036 if (len > 0 && trailing_newline) { 2037 ret = _fpf(arg1, "\n"); 2038 if (ret < 0) 2039 break; 2040 } 2041 } 2042 } 2043 2044 struct hdtb_context { 2045 char *linebuf; 2046 size_t linebuflen; 2047 int written; 2048 }; 2049 2050 static int 2051 hdtb_cb(void *arg, const char *format, ...) 2052 { 2053 struct hdtb_context *context; 2054 int written; 2055 va_list args; 2056 2057 context = arg; 2058 2059 va_start(args, format); 2060 written = vsnprintf( 2061 context->linebuf, context->linebuflen, format, args); 2062 va_end(args); 2063 2064 if (written < 0) 2065 return (written); 2066 2067 /* 2068 * Linux' hex_dump_to_buffer() function has the same behaviour as 2069 * snprintf() basically. Therefore, it returns the number of bytes it 2070 * would have written if the destination buffer was large enough. 2071 * 2072 * If the destination buffer was exhausted, lkpi_hex_dump() will 2073 * continue to call this callback but it will only compute the bytes it 2074 * would have written but write nothing to that buffer. 2075 */ 2076 context->written += written; 2077 2078 if (written < context->linebuflen) { 2079 context->linebuf += written; 2080 context->linebuflen -= written; 2081 } else { 2082 context->linebuf += context->linebuflen; 2083 context->linebuflen = 0; 2084 } 2085 2086 return (written); 2087 } 2088 2089 int 2090 lkpi_hex_dump_to_buffer(const void *buf, size_t len, int rowsize, 2091 int groupsize, char *linebuf, size_t linebuflen, bool ascii) 2092 { 2093 int written; 2094 struct hdtb_context context; 2095 2096 context.linebuf = linebuf; 2097 context.linebuflen = linebuflen; 2098 context.written = 0; 2099 2100 if (rowsize != 16 && rowsize != 32) 2101 rowsize = 16; 2102 2103 len = min(len, rowsize); 2104 2105 lkpi_hex_dump( 2106 hdtb_cb, &context, NULL, NULL, DUMP_PREFIX_NONE, 2107 rowsize, groupsize, buf, len, ascii, false); 2108 2109 written = context.written; 2110 2111 return (written); 2112 } 2113 2114 static void 2115 linux_timer_callback_wrapper(void *context) 2116 { 2117 struct timer_list *timer; 2118 2119 timer = context; 2120 2121 /* the timer is about to be shutdown permanently */ 2122 if (timer->function == NULL) 2123 return; 2124 2125 if (linux_set_current_flags(curthread, M_NOWAIT)) { 2126 /* try again later */ 2127 callout_reset(&timer->callout, 1, 2128 &linux_timer_callback_wrapper, timer); 2129 return; 2130 } 2131 2132 timer->function(timer->data); 2133 } 2134 2135 static int 2136 linux_timer_jiffies_until(unsigned long expires) 2137 { 2138 unsigned long delta = expires - jiffies; 2139 2140 /* 2141 * Guard against already expired values and make sure that the value can 2142 * be used as a tick count, rather than a jiffies count. 2143 */ 2144 if ((long)delta < 1) 2145 delta = 1; 2146 else if (delta > INT_MAX) 2147 delta = INT_MAX; 2148 return ((int)delta); 2149 } 2150 2151 int 2152 mod_timer(struct timer_list *timer, unsigned long expires) 2153 { 2154 int ret; 2155 2156 timer->expires = expires; 2157 ret = callout_reset(&timer->callout, 2158 linux_timer_jiffies_until(expires), 2159 &linux_timer_callback_wrapper, timer); 2160 2161 MPASS(ret == 0 || ret == 1); 2162 2163 return (ret == 1); 2164 } 2165 2166 void 2167 add_timer(struct timer_list *timer) 2168 { 2169 2170 callout_reset(&timer->callout, 2171 linux_timer_jiffies_until(timer->expires), 2172 &linux_timer_callback_wrapper, timer); 2173 } 2174 2175 void 2176 add_timer_on(struct timer_list *timer, int cpu) 2177 { 2178 2179 callout_reset_on(&timer->callout, 2180 linux_timer_jiffies_until(timer->expires), 2181 &linux_timer_callback_wrapper, timer, cpu); 2182 } 2183 2184 int 2185 timer_delete(struct timer_list *timer) 2186 { 2187 2188 if (callout_stop(&(timer)->callout) == -1) 2189 return (0); 2190 return (1); 2191 } 2192 2193 int 2194 timer_delete_sync(struct timer_list *timer) 2195 { 2196 2197 if (callout_drain(&(timer)->callout) == -1) 2198 return (0); 2199 return (1); 2200 } 2201 2202 int 2203 timer_shutdown_sync(struct timer_list *timer) 2204 { 2205 2206 timer->function = NULL; 2207 return (del_timer_sync(timer)); 2208 } 2209 2210 /* greatest common divisor, Euclid equation */ 2211 static uint64_t 2212 lkpi_gcd_64(uint64_t a, uint64_t b) 2213 { 2214 uint64_t an; 2215 uint64_t bn; 2216 2217 while (b != 0) { 2218 an = b; 2219 bn = a % b; 2220 a = an; 2221 b = bn; 2222 } 2223 return (a); 2224 } 2225 2226 uint64_t lkpi_nsec2hz_rem; 2227 uint64_t lkpi_nsec2hz_div = 1000000000ULL; 2228 uint64_t lkpi_nsec2hz_max; 2229 2230 uint64_t lkpi_usec2hz_rem; 2231 uint64_t lkpi_usec2hz_div = 1000000ULL; 2232 uint64_t lkpi_usec2hz_max; 2233 2234 uint64_t lkpi_msec2hz_rem; 2235 uint64_t lkpi_msec2hz_div = 1000ULL; 2236 uint64_t lkpi_msec2hz_max; 2237 2238 static void 2239 linux_timer_init(void *arg) 2240 { 2241 uint64_t gcd; 2242 2243 /* 2244 * Compute an internal HZ value which can divide 2**32 to 2245 * avoid timer rounding problems when the tick value wraps 2246 * around 2**32: 2247 */ 2248 linux_timer_hz_mask = 1; 2249 while (linux_timer_hz_mask < (unsigned long)hz) 2250 linux_timer_hz_mask *= 2; 2251 linux_timer_hz_mask--; 2252 2253 /* compute some internal constants */ 2254 2255 lkpi_nsec2hz_rem = hz; 2256 lkpi_usec2hz_rem = hz; 2257 lkpi_msec2hz_rem = hz; 2258 2259 gcd = lkpi_gcd_64(lkpi_nsec2hz_rem, lkpi_nsec2hz_div); 2260 lkpi_nsec2hz_rem /= gcd; 2261 lkpi_nsec2hz_div /= gcd; 2262 lkpi_nsec2hz_max = -1ULL / lkpi_nsec2hz_rem; 2263 2264 gcd = lkpi_gcd_64(lkpi_usec2hz_rem, lkpi_usec2hz_div); 2265 lkpi_usec2hz_rem /= gcd; 2266 lkpi_usec2hz_div /= gcd; 2267 lkpi_usec2hz_max = -1ULL / lkpi_usec2hz_rem; 2268 2269 gcd = lkpi_gcd_64(lkpi_msec2hz_rem, lkpi_msec2hz_div); 2270 lkpi_msec2hz_rem /= gcd; 2271 lkpi_msec2hz_div /= gcd; 2272 lkpi_msec2hz_max = -1ULL / lkpi_msec2hz_rem; 2273 } 2274 SYSINIT(linux_timer, SI_SUB_DRIVERS, SI_ORDER_FIRST, linux_timer_init, NULL); 2275 2276 void 2277 linux_complete_common(struct completion *c, int all) 2278 { 2279 sleepq_lock(c); 2280 if (all) { 2281 c->done = UINT_MAX; 2282 sleepq_broadcast(c, SLEEPQ_SLEEP, 0, 0); 2283 } else { 2284 if (c->done != UINT_MAX) 2285 c->done++; 2286 sleepq_signal(c, SLEEPQ_SLEEP, 0, 0); 2287 } 2288 sleepq_release(c); 2289 } 2290 2291 /* 2292 * Indefinite wait for done != 0 with or without signals. 2293 */ 2294 int 2295 linux_wait_for_common(struct completion *c, int flags) 2296 { 2297 struct task_struct *task; 2298 int error; 2299 2300 if (SCHEDULER_STOPPED()) 2301 return (0); 2302 2303 task = current; 2304 2305 if (flags != 0) 2306 flags = SLEEPQ_INTERRUPTIBLE | SLEEPQ_SLEEP; 2307 else 2308 flags = SLEEPQ_SLEEP; 2309 error = 0; 2310 for (;;) { 2311 sleepq_lock(c); 2312 if (c->done) 2313 break; 2314 sleepq_add(c, NULL, "completion", flags, 0); 2315 if (flags & SLEEPQ_INTERRUPTIBLE) { 2316 DROP_GIANT(); 2317 error = -sleepq_wait_sig(c, 0); 2318 PICKUP_GIANT(); 2319 if (error != 0) { 2320 linux_schedule_save_interrupt_value(task, error); 2321 error = -ERESTARTSYS; 2322 goto intr; 2323 } 2324 } else { 2325 DROP_GIANT(); 2326 sleepq_wait(c, 0); 2327 PICKUP_GIANT(); 2328 } 2329 } 2330 if (c->done != UINT_MAX) 2331 c->done--; 2332 sleepq_release(c); 2333 2334 intr: 2335 return (error); 2336 } 2337 2338 /* 2339 * Time limited wait for done != 0 with or without signals. 2340 */ 2341 unsigned long 2342 linux_wait_for_timeout_common(struct completion *c, unsigned long timeout, 2343 int flags) 2344 { 2345 struct task_struct *task; 2346 unsigned long end = jiffies + timeout, error; 2347 2348 if (SCHEDULER_STOPPED()) 2349 return (0); 2350 2351 task = current; 2352 2353 if (flags != 0) 2354 flags = SLEEPQ_INTERRUPTIBLE | SLEEPQ_SLEEP; 2355 else 2356 flags = SLEEPQ_SLEEP; 2357 2358 for (;;) { 2359 sleepq_lock(c); 2360 if (c->done) 2361 break; 2362 sleepq_add(c, NULL, "completion", flags, 0); 2363 sleepq_set_timeout(c, linux_timer_jiffies_until(end)); 2364 2365 DROP_GIANT(); 2366 if (flags & SLEEPQ_INTERRUPTIBLE) 2367 error = -sleepq_timedwait_sig(c, 0); 2368 else 2369 error = -sleepq_timedwait(c, 0); 2370 PICKUP_GIANT(); 2371 2372 if (error != 0) { 2373 /* check for timeout */ 2374 if (error == -EWOULDBLOCK) { 2375 error = 0; /* timeout */ 2376 } else { 2377 /* signal happened */ 2378 linux_schedule_save_interrupt_value(task, error); 2379 error = -ERESTARTSYS; 2380 } 2381 goto done; 2382 } 2383 } 2384 if (c->done != UINT_MAX) 2385 c->done--; 2386 sleepq_release(c); 2387 2388 /* return how many jiffies are left */ 2389 error = linux_timer_jiffies_until(end); 2390 done: 2391 return (error); 2392 } 2393 2394 int 2395 linux_try_wait_for_completion(struct completion *c) 2396 { 2397 int isdone; 2398 2399 sleepq_lock(c); 2400 isdone = (c->done != 0); 2401 if (c->done != 0 && c->done != UINT_MAX) 2402 c->done--; 2403 sleepq_release(c); 2404 return (isdone); 2405 } 2406 2407 int 2408 linux_completion_done(struct completion *c) 2409 { 2410 int isdone; 2411 2412 sleepq_lock(c); 2413 isdone = (c->done != 0); 2414 sleepq_release(c); 2415 return (isdone); 2416 } 2417 2418 static void 2419 linux_cdev_deref(struct linux_cdev *ldev) 2420 { 2421 if (refcount_release(&ldev->refs) && 2422 ldev->kobj.ktype == &linux_cdev_ktype) 2423 kfree(ldev); 2424 } 2425 2426 static void 2427 linux_cdev_release(struct kobject *kobj) 2428 { 2429 struct linux_cdev *cdev; 2430 struct kobject *parent; 2431 2432 cdev = container_of(kobj, struct linux_cdev, kobj); 2433 parent = kobj->parent; 2434 linux_destroy_dev(cdev); 2435 linux_cdev_deref(cdev); 2436 kobject_put(parent); 2437 } 2438 2439 static void 2440 linux_cdev_static_release(struct kobject *kobj) 2441 { 2442 struct cdev *cdev; 2443 struct linux_cdev *ldev; 2444 2445 ldev = container_of(kobj, struct linux_cdev, kobj); 2446 cdev = ldev->cdev; 2447 if (cdev != NULL) { 2448 destroy_dev(cdev); 2449 ldev->cdev = NULL; 2450 } 2451 kobject_put(kobj->parent); 2452 } 2453 2454 int 2455 linux_cdev_device_add(struct linux_cdev *ldev, struct device *dev) 2456 { 2457 int ret; 2458 2459 if (dev->devt != 0) { 2460 /* Set parent kernel object. */ 2461 ldev->kobj.parent = &dev->kobj; 2462 2463 /* 2464 * Unlike Linux we require the kobject of the 2465 * character device structure to have a valid name 2466 * before calling this function: 2467 */ 2468 if (ldev->kobj.name == NULL) 2469 return (-EINVAL); 2470 2471 ret = cdev_add(ldev, dev->devt, 1); 2472 if (ret) 2473 return (ret); 2474 } 2475 ret = device_add(dev); 2476 if (ret != 0 && dev->devt != 0) 2477 cdev_del(ldev); 2478 return (ret); 2479 } 2480 2481 void 2482 linux_cdev_device_del(struct linux_cdev *ldev, struct device *dev) 2483 { 2484 device_del(dev); 2485 2486 if (dev->devt != 0) 2487 cdev_del(ldev); 2488 } 2489 2490 static void 2491 linux_destroy_dev(struct linux_cdev *ldev) 2492 { 2493 2494 if (ldev->cdev == NULL) 2495 return; 2496 2497 MPASS((ldev->siref & LDEV_SI_DTR) == 0); 2498 MPASS(ldev->kobj.ktype == &linux_cdev_ktype); 2499 2500 atomic_set_int(&ldev->siref, LDEV_SI_DTR); 2501 while ((atomic_load_int(&ldev->siref) & ~LDEV_SI_DTR) != 0) 2502 pause("ldevdtr", hz / 4); 2503 2504 destroy_dev(ldev->cdev); 2505 ldev->cdev = NULL; 2506 } 2507 2508 const struct kobj_type linux_cdev_ktype = { 2509 .release = linux_cdev_release, 2510 }; 2511 2512 const struct kobj_type linux_cdev_static_ktype = { 2513 .release = linux_cdev_static_release, 2514 }; 2515 2516 static void 2517 linux_handle_ifnet_link_event(void *arg, struct ifnet *ifp, int linkstate) 2518 { 2519 struct notifier_block *nb; 2520 struct netdev_notifier_info ni; 2521 2522 nb = arg; 2523 ni.ifp = ifp; 2524 ni.dev = (struct net_device *)ifp; 2525 if (linkstate == LINK_STATE_UP) 2526 nb->notifier_call(nb, NETDEV_UP, &ni); 2527 else 2528 nb->notifier_call(nb, NETDEV_DOWN, &ni); 2529 } 2530 2531 static void 2532 linux_handle_ifnet_arrival_event(void *arg, struct ifnet *ifp) 2533 { 2534 struct notifier_block *nb; 2535 struct netdev_notifier_info ni; 2536 2537 nb = arg; 2538 ni.ifp = ifp; 2539 ni.dev = (struct net_device *)ifp; 2540 nb->notifier_call(nb, NETDEV_REGISTER, &ni); 2541 } 2542 2543 static void 2544 linux_handle_ifnet_departure_event(void *arg, struct ifnet *ifp) 2545 { 2546 struct notifier_block *nb; 2547 struct netdev_notifier_info ni; 2548 2549 nb = arg; 2550 ni.ifp = ifp; 2551 ni.dev = (struct net_device *)ifp; 2552 nb->notifier_call(nb, NETDEV_UNREGISTER, &ni); 2553 } 2554 2555 static void 2556 linux_handle_iflladdr_event(void *arg, struct ifnet *ifp) 2557 { 2558 struct notifier_block *nb; 2559 struct netdev_notifier_info ni; 2560 2561 nb = arg; 2562 ni.ifp = ifp; 2563 ni.dev = (struct net_device *)ifp; 2564 nb->notifier_call(nb, NETDEV_CHANGEADDR, &ni); 2565 } 2566 2567 static void 2568 linux_handle_ifaddr_event(void *arg, struct ifnet *ifp) 2569 { 2570 struct notifier_block *nb; 2571 struct netdev_notifier_info ni; 2572 2573 nb = arg; 2574 ni.ifp = ifp; 2575 ni.dev = (struct net_device *)ifp; 2576 nb->notifier_call(nb, NETDEV_CHANGEIFADDR, &ni); 2577 } 2578 2579 int 2580 register_netdevice_notifier(struct notifier_block *nb) 2581 { 2582 2583 nb->tags[NETDEV_UP] = EVENTHANDLER_REGISTER( 2584 ifnet_link_event, linux_handle_ifnet_link_event, nb, 0); 2585 nb->tags[NETDEV_REGISTER] = EVENTHANDLER_REGISTER( 2586 ifnet_arrival_event, linux_handle_ifnet_arrival_event, nb, 0); 2587 nb->tags[NETDEV_UNREGISTER] = EVENTHANDLER_REGISTER( 2588 ifnet_departure_event, linux_handle_ifnet_departure_event, nb, 0); 2589 nb->tags[NETDEV_CHANGEADDR] = EVENTHANDLER_REGISTER( 2590 iflladdr_event, linux_handle_iflladdr_event, nb, 0); 2591 2592 return (0); 2593 } 2594 2595 int 2596 register_inetaddr_notifier(struct notifier_block *nb) 2597 { 2598 2599 nb->tags[NETDEV_CHANGEIFADDR] = EVENTHANDLER_REGISTER( 2600 ifaddr_event, linux_handle_ifaddr_event, nb, 0); 2601 return (0); 2602 } 2603 2604 int 2605 unregister_netdevice_notifier(struct notifier_block *nb) 2606 { 2607 2608 EVENTHANDLER_DEREGISTER(ifnet_link_event, 2609 nb->tags[NETDEV_UP]); 2610 EVENTHANDLER_DEREGISTER(ifnet_arrival_event, 2611 nb->tags[NETDEV_REGISTER]); 2612 EVENTHANDLER_DEREGISTER(ifnet_departure_event, 2613 nb->tags[NETDEV_UNREGISTER]); 2614 EVENTHANDLER_DEREGISTER(iflladdr_event, 2615 nb->tags[NETDEV_CHANGEADDR]); 2616 2617 return (0); 2618 } 2619 2620 int 2621 unregister_inetaddr_notifier(struct notifier_block *nb) 2622 { 2623 2624 EVENTHANDLER_DEREGISTER(ifaddr_event, 2625 nb->tags[NETDEV_CHANGEIFADDR]); 2626 2627 return (0); 2628 } 2629 2630 struct list_sort_thunk { 2631 int (*cmp)(void *, struct list_head *, struct list_head *); 2632 void *priv; 2633 }; 2634 2635 static inline int 2636 linux_le_cmp(const void *d1, const void *d2, void *priv) 2637 { 2638 struct list_head *le1, *le2; 2639 struct list_sort_thunk *thunk; 2640 2641 thunk = priv; 2642 le1 = *(__DECONST(struct list_head **, d1)); 2643 le2 = *(__DECONST(struct list_head **, d2)); 2644 return ((thunk->cmp)(thunk->priv, le1, le2)); 2645 } 2646 2647 void 2648 list_sort(void *priv, struct list_head *head, int (*cmp)(void *priv, 2649 struct list_head *a, struct list_head *b)) 2650 { 2651 struct list_sort_thunk thunk; 2652 struct list_head **ar, *le; 2653 size_t count, i; 2654 2655 count = 0; 2656 list_for_each(le, head) 2657 count++; 2658 ar = malloc(sizeof(struct list_head *) * count, M_KMALLOC, M_WAITOK); 2659 i = 0; 2660 list_for_each(le, head) 2661 ar[i++] = le; 2662 thunk.cmp = cmp; 2663 thunk.priv = priv; 2664 qsort_r(ar, count, sizeof(struct list_head *), linux_le_cmp, &thunk); 2665 INIT_LIST_HEAD(head); 2666 for (i = 0; i < count; i++) 2667 list_add_tail(ar[i], head); 2668 free(ar, M_KMALLOC); 2669 } 2670 2671 #if defined(__i386__) || defined(__amd64__) 2672 int 2673 linux_wbinvd_on_all_cpus(void) 2674 { 2675 2676 pmap_invalidate_cache(); 2677 return (0); 2678 } 2679 #endif 2680 2681 int 2682 linux_on_each_cpu(void callback(void *), void *data) 2683 { 2684 2685 smp_rendezvous(smp_no_rendezvous_barrier, callback, 2686 smp_no_rendezvous_barrier, data); 2687 return (0); 2688 } 2689 2690 int 2691 linux_in_atomic(void) 2692 { 2693 2694 return ((curthread->td_pflags & TDP_NOFAULTING) != 0); 2695 } 2696 2697 struct linux_cdev * 2698 linux_find_cdev(const char *name, unsigned major, unsigned minor) 2699 { 2700 dev_t dev = MKDEV(major, minor); 2701 struct cdev *cdev; 2702 2703 dev_lock(); 2704 LIST_FOREACH(cdev, &linuxcdevsw.d_devs, si_list) { 2705 struct linux_cdev *ldev = cdev->si_drv1; 2706 if (ldev->dev == dev && 2707 strcmp(kobject_name(&ldev->kobj), name) == 0) { 2708 break; 2709 } 2710 } 2711 dev_unlock(); 2712 2713 return (cdev != NULL ? cdev->si_drv1 : NULL); 2714 } 2715 2716 int 2717 __register_chrdev(unsigned int major, unsigned int baseminor, 2718 unsigned int count, const char *name, 2719 const struct file_operations *fops) 2720 { 2721 struct linux_cdev *cdev; 2722 int ret = 0; 2723 int i; 2724 2725 for (i = baseminor; i < baseminor + count; i++) { 2726 cdev = cdev_alloc(); 2727 cdev->ops = fops; 2728 kobject_set_name(&cdev->kobj, name); 2729 2730 ret = cdev_add(cdev, makedev(major, i), 1); 2731 if (ret != 0) 2732 break; 2733 } 2734 return (ret); 2735 } 2736 2737 int 2738 __register_chrdev_p(unsigned int major, unsigned int baseminor, 2739 unsigned int count, const char *name, 2740 const struct file_operations *fops, uid_t uid, 2741 gid_t gid, int mode) 2742 { 2743 struct linux_cdev *cdev; 2744 int ret = 0; 2745 int i; 2746 2747 for (i = baseminor; i < baseminor + count; i++) { 2748 cdev = cdev_alloc(); 2749 cdev->ops = fops; 2750 kobject_set_name(&cdev->kobj, name); 2751 2752 ret = cdev_add_ext(cdev, makedev(major, i), uid, gid, mode); 2753 if (ret != 0) 2754 break; 2755 } 2756 return (ret); 2757 } 2758 2759 void 2760 __unregister_chrdev(unsigned int major, unsigned int baseminor, 2761 unsigned int count, const char *name) 2762 { 2763 struct linux_cdev *cdevp; 2764 int i; 2765 2766 for (i = baseminor; i < baseminor + count; i++) { 2767 cdevp = linux_find_cdev(name, major, i); 2768 if (cdevp != NULL) 2769 cdev_del(cdevp); 2770 } 2771 } 2772 2773 void 2774 linux_dump_stack(void) 2775 { 2776 #ifdef STACK 2777 struct stack st; 2778 2779 stack_save(&st); 2780 stack_print(&st); 2781 #endif 2782 } 2783 2784 int 2785 linuxkpi_net_ratelimit(void) 2786 { 2787 2788 return (ppsratecheck(&lkpi_net_lastlog, &lkpi_net_curpps, 2789 lkpi_net_maxpps)); 2790 } 2791 2792 struct io_mapping * 2793 io_mapping_create_wc(resource_size_t base, unsigned long size) 2794 { 2795 struct io_mapping *mapping; 2796 2797 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 2798 if (mapping == NULL) 2799 return (NULL); 2800 return (io_mapping_init_wc(mapping, base, size)); 2801 } 2802 2803 /* We likely want a linuxkpi_device.c at some point. */ 2804 bool 2805 device_can_wakeup(struct device *dev) 2806 { 2807 2808 if (dev == NULL) 2809 return (false); 2810 /* 2811 * XXX-BZ iwlwifi queries it as part of enabling WoWLAN. 2812 * Normally this would be based on a bool in dev->power.XXX. 2813 * Check such as PCI PCIM_PCAP_*PME. We have no way to enable this yet. 2814 * We may get away by directly calling into bsddev for as long as 2815 * we can assume PCI only avoiding changing struct device breaking KBI. 2816 */ 2817 pr_debug("%s:%d: not enabled; see comment.\n", __func__, __LINE__); 2818 return (false); 2819 } 2820 2821 static void 2822 devm_device_group_remove(struct device *dev, void *p) 2823 { 2824 const struct attribute_group **dr = p; 2825 const struct attribute_group *group = *dr; 2826 2827 sysfs_remove_group(&dev->kobj, group); 2828 } 2829 2830 int 2831 lkpi_devm_device_add_group(struct device *dev, 2832 const struct attribute_group *group) 2833 { 2834 const struct attribute_group **dr; 2835 int ret; 2836 2837 dr = devres_alloc(devm_device_group_remove, sizeof(*dr), GFP_KERNEL); 2838 if (dr == NULL) 2839 return (-ENOMEM); 2840 2841 ret = sysfs_create_group(&dev->kobj, group); 2842 if (ret == 0) { 2843 *dr = group; 2844 devres_add(dev, dr); 2845 } else 2846 devres_free(dr); 2847 2848 return (ret); 2849 } 2850 2851 #if defined(__i386__) || defined(__amd64__) 2852 bool linux_cpu_has_clflush; 2853 struct cpuinfo_x86 boot_cpu_data; 2854 struct cpuinfo_x86 *__cpu_data; 2855 #endif 2856 2857 cpumask_t * 2858 lkpi_get_static_single_cpu_mask(int cpuid) 2859 { 2860 2861 KASSERT((cpuid >= 0 && cpuid <= mp_maxid), ("%s: invalid cpuid %d\n", 2862 __func__, cpuid)); 2863 KASSERT(!CPU_ABSENT(cpuid), ("%s: cpu with cpuid %d is absent\n", 2864 __func__, cpuid)); 2865 2866 return (static_single_cpu_mask[cpuid]); 2867 } 2868 2869 bool 2870 lkpi_xen_initial_domain(void) 2871 { 2872 #ifdef XENHVM 2873 return (xen_initial_domain()); 2874 #else 2875 return (false); 2876 #endif 2877 } 2878 2879 bool 2880 lkpi_xen_pv_domain(void) 2881 { 2882 #ifdef XENHVM 2883 return (xen_pv_domain()); 2884 #else 2885 return (false); 2886 #endif 2887 } 2888 2889 static void 2890 linux_compat_init(void *arg) 2891 { 2892 struct sysctl_oid *rootoid; 2893 int i; 2894 2895 #if defined(__i386__) || defined(__amd64__) 2896 static const uint32_t x86_vendors[X86_VENDOR_NUM] = { 2897 [X86_VENDOR_INTEL] = CPU_VENDOR_INTEL, 2898 [X86_VENDOR_CYRIX] = CPU_VENDOR_CYRIX, 2899 [X86_VENDOR_AMD] = CPU_VENDOR_AMD, 2900 [X86_VENDOR_UMC] = CPU_VENDOR_UMC, 2901 [X86_VENDOR_CENTAUR] = CPU_VENDOR_CENTAUR, 2902 [X86_VENDOR_TRANSMETA] = CPU_VENDOR_TRANSMETA, 2903 [X86_VENDOR_NSC] = CPU_VENDOR_NSC, 2904 [X86_VENDOR_HYGON] = CPU_VENDOR_HYGON, 2905 }; 2906 uint8_t x86_vendor = X86_VENDOR_UNKNOWN; 2907 2908 for (i = 0; i < X86_VENDOR_NUM; i++) { 2909 if (cpu_vendor_id != 0 && cpu_vendor_id == x86_vendors[i]) { 2910 x86_vendor = i; 2911 break; 2912 } 2913 } 2914 linux_cpu_has_clflush = (cpu_feature & CPUID_CLFSH); 2915 boot_cpu_data.x86_clflush_size = cpu_clflush_line_size; 2916 boot_cpu_data.x86_max_cores = mp_ncpus; 2917 boot_cpu_data.x86 = CPUID_TO_FAMILY(cpu_id); 2918 boot_cpu_data.x86_model = CPUID_TO_MODEL(cpu_id); 2919 boot_cpu_data.x86_vendor = x86_vendor; 2920 boot_cpu_data.x86_stepping = CPUID_TO_STEPPING(cpu_id); 2921 2922 __cpu_data = kmalloc_array(mp_maxid + 1, 2923 sizeof(*__cpu_data), M_WAITOK | M_ZERO); 2924 CPU_FOREACH(i) { 2925 __cpu_data[i].x86_clflush_size = cpu_clflush_line_size; 2926 __cpu_data[i].x86_max_cores = mp_ncpus; 2927 __cpu_data[i].x86 = CPUID_TO_FAMILY(cpu_id); 2928 __cpu_data[i].x86_model = CPUID_TO_MODEL(cpu_id); 2929 __cpu_data[i].x86_vendor = x86_vendor; 2930 } 2931 #endif 2932 rw_init(&linux_vma_lock, "lkpi-vma-lock"); 2933 2934 rootoid = SYSCTL_ADD_ROOT_NODE(NULL, 2935 OID_AUTO, "sys", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "sys"); 2936 kobject_init(&linux_class_root, &linux_class_ktype); 2937 kobject_set_name(&linux_class_root, "class"); 2938 linux_class_root.oidp = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(rootoid), 2939 OID_AUTO, "class", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "class"); 2940 kobject_init(&linux_root_device.kobj, &linux_dev_ktype); 2941 kobject_set_name(&linux_root_device.kobj, "device"); 2942 linux_root_device.kobj.oidp = SYSCTL_ADD_NODE(NULL, 2943 SYSCTL_CHILDREN(rootoid), OID_AUTO, "device", 2944 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "device"); 2945 linux_root_device.bsddev = root_bus; 2946 linux_class_misc.name = "misc"; 2947 class_register(&linux_class_misc); 2948 INIT_LIST_HEAD(&pci_drivers); 2949 INIT_LIST_HEAD(&pci_devices); 2950 spin_lock_init(&pci_lock); 2951 init_waitqueue_head(&linux_bit_waitq); 2952 init_waitqueue_head(&linux_var_waitq); 2953 2954 CPU_COPY(&all_cpus, &cpu_online_mask); 2955 /* 2956 * Generate a single-CPU cpumask_t for each CPU (possibly) in the system. 2957 * CPUs are indexed from 0..(mp_maxid). The entry for cpuid 0 will only 2958 * have itself in the cpumask, cupid 1 only itself on entry 1, and so on. 2959 * This is used by cpumask_of() (and possibly others in the future) for, 2960 * e.g., drivers to pass hints to irq_set_affinity_hint(). 2961 */ 2962 static_single_cpu_mask = kmalloc_array(mp_maxid + 1, 2963 sizeof(static_single_cpu_mask), M_WAITOK | M_ZERO); 2964 2965 /* 2966 * When the number of CPUs reach a threshold, we start to save memory 2967 * given the sets are static by overlapping those having their single 2968 * bit set at same position in a bitset word. Asymptotically, this 2969 * regular scheme is in O(n²) whereas the overlapping one is in O(n) 2970 * only with n being the maximum number of CPUs, so the gain will become 2971 * huge quite quickly. The threshold for 64-bit architectures is 128 2972 * CPUs. 2973 */ 2974 if (mp_ncpus < (2 * _BITSET_BITS)) { 2975 cpumask_t *sscm_ptr; 2976 2977 /* 2978 * This represents 'mp_ncpus * __bitset_words(CPU_SETSIZE) * 2979 * (_BITSET_BITS / 8)' bytes (for comparison with the 2980 * overlapping scheme). 2981 */ 2982 static_single_cpu_mask_lcs = kmalloc_array(mp_ncpus, 2983 sizeof(*static_single_cpu_mask_lcs), 2984 M_WAITOK | M_ZERO); 2985 2986 sscm_ptr = static_single_cpu_mask_lcs; 2987 CPU_FOREACH(i) { 2988 static_single_cpu_mask[i] = sscm_ptr++; 2989 CPU_SET(i, static_single_cpu_mask[i]); 2990 } 2991 } else { 2992 /* Pointer to a bitset word. */ 2993 __typeof(((cpuset_t *)NULL)->__bits[0]) *bwp; 2994 2995 /* 2996 * Allocate memory for (static) spans of 'cpumask_t' ('cpuset_t' 2997 * really) with a single bit set that can be reused for all 2998 * single CPU masks by making them start at different offsets. 2999 * We need '__bitset_words(CPU_SETSIZE) - 1' bitset words before 3000 * the word having its single bit set, and the same amount 3001 * after. 3002 */ 3003 static_single_cpu_mask_lcs = mallocarray(_BITSET_BITS, 3004 (2 * __bitset_words(CPU_SETSIZE) - 1) * (_BITSET_BITS / 8), 3005 M_KMALLOC, M_WAITOK | M_ZERO); 3006 3007 /* 3008 * We rely below on cpuset_t and the bitset generic 3009 * implementation assigning words in the '__bits' array in the 3010 * same order of bits (i.e., little-endian ordering, not to be 3011 * confused with machine endianness, which concerns bits in 3012 * words and other integers). This is an imperfect test, but it 3013 * will detect a change to big-endian ordering. 3014 */ 3015 _Static_assert( 3016 __bitset_word(_BITSET_BITS + 1, _BITSET_BITS) == 1, 3017 "Assumes a bitset implementation that is little-endian " 3018 "on its words"); 3019 3020 /* Initialize the single bit of each static span. */ 3021 bwp = (__typeof(bwp))static_single_cpu_mask_lcs + 3022 (__bitset_words(CPU_SETSIZE) - 1); 3023 for (i = 0; i < _BITSET_BITS; i++) { 3024 CPU_SET(i, (cpuset_t *)bwp); 3025 bwp += (2 * __bitset_words(CPU_SETSIZE) - 1); 3026 } 3027 3028 /* 3029 * Finally set all CPU masks to the proper word in their 3030 * relevant span. 3031 */ 3032 CPU_FOREACH(i) { 3033 bwp = (__typeof(bwp))static_single_cpu_mask_lcs; 3034 /* Find the non-zero word of the relevant span. */ 3035 bwp += (2 * __bitset_words(CPU_SETSIZE) - 1) * 3036 (i % _BITSET_BITS) + 3037 __bitset_words(CPU_SETSIZE) - 1; 3038 /* Shift to find the CPU mask start. */ 3039 bwp -= (i / _BITSET_BITS); 3040 static_single_cpu_mask[i] = (cpuset_t *)bwp; 3041 } 3042 } 3043 3044 strlcpy(init_uts_ns.name.release, osrelease, sizeof(init_uts_ns.name.release)); 3045 } 3046 SYSINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_init, NULL); 3047 3048 static void 3049 linux_compat_uninit(void *arg) 3050 { 3051 linux_kobject_kfree_name(&linux_class_root); 3052 linux_kobject_kfree_name(&linux_root_device.kobj); 3053 linux_kobject_kfree_name(&linux_class_misc.kobj); 3054 3055 free(static_single_cpu_mask_lcs, M_KMALLOC); 3056 free(static_single_cpu_mask, M_KMALLOC); 3057 #if defined(__i386__) || defined(__amd64__) 3058 free(__cpu_data, M_KMALLOC); 3059 #endif 3060 3061 spin_lock_destroy(&pci_lock); 3062 rw_destroy(&linux_vma_lock); 3063 } 3064 SYSUNINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_uninit, NULL); 3065 3066 /* 3067 * NOTE: Linux frequently uses "unsigned long" for pointer to integer 3068 * conversion and vice versa, where in FreeBSD "uintptr_t" would be 3069 * used. Assert these types have the same size, else some parts of the 3070 * LinuxKPI may not work like expected: 3071 */ 3072 CTASSERT(sizeof(unsigned long) == sizeof(uintptr_t)); 3073