1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013-2021 Mellanox Technologies, Ltd. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice unmodified, this list of conditions, and the following 13 * disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 #include "opt_global.h" 32 #include "opt_stack.h" 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/malloc.h> 37 #include <sys/kernel.h> 38 #include <sys/sysctl.h> 39 #include <sys/proc.h> 40 #include <sys/sglist.h> 41 #include <sys/sleepqueue.h> 42 #include <sys/refcount.h> 43 #include <sys/lock.h> 44 #include <sys/mutex.h> 45 #include <sys/bus.h> 46 #include <sys/eventhandler.h> 47 #include <sys/fcntl.h> 48 #include <sys/file.h> 49 #include <sys/filio.h> 50 #include <sys/rwlock.h> 51 #include <sys/mman.h> 52 #include <sys/stack.h> 53 #include <sys/stdarg.h> 54 #include <sys/syscall.h> 55 #include <sys/sysent.h> 56 #include <sys/time.h> 57 #include <sys/user.h> 58 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #include <vm/vm_object.h> 62 #include <vm/vm_page.h> 63 #include <vm/vm_pager.h> 64 #include <vm/vm_radix.h> 65 66 #if defined(__i386__) || defined(__amd64__) 67 #include <machine/cputypes.h> 68 #include <machine/md_var.h> 69 #endif 70 71 #include <linux/kobject.h> 72 #include <linux/cpu.h> 73 #include <linux/device.h> 74 #include <linux/slab.h> 75 #include <linux/module.h> 76 #include <linux/moduleparam.h> 77 #include <linux/cdev.h> 78 #include <linux/file.h> 79 #include <linux/fs.h> 80 #include <linux/sysfs.h> 81 #include <linux/mm.h> 82 #include <linux/io.h> 83 #include <linux/vmalloc.h> 84 #include <linux/netdevice.h> 85 #include <linux/timer.h> 86 #include <linux/interrupt.h> 87 #include <linux/uaccess.h> 88 #include <linux/utsname.h> 89 #include <linux/list.h> 90 #include <linux/kthread.h> 91 #include <linux/kernel.h> 92 #include <linux/compat.h> 93 #include <linux/io-mapping.h> 94 #include <linux/poll.h> 95 #include <linux/smp.h> 96 #include <linux/wait_bit.h> 97 #include <linux/rcupdate.h> 98 #include <linux/interval_tree.h> 99 #include <linux/interval_tree_generic.h> 100 #include <linux/printk.h> 101 #include <linux/seq_file.h> 102 #include <linux/uuid.h> 103 104 #if defined(__i386__) || defined(__amd64__) 105 #include <asm/smp.h> 106 #include <asm/processor.h> 107 #endif 108 109 #include <xen/xen.h> 110 #ifdef XENHVM 111 #undef xen_pv_domain 112 #undef xen_initial_domain 113 /* xen/xen-os.h redefines __must_check */ 114 #undef __must_check 115 #include <xen/xen-os.h> 116 #endif 117 118 SYSCTL_NODE(_compat, OID_AUTO, linuxkpi, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 119 "LinuxKPI parameters"); 120 121 int linuxkpi_debug; 122 SYSCTL_INT(_compat_linuxkpi, OID_AUTO, debug, CTLFLAG_RWTUN, 123 &linuxkpi_debug, 0, "Set to enable pr_debug() prints. Clear to disable."); 124 125 int linuxkpi_rcu_debug; 126 SYSCTL_INT(_compat_linuxkpi, OID_AUTO, rcu_debug, CTLFLAG_RWTUN, 127 &linuxkpi_rcu_debug, 0, "Set to enable RCU warning. Clear to disable."); 128 129 int linuxkpi_warn_dump_stack = 0; 130 SYSCTL_INT(_compat_linuxkpi, OID_AUTO, warn_dump_stack, CTLFLAG_RWTUN, 131 &linuxkpi_warn_dump_stack, 0, 132 "Set to enable stack traces from WARN_ON(). Clear to disable."); 133 134 static struct timeval lkpi_net_lastlog; 135 static int lkpi_net_curpps; 136 static int lkpi_net_maxpps = 99; 137 SYSCTL_INT(_compat_linuxkpi, OID_AUTO, net_ratelimit, CTLFLAG_RWTUN, 138 &lkpi_net_maxpps, 0, "Limit number of LinuxKPI net messages per second."); 139 140 MALLOC_DEFINE(M_KMALLOC, "lkpikmalloc", "Linux kmalloc compat"); 141 142 #include <linux/rbtree.h> 143 /* Undo Linux compat changes. */ 144 #undef RB_ROOT 145 #undef file 146 #undef cdev 147 #define RB_ROOT(head) (head)->rbh_root 148 149 static void linux_destroy_dev(struct linux_cdev *); 150 static void linux_cdev_deref(struct linux_cdev *ldev); 151 static struct vm_area_struct *linux_cdev_handle_find(void *handle); 152 153 cpumask_t cpu_online_mask; 154 static cpumask_t **static_single_cpu_mask; 155 static cpumask_t *static_single_cpu_mask_lcs; 156 struct kobject linux_class_root; 157 struct device linux_root_device; 158 struct class linux_class_misc; 159 struct list_head pci_drivers; 160 struct list_head pci_devices; 161 spinlock_t pci_lock; 162 struct uts_namespace init_uts_ns; 163 164 unsigned long linux_timer_hz_mask; 165 166 wait_queue_head_t linux_bit_waitq; 167 wait_queue_head_t linux_var_waitq; 168 169 const guid_t guid_null; 170 171 enum system_states system_state = SYSTEM_RUNNING; 172 173 struct task_struct * 174 __lkpi_current(void) 175 { 176 struct thread *td; 177 178 td = curthread; 179 linux_set_current(td); 180 return ((struct task_struct *)td->td_lkpi_task); 181 } 182 183 int 184 panic_cmp(struct rb_node *one, struct rb_node *two) 185 { 186 panic("no cmp"); 187 } 188 189 RB_GENERATE(linux_root, rb_node, __entry, panic_cmp); 190 191 #define START(node) ((node)->start) 192 #define LAST(node) ((node)->last) 193 194 INTERVAL_TREE_DEFINE(struct interval_tree_node, rb, unsigned long,, START, 195 LAST,, lkpi_interval_tree) 196 197 static void 198 linux_device_release(struct device *dev) 199 { 200 pr_debug("linux_device_release: %s\n", dev_name(dev)); 201 kfree(dev); 202 } 203 204 static ssize_t 205 linux_class_show(struct kobject *kobj, struct attribute *attr, char *buf) 206 { 207 struct class_attribute *dattr; 208 ssize_t error; 209 210 dattr = container_of(attr, struct class_attribute, attr); 211 error = -EIO; 212 if (dattr->show) 213 error = dattr->show(container_of(kobj, struct class, kobj), 214 dattr, buf); 215 return (error); 216 } 217 218 static ssize_t 219 linux_class_store(struct kobject *kobj, struct attribute *attr, const char *buf, 220 size_t count) 221 { 222 struct class_attribute *dattr; 223 ssize_t error; 224 225 dattr = container_of(attr, struct class_attribute, attr); 226 error = -EIO; 227 if (dattr->store) 228 error = dattr->store(container_of(kobj, struct class, kobj), 229 dattr, buf, count); 230 return (error); 231 } 232 233 static void 234 linux_class_release(struct kobject *kobj) 235 { 236 struct class *class; 237 238 class = container_of(kobj, struct class, kobj); 239 if (class->class_release) 240 class->class_release(class); 241 } 242 243 static const struct sysfs_ops linux_class_sysfs = { 244 .show = linux_class_show, 245 .store = linux_class_store, 246 }; 247 248 const struct kobj_type linux_class_ktype = { 249 .release = linux_class_release, 250 .sysfs_ops = &linux_class_sysfs 251 }; 252 253 static void 254 linux_dev_release(struct kobject *kobj) 255 { 256 struct device *dev; 257 258 dev = container_of(kobj, struct device, kobj); 259 /* This is the precedence defined by linux. */ 260 if (dev->release) 261 dev->release(dev); 262 else if (dev->class && dev->class->dev_release) 263 dev->class->dev_release(dev); 264 } 265 266 static ssize_t 267 linux_dev_show(struct kobject *kobj, struct attribute *attr, char *buf) 268 { 269 struct device_attribute *dattr; 270 ssize_t error; 271 272 dattr = container_of(attr, struct device_attribute, attr); 273 error = -EIO; 274 if (dattr->show) 275 error = dattr->show(container_of(kobj, struct device, kobj), 276 dattr, buf); 277 return (error); 278 } 279 280 static ssize_t 281 linux_dev_store(struct kobject *kobj, struct attribute *attr, const char *buf, 282 size_t count) 283 { 284 struct device_attribute *dattr; 285 ssize_t error; 286 287 dattr = container_of(attr, struct device_attribute, attr); 288 error = -EIO; 289 if (dattr->store) 290 error = dattr->store(container_of(kobj, struct device, kobj), 291 dattr, buf, count); 292 return (error); 293 } 294 295 static const struct sysfs_ops linux_dev_sysfs = { 296 .show = linux_dev_show, 297 .store = linux_dev_store, 298 }; 299 300 const struct kobj_type linux_dev_ktype = { 301 .release = linux_dev_release, 302 .sysfs_ops = &linux_dev_sysfs 303 }; 304 305 struct device * 306 device_create(struct class *class, struct device *parent, dev_t devt, 307 void *drvdata, const char *fmt, ...) 308 { 309 struct device *dev; 310 va_list args; 311 312 dev = kzalloc(sizeof(*dev), M_WAITOK); 313 dev->parent = parent; 314 dev->class = class; 315 dev->devt = devt; 316 dev->driver_data = drvdata; 317 dev->release = linux_device_release; 318 va_start(args, fmt); 319 kobject_set_name_vargs(&dev->kobj, fmt, args); 320 va_end(args); 321 device_register(dev); 322 323 return (dev); 324 } 325 326 struct device * 327 device_create_groups_vargs(struct class *class, struct device *parent, 328 dev_t devt, void *drvdata, const struct attribute_group **groups, 329 const char *fmt, va_list args) 330 { 331 struct device *dev = NULL; 332 int retval = -ENODEV; 333 334 if (class == NULL || IS_ERR(class)) 335 goto error; 336 337 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 338 if (!dev) { 339 retval = -ENOMEM; 340 goto error; 341 } 342 343 dev->devt = devt; 344 dev->class = class; 345 dev->parent = parent; 346 dev->groups = groups; 347 dev->release = device_create_release; 348 /* device_initialize() needs the class and parent to be set */ 349 device_initialize(dev); 350 dev_set_drvdata(dev, drvdata); 351 352 retval = kobject_set_name_vargs(&dev->kobj, fmt, args); 353 if (retval) 354 goto error; 355 356 retval = device_add(dev); 357 if (retval) 358 goto error; 359 360 return dev; 361 362 error: 363 put_device(dev); 364 return ERR_PTR(retval); 365 } 366 367 struct class * 368 lkpi_class_create(const char *name) 369 { 370 struct class *class; 371 int error; 372 373 class = kzalloc(sizeof(*class), M_WAITOK); 374 class->name = name; 375 class->class_release = linux_class_kfree; 376 error = class_register(class); 377 if (error) { 378 kfree(class); 379 return (NULL); 380 } 381 382 return (class); 383 } 384 385 static void 386 linux_kq_lock(void *arg) 387 { 388 spinlock_t *s = arg; 389 390 spin_lock(s); 391 } 392 static void 393 linux_kq_unlock(void *arg) 394 { 395 spinlock_t *s = arg; 396 397 spin_unlock(s); 398 } 399 400 static void 401 linux_kq_assert_lock(void *arg, int what) 402 { 403 #ifdef INVARIANTS 404 spinlock_t *s = arg; 405 406 if (what == LA_LOCKED) 407 mtx_assert(s, MA_OWNED); 408 else 409 mtx_assert(s, MA_NOTOWNED); 410 #endif 411 } 412 413 static void 414 linux_file_kqfilter_poll(struct linux_file *, int); 415 416 struct linux_file * 417 linux_file_alloc(void) 418 { 419 struct linux_file *filp; 420 421 filp = kzalloc(sizeof(*filp), GFP_KERNEL); 422 423 /* set initial refcount */ 424 filp->f_count = 1; 425 426 /* setup fields needed by kqueue support */ 427 spin_lock_init(&filp->f_kqlock); 428 knlist_init(&filp->f_selinfo.si_note, &filp->f_kqlock, 429 linux_kq_lock, linux_kq_unlock, linux_kq_assert_lock); 430 431 return (filp); 432 } 433 434 void 435 linux_file_free(struct linux_file *filp) 436 { 437 if (filp->_file == NULL) { 438 if (filp->f_op != NULL && filp->f_op->release != NULL) 439 filp->f_op->release(filp->f_vnode, filp); 440 if (filp->f_shmem != NULL) 441 vm_object_deallocate(filp->f_shmem); 442 kfree_rcu(filp, rcu); 443 } else { 444 /* 445 * The close method of the character device or file 446 * will free the linux_file structure: 447 */ 448 _fdrop(filp->_file, curthread); 449 } 450 } 451 452 struct linux_cdev * 453 cdev_alloc(void) 454 { 455 struct linux_cdev *cdev; 456 457 cdev = kzalloc(sizeof(struct linux_cdev), M_WAITOK); 458 kobject_init(&cdev->kobj, &linux_cdev_ktype); 459 cdev->refs = 1; 460 return (cdev); 461 } 462 463 static int 464 linux_cdev_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, 465 vm_page_t *mres) 466 { 467 struct vm_area_struct *vmap; 468 469 vmap = linux_cdev_handle_find(vm_obj->handle); 470 471 MPASS(vmap != NULL); 472 MPASS(vmap->vm_private_data == vm_obj->handle); 473 474 if (likely(vmap->vm_ops != NULL && offset < vmap->vm_len)) { 475 vm_paddr_t paddr = IDX_TO_OFF(vmap->vm_pfn) + offset; 476 vm_page_t page; 477 478 if (((*mres)->flags & PG_FICTITIOUS) != 0) { 479 /* 480 * If the passed in result page is a fake 481 * page, update it with the new physical 482 * address. 483 */ 484 page = *mres; 485 vm_page_updatefake(page, paddr, vm_obj->memattr); 486 } else { 487 /* 488 * Replace the passed in "mres" page with our 489 * own fake page and free up the all of the 490 * original pages. 491 */ 492 VM_OBJECT_WUNLOCK(vm_obj); 493 page = vm_page_getfake(paddr, vm_obj->memattr); 494 VM_OBJECT_WLOCK(vm_obj); 495 496 vm_page_replace(page, vm_obj, (*mres)->pindex, *mres); 497 *mres = page; 498 } 499 vm_page_valid(page); 500 return (VM_PAGER_OK); 501 } 502 return (VM_PAGER_FAIL); 503 } 504 505 static int 506 linux_cdev_pager_populate(vm_object_t vm_obj, vm_pindex_t pidx, int fault_type, 507 vm_prot_t max_prot, vm_pindex_t *first, vm_pindex_t *last) 508 { 509 struct vm_area_struct *vmap; 510 int err; 511 512 /* get VM area structure */ 513 vmap = linux_cdev_handle_find(vm_obj->handle); 514 MPASS(vmap != NULL); 515 MPASS(vmap->vm_private_data == vm_obj->handle); 516 517 VM_OBJECT_WUNLOCK(vm_obj); 518 519 linux_set_current(curthread); 520 521 down_write(&vmap->vm_mm->mmap_sem); 522 if (unlikely(vmap->vm_ops == NULL)) { 523 err = VM_FAULT_SIGBUS; 524 } else { 525 struct vm_fault vmf; 526 527 /* fill out VM fault structure */ 528 vmf.virtual_address = (void *)(uintptr_t)IDX_TO_OFF(pidx); 529 vmf.flags = (fault_type & VM_PROT_WRITE) ? FAULT_FLAG_WRITE : 0; 530 vmf.pgoff = 0; 531 vmf.page = NULL; 532 vmf.vma = vmap; 533 534 vmap->vm_pfn_count = 0; 535 vmap->vm_pfn_pcount = &vmap->vm_pfn_count; 536 vmap->vm_obj = vm_obj; 537 538 err = vmap->vm_ops->fault(&vmf); 539 540 while (vmap->vm_pfn_count == 0 && err == VM_FAULT_NOPAGE) { 541 kern_yield(PRI_USER); 542 err = vmap->vm_ops->fault(&vmf); 543 } 544 } 545 546 /* translate return code */ 547 switch (err) { 548 case VM_FAULT_OOM: 549 err = VM_PAGER_AGAIN; 550 break; 551 case VM_FAULT_SIGBUS: 552 err = VM_PAGER_BAD; 553 break; 554 case VM_FAULT_NOPAGE: 555 /* 556 * By contract the fault handler will return having 557 * busied all the pages itself. If pidx is already 558 * found in the object, it will simply xbusy the first 559 * page and return with vm_pfn_count set to 1. 560 */ 561 *first = vmap->vm_pfn_first; 562 *last = *first + vmap->vm_pfn_count - 1; 563 MPASS(pidx >= *first); 564 MPASS(pidx <= *last); 565 err = VM_PAGER_OK; 566 break; 567 default: 568 err = VM_PAGER_ERROR; 569 break; 570 } 571 up_write(&vmap->vm_mm->mmap_sem); 572 VM_OBJECT_WLOCK(vm_obj); 573 return (err); 574 } 575 576 static struct rwlock linux_vma_lock; 577 static TAILQ_HEAD(, vm_area_struct) linux_vma_head = 578 TAILQ_HEAD_INITIALIZER(linux_vma_head); 579 580 static void 581 linux_cdev_handle_free(struct vm_area_struct *vmap) 582 { 583 /* Drop reference on vm_file */ 584 if (vmap->vm_file != NULL) 585 fput(vmap->vm_file); 586 587 /* Drop reference on mm_struct */ 588 mmput(vmap->vm_mm); 589 590 kfree(vmap); 591 } 592 593 static void 594 linux_cdev_handle_remove(struct vm_area_struct *vmap) 595 { 596 rw_wlock(&linux_vma_lock); 597 TAILQ_REMOVE(&linux_vma_head, vmap, vm_entry); 598 rw_wunlock(&linux_vma_lock); 599 } 600 601 static struct vm_area_struct * 602 linux_cdev_handle_find(void *handle) 603 { 604 struct vm_area_struct *vmap; 605 606 rw_rlock(&linux_vma_lock); 607 TAILQ_FOREACH(vmap, &linux_vma_head, vm_entry) { 608 if (vmap->vm_private_data == handle) 609 break; 610 } 611 rw_runlock(&linux_vma_lock); 612 return (vmap); 613 } 614 615 static int 616 linux_cdev_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 617 vm_ooffset_t foff, struct ucred *cred, u_short *color) 618 { 619 620 MPASS(linux_cdev_handle_find(handle) != NULL); 621 *color = 0; 622 return (0); 623 } 624 625 static void 626 linux_cdev_pager_dtor(void *handle) 627 { 628 const struct vm_operations_struct *vm_ops; 629 struct vm_area_struct *vmap; 630 631 vmap = linux_cdev_handle_find(handle); 632 MPASS(vmap != NULL); 633 634 /* 635 * Remove handle before calling close operation to prevent 636 * other threads from reusing the handle pointer. 637 */ 638 linux_cdev_handle_remove(vmap); 639 640 down_write(&vmap->vm_mm->mmap_sem); 641 vm_ops = vmap->vm_ops; 642 if (likely(vm_ops != NULL)) 643 vm_ops->close(vmap); 644 up_write(&vmap->vm_mm->mmap_sem); 645 646 linux_cdev_handle_free(vmap); 647 } 648 649 static struct cdev_pager_ops linux_cdev_pager_ops[2] = { 650 { 651 /* OBJT_MGTDEVICE */ 652 .cdev_pg_populate = linux_cdev_pager_populate, 653 .cdev_pg_ctor = linux_cdev_pager_ctor, 654 .cdev_pg_dtor = linux_cdev_pager_dtor 655 }, 656 { 657 /* OBJT_DEVICE */ 658 .cdev_pg_fault = linux_cdev_pager_fault, 659 .cdev_pg_ctor = linux_cdev_pager_ctor, 660 .cdev_pg_dtor = linux_cdev_pager_dtor 661 }, 662 }; 663 664 int 665 zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, 666 unsigned long size) 667 { 668 struct pctrie_iter pages; 669 vm_object_t obj; 670 vm_page_t m; 671 672 obj = vma->vm_obj; 673 if (obj == NULL || (obj->flags & OBJ_UNMANAGED) != 0) 674 return (-ENOTSUP); 675 VM_OBJECT_RLOCK(obj); 676 vm_page_iter_limit_init(&pages, obj, OFF_TO_IDX(address + size)); 677 VM_RADIX_FOREACH_FROM(m, &pages, OFF_TO_IDX(address)) 678 pmap_remove_all(m); 679 VM_OBJECT_RUNLOCK(obj); 680 return (0); 681 } 682 683 void 684 vma_set_file(struct vm_area_struct *vma, struct linux_file *file) 685 { 686 struct linux_file *tmp; 687 688 /* Changing an anonymous vma with this is illegal */ 689 get_file(file); 690 tmp = vma->vm_file; 691 vma->vm_file = file; 692 fput(tmp); 693 } 694 695 static struct file_operations dummy_ldev_ops = { 696 /* XXXKIB */ 697 }; 698 699 static struct linux_cdev dummy_ldev = { 700 .ops = &dummy_ldev_ops, 701 }; 702 703 #define LDEV_SI_DTR 0x0001 704 #define LDEV_SI_REF 0x0002 705 706 static void 707 linux_get_fop(struct linux_file *filp, const struct file_operations **fop, 708 struct linux_cdev **dev) 709 { 710 struct linux_cdev *ldev; 711 u_int siref; 712 713 ldev = filp->f_cdev; 714 *fop = filp->f_op; 715 if (ldev != NULL) { 716 if (ldev->kobj.ktype == &linux_cdev_static_ktype) { 717 refcount_acquire(&ldev->refs); 718 } else { 719 for (siref = ldev->siref;;) { 720 if ((siref & LDEV_SI_DTR) != 0) { 721 ldev = &dummy_ldev; 722 *fop = ldev->ops; 723 siref = ldev->siref; 724 MPASS((ldev->siref & LDEV_SI_DTR) == 0); 725 } else if (atomic_fcmpset_int(&ldev->siref, 726 &siref, siref + LDEV_SI_REF)) { 727 break; 728 } 729 } 730 } 731 } 732 *dev = ldev; 733 } 734 735 static void 736 linux_drop_fop(struct linux_cdev *ldev) 737 { 738 739 if (ldev == NULL) 740 return; 741 if (ldev->kobj.ktype == &linux_cdev_static_ktype) { 742 linux_cdev_deref(ldev); 743 } else { 744 MPASS(ldev->kobj.ktype == &linux_cdev_ktype); 745 MPASS((ldev->siref & ~LDEV_SI_DTR) != 0); 746 atomic_subtract_int(&ldev->siref, LDEV_SI_REF); 747 } 748 } 749 750 #define OPW(fp,td,code) ({ \ 751 struct file *__fpop; \ 752 __typeof(code) __retval; \ 753 \ 754 __fpop = (td)->td_fpop; \ 755 (td)->td_fpop = (fp); \ 756 __retval = (code); \ 757 (td)->td_fpop = __fpop; \ 758 __retval; \ 759 }) 760 761 static int 762 linux_dev_fdopen(struct cdev *dev, int fflags, struct thread *td, 763 struct file *file) 764 { 765 struct linux_cdev *ldev; 766 struct linux_file *filp; 767 const struct file_operations *fop; 768 int error; 769 770 ldev = dev->si_drv1; 771 772 filp = linux_file_alloc(); 773 filp->f_dentry = &filp->f_dentry_store; 774 filp->f_op = ldev->ops; 775 filp->f_mode = file->f_flag; 776 filp->f_flags = file->f_flag; 777 filp->f_vnode = file->f_vnode; 778 filp->_file = file; 779 refcount_acquire(&ldev->refs); 780 filp->f_cdev = ldev; 781 782 linux_set_current(td); 783 linux_get_fop(filp, &fop, &ldev); 784 785 if (fop->open != NULL) { 786 error = -fop->open(file->f_vnode, filp); 787 if (error != 0) { 788 linux_drop_fop(ldev); 789 linux_cdev_deref(filp->f_cdev); 790 kfree(filp); 791 return (error); 792 } 793 } 794 795 /* hold on to the vnode - used for fstat() */ 796 vref(filp->f_vnode); 797 798 /* release the file from devfs */ 799 finit(file, filp->f_mode, DTYPE_DEV, filp, &linuxfileops); 800 linux_drop_fop(ldev); 801 return (ENXIO); 802 } 803 804 #define LINUX_IOCTL_MIN_PTR 0x10000UL 805 #define LINUX_IOCTL_MAX_PTR (LINUX_IOCTL_MIN_PTR + IOCPARM_MAX) 806 807 static inline int 808 linux_remap_address(void **uaddr, size_t len) 809 { 810 uintptr_t uaddr_val = (uintptr_t)(*uaddr); 811 812 if (unlikely(uaddr_val >= LINUX_IOCTL_MIN_PTR && 813 uaddr_val < LINUX_IOCTL_MAX_PTR)) { 814 struct task_struct *pts = current; 815 if (pts == NULL) { 816 *uaddr = NULL; 817 return (1); 818 } 819 820 /* compute data offset */ 821 uaddr_val -= LINUX_IOCTL_MIN_PTR; 822 823 /* check that length is within bounds */ 824 if ((len > IOCPARM_MAX) || 825 (uaddr_val + len) > pts->bsd_ioctl_len) { 826 *uaddr = NULL; 827 return (1); 828 } 829 830 /* re-add kernel buffer address */ 831 uaddr_val += (uintptr_t)pts->bsd_ioctl_data; 832 833 /* update address location */ 834 *uaddr = (void *)uaddr_val; 835 return (1); 836 } 837 return (0); 838 } 839 840 int 841 linux_copyin(const void *uaddr, void *kaddr, size_t len) 842 { 843 if (linux_remap_address(__DECONST(void **, &uaddr), len)) { 844 if (uaddr == NULL) 845 return (-EFAULT); 846 memcpy(kaddr, uaddr, len); 847 return (0); 848 } 849 return (-copyin(uaddr, kaddr, len)); 850 } 851 852 int 853 linux_copyout(const void *kaddr, void *uaddr, size_t len) 854 { 855 if (linux_remap_address(&uaddr, len)) { 856 if (uaddr == NULL) 857 return (-EFAULT); 858 memcpy(uaddr, kaddr, len); 859 return (0); 860 } 861 return (-copyout(kaddr, uaddr, len)); 862 } 863 864 size_t 865 linux_clear_user(void *_uaddr, size_t _len) 866 { 867 uint8_t *uaddr = _uaddr; 868 size_t len = _len; 869 870 /* make sure uaddr is aligned before going into the fast loop */ 871 while (((uintptr_t)uaddr & 7) != 0 && len > 7) { 872 if (subyte(uaddr, 0)) 873 return (_len); 874 uaddr++; 875 len--; 876 } 877 878 /* zero 8 bytes at a time */ 879 while (len > 7) { 880 #ifdef __LP64__ 881 if (suword64(uaddr, 0)) 882 return (_len); 883 #else 884 if (suword32(uaddr, 0)) 885 return (_len); 886 if (suword32(uaddr + 4, 0)) 887 return (_len); 888 #endif 889 uaddr += 8; 890 len -= 8; 891 } 892 893 /* zero fill end, if any */ 894 while (len > 0) { 895 if (subyte(uaddr, 0)) 896 return (_len); 897 uaddr++; 898 len--; 899 } 900 return (0); 901 } 902 903 int 904 linux_access_ok(const void *uaddr, size_t len) 905 { 906 uintptr_t saddr; 907 uintptr_t eaddr; 908 909 /* get start and end address */ 910 saddr = (uintptr_t)uaddr; 911 eaddr = (uintptr_t)uaddr + len; 912 913 /* verify addresses are valid for userspace */ 914 return ((saddr == eaddr) || 915 (eaddr > saddr && eaddr <= VM_MAXUSER_ADDRESS)); 916 } 917 918 /* 919 * This function should return either EINTR or ERESTART depending on 920 * the signal type sent to this thread: 921 */ 922 static int 923 linux_get_error(struct task_struct *task, int error) 924 { 925 /* check for signal type interrupt code */ 926 if (error == EINTR || error == ERESTARTSYS || error == ERESTART) { 927 error = -linux_schedule_get_interrupt_value(task); 928 if (error == 0) 929 error = EINTR; 930 } 931 return (error); 932 } 933 934 static int 935 linux_file_ioctl_sub(struct file *fp, struct linux_file *filp, 936 const struct file_operations *fop, u_long cmd, caddr_t data, 937 struct thread *td) 938 { 939 struct task_struct *task = current; 940 unsigned size; 941 int error; 942 bool direct; 943 944 size = IOCPARM_LEN(cmd); 945 /* refer to logic in sys_ioctl() */ 946 direct = false; 947 if (size > 0) { 948 /* 949 * Setup hint for linux_copyin() and linux_copyout(). 950 * 951 * Background: Linux kernel code expects to operate on 952 * userspace addresses, but FreeBSD's kern_ioctl() 953 * will generally provide a kernel address. For the 954 * native process ABI, where we know how to find the 955 * original address, we reach directly into the system 956 * call args to get it. Then, if the Linux driver 957 * copied out to that address, we copy the whole block 958 * back into the kernel buffer allocated by 959 * kern_ioctl() so that kern_ioctl() itself doesn't 960 * clobber the driver's data. 961 * 962 * Otherwise, fall back to the LINUX_IOCTL_MIN_PTR 963 * hack. 964 */ 965 task->bsd_ioctl_data = data; 966 task->bsd_ioctl_len = size; 967 if ((td->td_pflags & TDP_KTHREAD) == 0 && 968 SV_PROC_ABI(td->td_proc) == SV_ABI_FREEBSD && 969 td->td_sa.code == SYS_ioctl) { 970 direct = true; 971 data = (void *)(uintptr_t)td->td_sa.args[2]; 972 } else { 973 data = (void *)LINUX_IOCTL_MIN_PTR; 974 } 975 } else { 976 /* fetch user-space pointer */ 977 data = *(void **)data; 978 } 979 #ifdef COMPAT_FREEBSD32 980 if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) { 981 /* try the compat IOCTL handler first */ 982 if (fop->compat_ioctl != NULL) { 983 error = -OPW(fp, td, fop->compat_ioctl(filp, 984 cmd, (u_long)data)); 985 } else { 986 error = ENOTTY; 987 } 988 989 /* fallback to the regular IOCTL handler, if any */ 990 if (error == ENOTTY && fop->unlocked_ioctl != NULL) { 991 error = -OPW(fp, td, fop->unlocked_ioctl(filp, 992 cmd, (u_long)data)); 993 } 994 } else 995 #endif 996 { 997 if (fop->unlocked_ioctl != NULL) { 998 error = -OPW(fp, td, fop->unlocked_ioctl(filp, 999 cmd, (u_long)data)); 1000 } else { 1001 error = ENOTTY; 1002 } 1003 } 1004 if (error == 0 && size > 0 && (cmd & IOC_OUT) != 0 && direct) { 1005 void *xdata; 1006 int error1; 1007 1008 /* 1009 * Ensure that the copyout in sys_generic.c copies 1010 * over the data which is possibly modified by the 1011 * driver. A possible error from the copyin() is 1012 * ignored since it is formally possible for the memory 1013 * to become unaccessible in the meantime. Do the copying 1014 * through the intermediate buffer instead of copying 1015 * directly to bsd_ioctl_data, to ensure atomicity of 1016 * the change with respect to the error. 1017 */ 1018 xdata = malloc(size, M_TEMP, M_WAITOK); 1019 error1 = copyin(data, xdata, size); 1020 if (error1 == 0) 1021 memcpy(task->bsd_ioctl_data, xdata, size); 1022 free(xdata, M_TEMP); 1023 } 1024 if (size > 0) { 1025 task->bsd_ioctl_data = NULL; 1026 task->bsd_ioctl_len = 0; 1027 } 1028 if (error == EWOULDBLOCK) { 1029 /* update kqfilter status, if any */ 1030 linux_file_kqfilter_poll(filp, 1031 LINUX_KQ_FLAG_HAS_READ | LINUX_KQ_FLAG_HAS_WRITE); 1032 } else { 1033 error = linux_get_error(task, error); 1034 } 1035 return (error); 1036 } 1037 1038 #define LINUX_POLL_TABLE_NORMAL ((poll_table *)1) 1039 1040 /* 1041 * This function atomically updates the poll wakeup state and returns 1042 * the previous state at the time of update. 1043 */ 1044 static uint8_t 1045 linux_poll_wakeup_state(atomic_t *v, const uint8_t *pstate) 1046 { 1047 int c, old; 1048 1049 c = v->counter; 1050 1051 while ((old = atomic_cmpxchg(v, c, pstate[c])) != c) 1052 c = old; 1053 1054 return (c); 1055 } 1056 1057 static int 1058 linux_poll_wakeup_callback(wait_queue_t *wq, unsigned int wq_state, int flags, void *key) 1059 { 1060 static const uint8_t state[LINUX_FWQ_STATE_MAX] = { 1061 [LINUX_FWQ_STATE_INIT] = LINUX_FWQ_STATE_INIT, /* NOP */ 1062 [LINUX_FWQ_STATE_NOT_READY] = LINUX_FWQ_STATE_NOT_READY, /* NOP */ 1063 [LINUX_FWQ_STATE_QUEUED] = LINUX_FWQ_STATE_READY, 1064 [LINUX_FWQ_STATE_READY] = LINUX_FWQ_STATE_READY, /* NOP */ 1065 }; 1066 struct linux_file *filp = container_of(wq, struct linux_file, f_wait_queue.wq); 1067 1068 switch (linux_poll_wakeup_state(&filp->f_wait_queue.state, state)) { 1069 case LINUX_FWQ_STATE_QUEUED: 1070 linux_poll_wakeup(filp); 1071 return (1); 1072 default: 1073 return (0); 1074 } 1075 } 1076 1077 void 1078 linux_poll_wait(struct linux_file *filp, wait_queue_head_t *wqh, poll_table *p) 1079 { 1080 static const uint8_t state[LINUX_FWQ_STATE_MAX] = { 1081 [LINUX_FWQ_STATE_INIT] = LINUX_FWQ_STATE_NOT_READY, 1082 [LINUX_FWQ_STATE_NOT_READY] = LINUX_FWQ_STATE_NOT_READY, /* NOP */ 1083 [LINUX_FWQ_STATE_QUEUED] = LINUX_FWQ_STATE_QUEUED, /* NOP */ 1084 [LINUX_FWQ_STATE_READY] = LINUX_FWQ_STATE_QUEUED, 1085 }; 1086 1087 /* check if we are called inside the select system call */ 1088 if (p == LINUX_POLL_TABLE_NORMAL) 1089 selrecord(curthread, &filp->f_selinfo); 1090 1091 switch (linux_poll_wakeup_state(&filp->f_wait_queue.state, state)) { 1092 case LINUX_FWQ_STATE_INIT: 1093 /* NOTE: file handles can only belong to one wait-queue */ 1094 filp->f_wait_queue.wqh = wqh; 1095 filp->f_wait_queue.wq.func = &linux_poll_wakeup_callback; 1096 add_wait_queue(wqh, &filp->f_wait_queue.wq); 1097 atomic_set(&filp->f_wait_queue.state, LINUX_FWQ_STATE_QUEUED); 1098 break; 1099 default: 1100 break; 1101 } 1102 } 1103 1104 static void 1105 linux_poll_wait_dequeue(struct linux_file *filp) 1106 { 1107 static const uint8_t state[LINUX_FWQ_STATE_MAX] = { 1108 [LINUX_FWQ_STATE_INIT] = LINUX_FWQ_STATE_INIT, /* NOP */ 1109 [LINUX_FWQ_STATE_NOT_READY] = LINUX_FWQ_STATE_INIT, 1110 [LINUX_FWQ_STATE_QUEUED] = LINUX_FWQ_STATE_INIT, 1111 [LINUX_FWQ_STATE_READY] = LINUX_FWQ_STATE_INIT, 1112 }; 1113 1114 seldrain(&filp->f_selinfo); 1115 1116 switch (linux_poll_wakeup_state(&filp->f_wait_queue.state, state)) { 1117 case LINUX_FWQ_STATE_NOT_READY: 1118 case LINUX_FWQ_STATE_QUEUED: 1119 case LINUX_FWQ_STATE_READY: 1120 remove_wait_queue(filp->f_wait_queue.wqh, &filp->f_wait_queue.wq); 1121 break; 1122 default: 1123 break; 1124 } 1125 } 1126 1127 void 1128 linux_poll_wakeup(struct linux_file *filp) 1129 { 1130 /* this function should be NULL-safe */ 1131 if (filp == NULL) 1132 return; 1133 1134 selwakeup(&filp->f_selinfo); 1135 1136 spin_lock(&filp->f_kqlock); 1137 filp->f_kqflags |= LINUX_KQ_FLAG_NEED_READ | 1138 LINUX_KQ_FLAG_NEED_WRITE; 1139 1140 /* make sure the "knote" gets woken up */ 1141 KNOTE_LOCKED(&filp->f_selinfo.si_note, 1); 1142 spin_unlock(&filp->f_kqlock); 1143 } 1144 1145 static struct linux_file * 1146 __get_file_rcu(struct linux_file **f) 1147 { 1148 struct linux_file *file1, *file2; 1149 1150 file1 = READ_ONCE(*f); 1151 if (file1 == NULL) 1152 return (NULL); 1153 1154 if (!refcount_acquire_if_not_zero( 1155 file1->_file == NULL ? &file1->f_count : &file1->_file->f_count)) 1156 return (ERR_PTR(-EAGAIN)); 1157 1158 file2 = READ_ONCE(*f); 1159 if (file2 == file1) 1160 return (file2); 1161 1162 fput(file1); 1163 return (ERR_PTR(-EAGAIN)); 1164 } 1165 1166 struct linux_file * 1167 linux_get_file_rcu(struct linux_file **f) 1168 { 1169 struct linux_file *file1; 1170 1171 for (;;) { 1172 file1 = __get_file_rcu(f); 1173 if (file1 == NULL) 1174 return (NULL); 1175 1176 if (IS_ERR(file1)) 1177 continue; 1178 1179 return (file1); 1180 } 1181 } 1182 1183 struct linux_file * 1184 get_file_active(struct linux_file **f) 1185 { 1186 struct linux_file *file1; 1187 1188 rcu_read_lock(); 1189 file1 = __get_file_rcu(f); 1190 rcu_read_unlock(); 1191 if (IS_ERR(file1)) 1192 file1 = NULL; 1193 1194 return (file1); 1195 } 1196 1197 static void 1198 linux_file_kqfilter_detach(struct knote *kn) 1199 { 1200 struct linux_file *filp = kn->kn_hook; 1201 1202 spin_lock(&filp->f_kqlock); 1203 knlist_remove(&filp->f_selinfo.si_note, kn, 1); 1204 spin_unlock(&filp->f_kqlock); 1205 } 1206 1207 static int 1208 linux_file_kqfilter_read_event(struct knote *kn, long hint) 1209 { 1210 struct linux_file *filp = kn->kn_hook; 1211 1212 mtx_assert(&filp->f_kqlock, MA_OWNED); 1213 1214 return ((filp->f_kqflags & LINUX_KQ_FLAG_NEED_READ) ? 1 : 0); 1215 } 1216 1217 static int 1218 linux_file_kqfilter_write_event(struct knote *kn, long hint) 1219 { 1220 struct linux_file *filp = kn->kn_hook; 1221 1222 mtx_assert(&filp->f_kqlock, MA_OWNED); 1223 1224 return ((filp->f_kqflags & LINUX_KQ_FLAG_NEED_WRITE) ? 1 : 0); 1225 } 1226 1227 static const struct filterops linux_dev_kqfiltops_read = { 1228 .f_isfd = 1, 1229 .f_detach = linux_file_kqfilter_detach, 1230 .f_event = linux_file_kqfilter_read_event, 1231 .f_copy = knote_triv_copy, 1232 }; 1233 1234 static const struct filterops linux_dev_kqfiltops_write = { 1235 .f_isfd = 1, 1236 .f_detach = linux_file_kqfilter_detach, 1237 .f_event = linux_file_kqfilter_write_event, 1238 .f_copy = knote_triv_copy, 1239 }; 1240 1241 static void 1242 linux_file_kqfilter_poll(struct linux_file *filp, int kqflags) 1243 { 1244 struct thread *td; 1245 const struct file_operations *fop; 1246 struct linux_cdev *ldev; 1247 int temp; 1248 1249 if ((filp->f_kqflags & kqflags) == 0) 1250 return; 1251 1252 td = curthread; 1253 1254 linux_get_fop(filp, &fop, &ldev); 1255 /* get the latest polling state */ 1256 temp = OPW(filp->_file, td, fop->poll(filp, NULL)); 1257 linux_drop_fop(ldev); 1258 1259 spin_lock(&filp->f_kqlock); 1260 /* clear kqflags */ 1261 filp->f_kqflags &= ~(LINUX_KQ_FLAG_NEED_READ | 1262 LINUX_KQ_FLAG_NEED_WRITE); 1263 /* update kqflags */ 1264 if ((temp & (POLLIN | POLLOUT)) != 0) { 1265 if ((temp & POLLIN) != 0) 1266 filp->f_kqflags |= LINUX_KQ_FLAG_NEED_READ; 1267 if ((temp & POLLOUT) != 0) 1268 filp->f_kqflags |= LINUX_KQ_FLAG_NEED_WRITE; 1269 1270 /* make sure the "knote" gets woken up */ 1271 KNOTE_LOCKED(&filp->f_selinfo.si_note, 0); 1272 } 1273 spin_unlock(&filp->f_kqlock); 1274 } 1275 1276 static int 1277 linux_file_kqfilter(struct file *file, struct knote *kn) 1278 { 1279 struct linux_file *filp; 1280 struct thread *td; 1281 int error; 1282 1283 td = curthread; 1284 filp = (struct linux_file *)file->f_data; 1285 filp->f_flags = file->f_flag; 1286 if (filp->f_op->poll == NULL) 1287 return (EINVAL); 1288 1289 spin_lock(&filp->f_kqlock); 1290 switch (kn->kn_filter) { 1291 case EVFILT_READ: 1292 filp->f_kqflags |= LINUX_KQ_FLAG_HAS_READ; 1293 kn->kn_fop = &linux_dev_kqfiltops_read; 1294 kn->kn_hook = filp; 1295 knlist_add(&filp->f_selinfo.si_note, kn, 1); 1296 error = 0; 1297 break; 1298 case EVFILT_WRITE: 1299 filp->f_kqflags |= LINUX_KQ_FLAG_HAS_WRITE; 1300 kn->kn_fop = &linux_dev_kqfiltops_write; 1301 kn->kn_hook = filp; 1302 knlist_add(&filp->f_selinfo.si_note, kn, 1); 1303 error = 0; 1304 break; 1305 default: 1306 error = EINVAL; 1307 break; 1308 } 1309 spin_unlock(&filp->f_kqlock); 1310 1311 if (error == 0) { 1312 linux_set_current(td); 1313 1314 /* update kqfilter status, if any */ 1315 linux_file_kqfilter_poll(filp, 1316 LINUX_KQ_FLAG_HAS_READ | LINUX_KQ_FLAG_HAS_WRITE); 1317 } 1318 return (error); 1319 } 1320 1321 static int 1322 linux_file_mmap_single(struct file *fp, const struct file_operations *fop, 1323 vm_ooffset_t *offset, vm_size_t size, struct vm_object **object, 1324 int nprot, bool is_shared, struct thread *td) 1325 { 1326 struct task_struct *task; 1327 struct vm_area_struct *vmap; 1328 struct mm_struct *mm; 1329 struct linux_file *filp; 1330 vm_memattr_t attr; 1331 int error; 1332 1333 filp = (struct linux_file *)fp->f_data; 1334 filp->f_flags = fp->f_flag; 1335 1336 if (fop->mmap == NULL) 1337 return (EOPNOTSUPP); 1338 1339 linux_set_current(td); 1340 1341 /* 1342 * The same VM object might be shared by multiple processes 1343 * and the mm_struct is usually freed when a process exits. 1344 * 1345 * The atomic reference below makes sure the mm_struct is 1346 * available as long as the vmap is in the linux_vma_head. 1347 */ 1348 task = current; 1349 mm = task->mm; 1350 if (atomic_inc_not_zero(&mm->mm_users) == 0) 1351 return (EINVAL); 1352 1353 vmap = kzalloc(sizeof(*vmap), GFP_KERNEL); 1354 vmap->vm_start = 0; 1355 vmap->vm_end = size; 1356 vmap->vm_pgoff = *offset / PAGE_SIZE; 1357 vmap->vm_pfn = 0; 1358 vmap->vm_flags = vmap->vm_page_prot = (nprot & VM_PROT_ALL); 1359 if (is_shared) 1360 vmap->vm_flags |= VM_SHARED; 1361 vmap->vm_ops = NULL; 1362 vmap->vm_file = get_file(filp); 1363 vmap->vm_mm = mm; 1364 1365 if (unlikely(down_write_killable(&vmap->vm_mm->mmap_sem))) { 1366 error = linux_get_error(task, EINTR); 1367 } else { 1368 error = -OPW(fp, td, fop->mmap(filp, vmap)); 1369 error = linux_get_error(task, error); 1370 up_write(&vmap->vm_mm->mmap_sem); 1371 } 1372 1373 if (error != 0) { 1374 linux_cdev_handle_free(vmap); 1375 return (error); 1376 } 1377 1378 attr = pgprot2cachemode(vmap->vm_page_prot); 1379 1380 if (vmap->vm_ops != NULL) { 1381 struct vm_area_struct *ptr; 1382 void *vm_private_data; 1383 bool vm_no_fault; 1384 1385 if (vmap->vm_ops->open == NULL || 1386 vmap->vm_ops->close == NULL || 1387 vmap->vm_private_data == NULL) { 1388 /* free allocated VM area struct */ 1389 linux_cdev_handle_free(vmap); 1390 return (EINVAL); 1391 } 1392 1393 vm_private_data = vmap->vm_private_data; 1394 1395 rw_wlock(&linux_vma_lock); 1396 TAILQ_FOREACH(ptr, &linux_vma_head, vm_entry) { 1397 if (ptr->vm_private_data == vm_private_data) 1398 break; 1399 } 1400 /* check if there is an existing VM area struct */ 1401 if (ptr != NULL) { 1402 /* check if the VM area structure is invalid */ 1403 if (ptr->vm_ops == NULL || 1404 ptr->vm_ops->open == NULL || 1405 ptr->vm_ops->close == NULL) { 1406 error = ESTALE; 1407 vm_no_fault = 1; 1408 } else { 1409 if (ptr->vm_start == vmap->vm_start && 1410 ptr->vm_end <= vmap->vm_end) { 1411 /* 1412 * Userspace wants to grow an existing 1413 * mapping. We already have a 1414 * `vm_object_t' for this mapping. We 1415 * just need to update the `struct 1416 * vm_area_struct` to have the correct 1417 * end address. 1418 */ 1419 ptr->vm_end = vmap->vm_end; 1420 } 1421 1422 error = EEXIST; 1423 vm_no_fault = (ptr->vm_ops->fault == NULL); 1424 } 1425 } else { 1426 /* insert VM area structure into list */ 1427 TAILQ_INSERT_TAIL(&linux_vma_head, vmap, vm_entry); 1428 error = 0; 1429 vm_no_fault = (vmap->vm_ops->fault == NULL); 1430 } 1431 rw_wunlock(&linux_vma_lock); 1432 1433 if (error != 0) { 1434 /* free allocated VM area struct */ 1435 linux_cdev_handle_free(vmap); 1436 /* check for stale VM area struct */ 1437 if (error != EEXIST) 1438 return (error); 1439 } 1440 1441 /* check if there is no fault handler */ 1442 if (vm_no_fault) { 1443 *object = cdev_pager_allocate(vm_private_data, OBJT_DEVICE, 1444 &linux_cdev_pager_ops[1], size, nprot, *offset, 1445 td->td_ucred); 1446 } else { 1447 *object = cdev_pager_allocate(vm_private_data, OBJT_MGTDEVICE, 1448 &linux_cdev_pager_ops[0], size, nprot, *offset, 1449 td->td_ucred); 1450 } 1451 1452 /* check if allocating the VM object failed */ 1453 if (*object == NULL) { 1454 if (error == 0) { 1455 /* remove VM area struct from list */ 1456 linux_cdev_handle_remove(vmap); 1457 /* free allocated VM area struct */ 1458 linux_cdev_handle_free(vmap); 1459 } 1460 return (EINVAL); 1461 } 1462 } else { 1463 struct sglist *sg; 1464 1465 sg = sglist_alloc(1, M_WAITOK); 1466 sglist_append_phys(sg, 1467 (vm_paddr_t)vmap->vm_pfn << PAGE_SHIFT, vmap->vm_len); 1468 1469 *object = vm_pager_allocate(OBJT_SG, sg, vmap->vm_len, 1470 nprot, 0, td->td_ucred); 1471 1472 linux_cdev_handle_free(vmap); 1473 1474 if (*object == NULL) { 1475 sglist_free(sg); 1476 return (EINVAL); 1477 } 1478 } 1479 1480 if (attr != VM_MEMATTR_DEFAULT) { 1481 VM_OBJECT_WLOCK(*object); 1482 vm_object_set_memattr(*object, attr); 1483 VM_OBJECT_WUNLOCK(*object); 1484 } 1485 *offset = 0; 1486 return (0); 1487 } 1488 1489 struct cdevsw linuxcdevsw = { 1490 .d_version = D_VERSION, 1491 .d_fdopen = linux_dev_fdopen, 1492 .d_name = "lkpidev", 1493 }; 1494 1495 static int 1496 linux_file_read(struct file *file, struct uio *uio, struct ucred *active_cred, 1497 int flags, struct thread *td) 1498 { 1499 struct linux_file *filp; 1500 const struct file_operations *fop; 1501 struct linux_cdev *ldev; 1502 ssize_t bytes; 1503 int error; 1504 1505 error = 0; 1506 filp = (struct linux_file *)file->f_data; 1507 filp->f_flags = file->f_flag; 1508 /* XXX no support for I/O vectors currently */ 1509 if (uio->uio_iovcnt != 1) 1510 return (EOPNOTSUPP); 1511 if (uio->uio_resid > DEVFS_IOSIZE_MAX) 1512 return (EINVAL); 1513 linux_set_current(td); 1514 linux_get_fop(filp, &fop, &ldev); 1515 if (fop->read != NULL) { 1516 bytes = OPW(file, td, fop->read(filp, 1517 uio->uio_iov->iov_base, 1518 uio->uio_iov->iov_len, &uio->uio_offset)); 1519 if (bytes >= 0) { 1520 uio->uio_iov->iov_base = 1521 ((uint8_t *)uio->uio_iov->iov_base) + bytes; 1522 uio->uio_iov->iov_len -= bytes; 1523 uio->uio_resid -= bytes; 1524 } else { 1525 error = linux_get_error(current, -bytes); 1526 } 1527 } else 1528 error = ENXIO; 1529 1530 /* update kqfilter status, if any */ 1531 linux_file_kqfilter_poll(filp, LINUX_KQ_FLAG_HAS_READ); 1532 linux_drop_fop(ldev); 1533 1534 return (error); 1535 } 1536 1537 static int 1538 linux_file_write(struct file *file, struct uio *uio, struct ucred *active_cred, 1539 int flags, struct thread *td) 1540 { 1541 struct linux_file *filp; 1542 const struct file_operations *fop; 1543 struct linux_cdev *ldev; 1544 ssize_t bytes; 1545 int error; 1546 1547 filp = (struct linux_file *)file->f_data; 1548 filp->f_flags = file->f_flag; 1549 /* XXX no support for I/O vectors currently */ 1550 if (uio->uio_iovcnt != 1) 1551 return (EOPNOTSUPP); 1552 if (uio->uio_resid > DEVFS_IOSIZE_MAX) 1553 return (EINVAL); 1554 linux_set_current(td); 1555 linux_get_fop(filp, &fop, &ldev); 1556 if (fop->write != NULL) { 1557 bytes = OPW(file, td, fop->write(filp, 1558 uio->uio_iov->iov_base, 1559 uio->uio_iov->iov_len, &uio->uio_offset)); 1560 if (bytes >= 0) { 1561 uio->uio_iov->iov_base = 1562 ((uint8_t *)uio->uio_iov->iov_base) + bytes; 1563 uio->uio_iov->iov_len -= bytes; 1564 uio->uio_resid -= bytes; 1565 error = 0; 1566 } else { 1567 error = linux_get_error(current, -bytes); 1568 } 1569 } else 1570 error = ENXIO; 1571 1572 /* update kqfilter status, if any */ 1573 linux_file_kqfilter_poll(filp, LINUX_KQ_FLAG_HAS_WRITE); 1574 1575 linux_drop_fop(ldev); 1576 1577 return (error); 1578 } 1579 1580 static int 1581 linux_file_poll(struct file *file, int events, struct ucred *active_cred, 1582 struct thread *td) 1583 { 1584 struct linux_file *filp; 1585 const struct file_operations *fop; 1586 struct linux_cdev *ldev; 1587 int revents; 1588 1589 filp = (struct linux_file *)file->f_data; 1590 filp->f_flags = file->f_flag; 1591 linux_set_current(td); 1592 linux_get_fop(filp, &fop, &ldev); 1593 if (fop->poll != NULL) { 1594 revents = OPW(file, td, fop->poll(filp, 1595 LINUX_POLL_TABLE_NORMAL)) & events; 1596 } else { 1597 revents = 0; 1598 } 1599 linux_drop_fop(ldev); 1600 return (revents); 1601 } 1602 1603 static int 1604 linux_file_close(struct file *file, struct thread *td) 1605 { 1606 struct linux_file *filp; 1607 int (*release)(struct inode *, struct linux_file *); 1608 const struct file_operations *fop; 1609 struct linux_cdev *ldev; 1610 int error; 1611 1612 filp = (struct linux_file *)file->f_data; 1613 1614 KASSERT(file_count(filp) == 0, 1615 ("File refcount(%d) is not zero", file_count(filp))); 1616 1617 if (td == NULL) 1618 td = curthread; 1619 1620 error = 0; 1621 filp->f_flags = file->f_flag; 1622 linux_set_current(td); 1623 linux_poll_wait_dequeue(filp); 1624 linux_get_fop(filp, &fop, &ldev); 1625 /* 1626 * Always use the real release function, if any, to avoid 1627 * leaking device resources: 1628 */ 1629 release = filp->f_op->release; 1630 if (release != NULL) 1631 error = -OPW(file, td, release(filp->f_vnode, filp)); 1632 funsetown(&filp->f_sigio); 1633 if (filp->f_vnode != NULL) 1634 vrele(filp->f_vnode); 1635 linux_drop_fop(ldev); 1636 ldev = filp->f_cdev; 1637 if (ldev != NULL) 1638 linux_cdev_deref(ldev); 1639 linux_synchronize_rcu(RCU_TYPE_REGULAR); 1640 kfree(filp); 1641 1642 return (error); 1643 } 1644 1645 static int 1646 linux_file_ioctl(struct file *fp, u_long cmd, void *data, struct ucred *cred, 1647 struct thread *td) 1648 { 1649 struct linux_file *filp; 1650 const struct file_operations *fop; 1651 struct linux_cdev *ldev; 1652 struct fiodgname_arg *fgn; 1653 const char *p; 1654 int error, i; 1655 1656 error = 0; 1657 filp = (struct linux_file *)fp->f_data; 1658 filp->f_flags = fp->f_flag; 1659 linux_get_fop(filp, &fop, &ldev); 1660 1661 linux_set_current(td); 1662 switch (cmd) { 1663 case FIONBIO: 1664 break; 1665 case FIOASYNC: 1666 if (fop->fasync == NULL) 1667 break; 1668 error = -OPW(fp, td, fop->fasync(0, filp, fp->f_flag & FASYNC)); 1669 break; 1670 case FIOSETOWN: 1671 error = fsetown(*(int *)data, &filp->f_sigio); 1672 if (error == 0) { 1673 if (fop->fasync == NULL) 1674 break; 1675 error = -OPW(fp, td, fop->fasync(0, filp, 1676 fp->f_flag & FASYNC)); 1677 } 1678 break; 1679 case FIOGETOWN: 1680 *(int *)data = fgetown(&filp->f_sigio); 1681 break; 1682 case FIODGNAME: 1683 #ifdef COMPAT_FREEBSD32 1684 case FIODGNAME_32: 1685 #endif 1686 if (filp->f_cdev == NULL || filp->f_cdev->cdev == NULL) { 1687 error = ENXIO; 1688 break; 1689 } 1690 fgn = data; 1691 p = devtoname(filp->f_cdev->cdev); 1692 i = strlen(p) + 1; 1693 if (i > fgn->len) { 1694 error = EINVAL; 1695 break; 1696 } 1697 error = copyout(p, fiodgname_buf_get_ptr(fgn, cmd), i); 1698 break; 1699 default: 1700 error = linux_file_ioctl_sub(fp, filp, fop, cmd, data, td); 1701 break; 1702 } 1703 linux_drop_fop(ldev); 1704 return (error); 1705 } 1706 1707 static int 1708 linux_file_mmap_sub(struct thread *td, vm_size_t objsize, vm_prot_t prot, 1709 vm_prot_t maxprot, int flags, struct file *fp, 1710 vm_ooffset_t *foff, const struct file_operations *fop, vm_object_t *objp) 1711 { 1712 /* 1713 * Character devices do not provide private mappings 1714 * of any kind: 1715 */ 1716 if ((maxprot & VM_PROT_WRITE) == 0 && 1717 (prot & VM_PROT_WRITE) != 0) 1718 return (EACCES); 1719 if ((flags & (MAP_PRIVATE | MAP_COPY)) != 0) 1720 return (EINVAL); 1721 1722 return (linux_file_mmap_single(fp, fop, foff, objsize, objp, 1723 (int)prot, (flags & MAP_SHARED) ? true : false, td)); 1724 } 1725 1726 static int 1727 linux_file_mmap(struct file *fp, vm_map_t map, vm_offset_t *addr, vm_size_t size, 1728 vm_prot_t prot, vm_prot_t cap_maxprot, int flags, vm_ooffset_t foff, 1729 struct thread *td) 1730 { 1731 struct linux_file *filp; 1732 const struct file_operations *fop; 1733 struct linux_cdev *ldev; 1734 struct mount *mp; 1735 struct vnode *vp; 1736 vm_object_t object; 1737 vm_prot_t maxprot; 1738 int error; 1739 1740 filp = (struct linux_file *)fp->f_data; 1741 1742 vp = filp->f_vnode; 1743 if (vp == NULL) 1744 return (EOPNOTSUPP); 1745 1746 /* 1747 * Ensure that file and memory protections are 1748 * compatible. 1749 */ 1750 mp = vp->v_mount; 1751 if (mp != NULL && (mp->mnt_flag & MNT_NOEXEC) != 0) { 1752 maxprot = VM_PROT_NONE; 1753 if ((prot & VM_PROT_EXECUTE) != 0) 1754 return (EACCES); 1755 } else 1756 maxprot = VM_PROT_EXECUTE; 1757 if ((fp->f_flag & FREAD) != 0) 1758 maxprot |= VM_PROT_READ; 1759 else if ((prot & VM_PROT_READ) != 0) 1760 return (EACCES); 1761 1762 /* 1763 * If we are sharing potential changes via MAP_SHARED and we 1764 * are trying to get write permission although we opened it 1765 * without asking for it, bail out. 1766 * 1767 * Note that most character devices always share mappings. 1768 * 1769 * Rely on linux_file_mmap_sub() to fail invalid MAP_PRIVATE 1770 * requests rather than doing it here. 1771 */ 1772 if ((flags & MAP_SHARED) != 0) { 1773 if ((fp->f_flag & FWRITE) != 0) 1774 maxprot |= VM_PROT_WRITE; 1775 else if ((prot & VM_PROT_WRITE) != 0) 1776 return (EACCES); 1777 } 1778 maxprot &= cap_maxprot; 1779 1780 linux_get_fop(filp, &fop, &ldev); 1781 error = linux_file_mmap_sub(td, size, prot, maxprot, flags, fp, 1782 &foff, fop, &object); 1783 if (error != 0) 1784 goto out; 1785 1786 error = vm_mmap_object(map, addr, size, prot, maxprot, flags, object, 1787 foff, FALSE, td); 1788 if (error != 0) 1789 vm_object_deallocate(object); 1790 out: 1791 linux_drop_fop(ldev); 1792 return (error); 1793 } 1794 1795 static int 1796 linux_file_stat(struct file *fp, struct stat *sb, struct ucred *active_cred) 1797 { 1798 struct linux_file *filp; 1799 struct vnode *vp; 1800 int error; 1801 1802 filp = (struct linux_file *)fp->f_data; 1803 if (filp->f_vnode == NULL) 1804 return (EOPNOTSUPP); 1805 1806 vp = filp->f_vnode; 1807 1808 vn_lock(vp, LK_SHARED | LK_RETRY); 1809 error = VOP_STAT(vp, sb, curthread->td_ucred, NOCRED); 1810 VOP_UNLOCK(vp); 1811 1812 return (error); 1813 } 1814 1815 static int 1816 linux_file_fill_kinfo(struct file *fp, struct kinfo_file *kif, 1817 struct filedesc *fdp) 1818 { 1819 struct linux_file *filp; 1820 struct vnode *vp; 1821 int error; 1822 1823 filp = fp->f_data; 1824 vp = filp->f_vnode; 1825 if (vp == NULL) { 1826 error = 0; 1827 kif->kf_type = KF_TYPE_DEV; 1828 } else { 1829 vref(vp); 1830 FILEDESC_SUNLOCK(fdp); 1831 error = vn_fill_kinfo_vnode(vp, kif); 1832 vrele(vp); 1833 kif->kf_type = KF_TYPE_VNODE; 1834 FILEDESC_SLOCK(fdp); 1835 } 1836 return (error); 1837 } 1838 1839 unsigned int 1840 linux_iminor(struct inode *inode) 1841 { 1842 struct linux_cdev *ldev; 1843 1844 if (inode == NULL || inode->v_rdev == NULL || 1845 inode->v_rdev->si_devsw != &linuxcdevsw) 1846 return (-1U); 1847 ldev = inode->v_rdev->si_drv1; 1848 if (ldev == NULL) 1849 return (-1U); 1850 1851 return (minor(ldev->dev)); 1852 } 1853 1854 static int 1855 linux_file_kcmp(struct file *fp1, struct file *fp2, struct thread *td) 1856 { 1857 struct linux_file *filp1, *filp2; 1858 1859 if (fp2->f_type != DTYPE_DEV) 1860 return (3); 1861 1862 filp1 = fp1->f_data; 1863 filp2 = fp2->f_data; 1864 return (kcmp_cmp((uintptr_t)filp1->f_cdev, (uintptr_t)filp2->f_cdev)); 1865 } 1866 1867 const struct fileops linuxfileops = { 1868 .fo_read = linux_file_read, 1869 .fo_write = linux_file_write, 1870 .fo_truncate = invfo_truncate, 1871 .fo_kqfilter = linux_file_kqfilter, 1872 .fo_stat = linux_file_stat, 1873 .fo_fill_kinfo = linux_file_fill_kinfo, 1874 .fo_poll = linux_file_poll, 1875 .fo_close = linux_file_close, 1876 .fo_ioctl = linux_file_ioctl, 1877 .fo_mmap = linux_file_mmap, 1878 .fo_chmod = invfo_chmod, 1879 .fo_chown = invfo_chown, 1880 .fo_sendfile = invfo_sendfile, 1881 .fo_cmp = linux_file_kcmp, 1882 .fo_flags = DFLAG_PASSABLE, 1883 }; 1884 1885 static char * 1886 devm_kvasprintf(struct device *dev, gfp_t gfp, const char *fmt, va_list ap) 1887 { 1888 unsigned int len; 1889 char *p; 1890 va_list aq; 1891 1892 va_copy(aq, ap); 1893 len = vsnprintf(NULL, 0, fmt, aq); 1894 va_end(aq); 1895 1896 if (dev != NULL) 1897 p = devm_kmalloc(dev, len + 1, gfp); 1898 else 1899 p = kmalloc(len + 1, gfp); 1900 if (p != NULL) 1901 vsnprintf(p, len + 1, fmt, ap); 1902 1903 return (p); 1904 } 1905 1906 char * 1907 kvasprintf(gfp_t gfp, const char *fmt, va_list ap) 1908 { 1909 1910 return (devm_kvasprintf(NULL, gfp, fmt, ap)); 1911 } 1912 1913 char * 1914 lkpi_devm_kasprintf(struct device *dev, gfp_t gfp, const char *fmt, ...) 1915 { 1916 va_list ap; 1917 char *p; 1918 1919 va_start(ap, fmt); 1920 p = devm_kvasprintf(dev, gfp, fmt, ap); 1921 va_end(ap); 1922 1923 return (p); 1924 } 1925 1926 char * 1927 kasprintf(gfp_t gfp, const char *fmt, ...) 1928 { 1929 va_list ap; 1930 char *p; 1931 1932 va_start(ap, fmt); 1933 p = kvasprintf(gfp, fmt, ap); 1934 va_end(ap); 1935 1936 return (p); 1937 } 1938 1939 int 1940 __lkpi_hexdump_printf(void *arg1 __unused, const char *fmt, ...) 1941 { 1942 va_list ap; 1943 int result; 1944 1945 va_start(ap, fmt); 1946 result = vprintf(fmt, ap); 1947 va_end(ap); 1948 return (result); 1949 } 1950 1951 int 1952 __lkpi_hexdump_sbuf_printf(void *arg1, const char *fmt, ...) 1953 { 1954 va_list ap; 1955 int result; 1956 1957 va_start(ap, fmt); 1958 result = sbuf_vprintf(arg1, fmt, ap); 1959 va_end(ap); 1960 return (result); 1961 } 1962 1963 void 1964 lkpi_hex_dump(int(*_fpf)(void *, const char *, ...), void *arg1, 1965 const char *level, const char *prefix_str, 1966 const int prefix_type, const int rowsize, const int groupsize, 1967 const void *buf, size_t len, const bool ascii, const bool trailing_newline) 1968 { 1969 typedef const struct { long long value; } __packed *print_64p_t; 1970 typedef const struct { uint32_t value; } __packed *print_32p_t; 1971 typedef const struct { uint16_t value; } __packed *print_16p_t; 1972 const void *buf_old = buf; 1973 int row, linelen, ret; 1974 1975 while (len > 0) { 1976 linelen = 0; 1977 if (level != NULL) { 1978 ret = _fpf(arg1, "%s", level); 1979 if (ret < 0) 1980 break; 1981 linelen += ret; 1982 } 1983 if (prefix_str != NULL) { 1984 ret = _fpf( 1985 arg1, "%s%s", linelen ? " " : "", prefix_str); 1986 if (ret < 0) 1987 break; 1988 linelen += ret; 1989 } 1990 1991 switch (prefix_type) { 1992 case DUMP_PREFIX_ADDRESS: 1993 ret = _fpf( 1994 arg1, "%s[%p]", linelen ? " " : "", buf); 1995 if (ret < 0) 1996 return; 1997 linelen += ret; 1998 break; 1999 case DUMP_PREFIX_OFFSET: 2000 ret = _fpf( 2001 arg1, "%s[%#tx]", linelen ? " " : "", 2002 ((const char *)buf - (const char *)buf_old)); 2003 if (ret < 0) 2004 return; 2005 linelen += ret; 2006 break; 2007 default: 2008 break; 2009 } 2010 for (row = 0; row != rowsize; row++) { 2011 if (groupsize == 8 && len > 7) { 2012 ret = _fpf( 2013 arg1, "%s%016llx", linelen ? " " : "", 2014 ((print_64p_t)buf)->value); 2015 if (ret < 0) 2016 return; 2017 linelen += ret; 2018 buf = (const uint8_t *)buf + 8; 2019 len -= 8; 2020 } else if (groupsize == 4 && len > 3) { 2021 ret = _fpf( 2022 arg1, "%s%08x", linelen ? " " : "", 2023 ((print_32p_t)buf)->value); 2024 if (ret < 0) 2025 return; 2026 linelen += ret; 2027 buf = (const uint8_t *)buf + 4; 2028 len -= 4; 2029 } else if (groupsize == 2 && len > 1) { 2030 ret = _fpf( 2031 arg1, "%s%04x", linelen ? " " : "", 2032 ((print_16p_t)buf)->value); 2033 if (ret < 0) 2034 return; 2035 linelen += ret; 2036 buf = (const uint8_t *)buf + 2; 2037 len -= 2; 2038 } else if (len > 0) { 2039 ret = _fpf( 2040 arg1, "%s%02x", linelen ? " " : "", 2041 *(const uint8_t *)buf); 2042 if (ret < 0) 2043 return; 2044 linelen += ret; 2045 buf = (const uint8_t *)buf + 1; 2046 len--; 2047 } else { 2048 break; 2049 } 2050 } 2051 if (len > 0 && trailing_newline) { 2052 ret = _fpf(arg1, "\n"); 2053 if (ret < 0) 2054 break; 2055 } 2056 } 2057 } 2058 2059 struct hdtb_context { 2060 char *linebuf; 2061 size_t linebuflen; 2062 int written; 2063 }; 2064 2065 static int 2066 hdtb_cb(void *arg, const char *format, ...) 2067 { 2068 struct hdtb_context *context; 2069 int written; 2070 va_list args; 2071 2072 context = arg; 2073 2074 va_start(args, format); 2075 written = vsnprintf( 2076 context->linebuf, context->linebuflen, format, args); 2077 va_end(args); 2078 2079 if (written < 0) 2080 return (written); 2081 2082 /* 2083 * Linux' hex_dump_to_buffer() function has the same behaviour as 2084 * snprintf() basically. Therefore, it returns the number of bytes it 2085 * would have written if the destination buffer was large enough. 2086 * 2087 * If the destination buffer was exhausted, lkpi_hex_dump() will 2088 * continue to call this callback but it will only compute the bytes it 2089 * would have written but write nothing to that buffer. 2090 */ 2091 context->written += written; 2092 2093 if (written < context->linebuflen) { 2094 context->linebuf += written; 2095 context->linebuflen -= written; 2096 } else { 2097 context->linebuf += context->linebuflen; 2098 context->linebuflen = 0; 2099 } 2100 2101 return (written); 2102 } 2103 2104 int 2105 lkpi_hex_dump_to_buffer(const void *buf, size_t len, int rowsize, 2106 int groupsize, char *linebuf, size_t linebuflen, bool ascii) 2107 { 2108 int written; 2109 struct hdtb_context context; 2110 2111 context.linebuf = linebuf; 2112 context.linebuflen = linebuflen; 2113 context.written = 0; 2114 2115 if (rowsize != 16 && rowsize != 32) 2116 rowsize = 16; 2117 2118 len = min(len, rowsize); 2119 2120 lkpi_hex_dump( 2121 hdtb_cb, &context, NULL, NULL, DUMP_PREFIX_NONE, 2122 rowsize, groupsize, buf, len, ascii, false); 2123 2124 written = context.written; 2125 2126 return (written); 2127 } 2128 2129 static void 2130 linux_timer_callback_wrapper(void *context) 2131 { 2132 struct timer_list *timer; 2133 2134 timer = context; 2135 2136 /* the timer is about to be shutdown permanently */ 2137 if (timer->function == NULL) 2138 return; 2139 2140 if (linux_set_current_flags(curthread, M_NOWAIT)) { 2141 /* try again later */ 2142 callout_reset(&timer->callout, 1, 2143 &linux_timer_callback_wrapper, timer); 2144 return; 2145 } 2146 2147 timer->function(timer->data); 2148 } 2149 2150 static int 2151 linux_timer_jiffies_until(unsigned long expires) 2152 { 2153 unsigned long delta = expires - jiffies; 2154 2155 /* 2156 * Guard against already expired values and make sure that the value can 2157 * be used as a tick count, rather than a jiffies count. 2158 */ 2159 if ((long)delta < 1) 2160 delta = 1; 2161 else if (delta > INT_MAX) 2162 delta = INT_MAX; 2163 return ((int)delta); 2164 } 2165 2166 int 2167 mod_timer(struct timer_list *timer, unsigned long expires) 2168 { 2169 int ret; 2170 2171 timer->expires = expires; 2172 ret = callout_reset(&timer->callout, 2173 linux_timer_jiffies_until(expires), 2174 &linux_timer_callback_wrapper, timer); 2175 2176 MPASS(ret == 0 || ret == 1); 2177 2178 return (ret == 1); 2179 } 2180 2181 void 2182 add_timer(struct timer_list *timer) 2183 { 2184 2185 callout_reset(&timer->callout, 2186 linux_timer_jiffies_until(timer->expires), 2187 &linux_timer_callback_wrapper, timer); 2188 } 2189 2190 void 2191 add_timer_on(struct timer_list *timer, int cpu) 2192 { 2193 2194 callout_reset_on(&timer->callout, 2195 linux_timer_jiffies_until(timer->expires), 2196 &linux_timer_callback_wrapper, timer, cpu); 2197 } 2198 2199 int 2200 timer_delete(struct timer_list *timer) 2201 { 2202 2203 if (callout_stop(&(timer)->callout) == -1) 2204 return (0); 2205 return (1); 2206 } 2207 2208 int 2209 timer_delete_sync(struct timer_list *timer) 2210 { 2211 2212 if (callout_drain(&(timer)->callout) == -1) 2213 return (0); 2214 return (1); 2215 } 2216 2217 int 2218 timer_shutdown_sync(struct timer_list *timer) 2219 { 2220 2221 timer->function = NULL; 2222 return (del_timer_sync(timer)); 2223 } 2224 2225 /* greatest common divisor, Euclid equation */ 2226 static uint64_t 2227 lkpi_gcd_64(uint64_t a, uint64_t b) 2228 { 2229 uint64_t an; 2230 uint64_t bn; 2231 2232 while (b != 0) { 2233 an = b; 2234 bn = a % b; 2235 a = an; 2236 b = bn; 2237 } 2238 return (a); 2239 } 2240 2241 uint64_t lkpi_nsec2hz_rem; 2242 uint64_t lkpi_nsec2hz_div = 1000000000ULL; 2243 uint64_t lkpi_nsec2hz_max; 2244 2245 uint64_t lkpi_usec2hz_rem; 2246 uint64_t lkpi_usec2hz_div = 1000000ULL; 2247 uint64_t lkpi_usec2hz_max; 2248 2249 uint64_t lkpi_msec2hz_rem; 2250 uint64_t lkpi_msec2hz_div = 1000ULL; 2251 uint64_t lkpi_msec2hz_max; 2252 2253 static void 2254 linux_timer_init(void *arg) 2255 { 2256 uint64_t gcd; 2257 2258 /* 2259 * Compute an internal HZ value which can divide 2**32 to 2260 * avoid timer rounding problems when the tick value wraps 2261 * around 2**32: 2262 */ 2263 linux_timer_hz_mask = 1; 2264 while (linux_timer_hz_mask < (unsigned long)hz) 2265 linux_timer_hz_mask *= 2; 2266 linux_timer_hz_mask--; 2267 2268 /* compute some internal constants */ 2269 2270 lkpi_nsec2hz_rem = hz; 2271 lkpi_usec2hz_rem = hz; 2272 lkpi_msec2hz_rem = hz; 2273 2274 gcd = lkpi_gcd_64(lkpi_nsec2hz_rem, lkpi_nsec2hz_div); 2275 lkpi_nsec2hz_rem /= gcd; 2276 lkpi_nsec2hz_div /= gcd; 2277 lkpi_nsec2hz_max = -1ULL / lkpi_nsec2hz_rem; 2278 2279 gcd = lkpi_gcd_64(lkpi_usec2hz_rem, lkpi_usec2hz_div); 2280 lkpi_usec2hz_rem /= gcd; 2281 lkpi_usec2hz_div /= gcd; 2282 lkpi_usec2hz_max = -1ULL / lkpi_usec2hz_rem; 2283 2284 gcd = lkpi_gcd_64(lkpi_msec2hz_rem, lkpi_msec2hz_div); 2285 lkpi_msec2hz_rem /= gcd; 2286 lkpi_msec2hz_div /= gcd; 2287 lkpi_msec2hz_max = -1ULL / lkpi_msec2hz_rem; 2288 } 2289 SYSINIT(linux_timer, SI_SUB_DRIVERS, SI_ORDER_FIRST, linux_timer_init, NULL); 2290 2291 void 2292 linux_complete_common(struct completion *c, int all) 2293 { 2294 sleepq_lock(c); 2295 if (all) { 2296 c->done = UINT_MAX; 2297 sleepq_broadcast(c, SLEEPQ_SLEEP, 0, 0); 2298 } else { 2299 if (c->done != UINT_MAX) 2300 c->done++; 2301 sleepq_signal(c, SLEEPQ_SLEEP, 0, 0); 2302 } 2303 sleepq_release(c); 2304 } 2305 2306 /* 2307 * Indefinite wait for done != 0 with or without signals. 2308 */ 2309 int 2310 linux_wait_for_common(struct completion *c, int flags) 2311 { 2312 struct task_struct *task; 2313 int error; 2314 2315 if (SCHEDULER_STOPPED()) 2316 return (0); 2317 2318 task = current; 2319 2320 if (flags != 0) 2321 flags = SLEEPQ_INTERRUPTIBLE | SLEEPQ_SLEEP; 2322 else 2323 flags = SLEEPQ_SLEEP; 2324 error = 0; 2325 for (;;) { 2326 sleepq_lock(c); 2327 if (c->done) 2328 break; 2329 sleepq_add(c, NULL, "completion", flags, 0); 2330 if (flags & SLEEPQ_INTERRUPTIBLE) { 2331 DROP_GIANT(); 2332 error = -sleepq_wait_sig(c, 0); 2333 PICKUP_GIANT(); 2334 if (error != 0) { 2335 linux_schedule_save_interrupt_value(task, error); 2336 error = -ERESTARTSYS; 2337 goto intr; 2338 } 2339 } else { 2340 DROP_GIANT(); 2341 sleepq_wait(c, 0); 2342 PICKUP_GIANT(); 2343 } 2344 } 2345 if (c->done != UINT_MAX) 2346 c->done--; 2347 sleepq_release(c); 2348 2349 intr: 2350 return (error); 2351 } 2352 2353 /* 2354 * Time limited wait for done != 0 with or without signals. 2355 */ 2356 unsigned long 2357 linux_wait_for_timeout_common(struct completion *c, unsigned long timeout, 2358 int flags) 2359 { 2360 struct task_struct *task; 2361 unsigned long end = jiffies + timeout, error; 2362 2363 if (SCHEDULER_STOPPED()) 2364 return (0); 2365 2366 task = current; 2367 2368 if (flags != 0) 2369 flags = SLEEPQ_INTERRUPTIBLE | SLEEPQ_SLEEP; 2370 else 2371 flags = SLEEPQ_SLEEP; 2372 2373 for (;;) { 2374 sleepq_lock(c); 2375 if (c->done) 2376 break; 2377 sleepq_add(c, NULL, "completion", flags, 0); 2378 sleepq_set_timeout(c, linux_timer_jiffies_until(end)); 2379 2380 DROP_GIANT(); 2381 if (flags & SLEEPQ_INTERRUPTIBLE) 2382 error = -sleepq_timedwait_sig(c, 0); 2383 else 2384 error = -sleepq_timedwait(c, 0); 2385 PICKUP_GIANT(); 2386 2387 if (error != 0) { 2388 /* check for timeout */ 2389 if (error == -EWOULDBLOCK) { 2390 error = 0; /* timeout */ 2391 } else { 2392 /* signal happened */ 2393 linux_schedule_save_interrupt_value(task, error); 2394 error = -ERESTARTSYS; 2395 } 2396 goto done; 2397 } 2398 } 2399 if (c->done != UINT_MAX) 2400 c->done--; 2401 sleepq_release(c); 2402 2403 /* return how many jiffies are left */ 2404 error = linux_timer_jiffies_until(end); 2405 done: 2406 return (error); 2407 } 2408 2409 int 2410 linux_try_wait_for_completion(struct completion *c) 2411 { 2412 int isdone; 2413 2414 sleepq_lock(c); 2415 isdone = (c->done != 0); 2416 if (c->done != 0 && c->done != UINT_MAX) 2417 c->done--; 2418 sleepq_release(c); 2419 return (isdone); 2420 } 2421 2422 int 2423 linux_completion_done(struct completion *c) 2424 { 2425 int isdone; 2426 2427 sleepq_lock(c); 2428 isdone = (c->done != 0); 2429 sleepq_release(c); 2430 return (isdone); 2431 } 2432 2433 static void 2434 linux_cdev_deref(struct linux_cdev *ldev) 2435 { 2436 if (refcount_release(&ldev->refs) && 2437 ldev->kobj.ktype == &linux_cdev_ktype) 2438 kfree(ldev); 2439 } 2440 2441 static void 2442 linux_cdev_release(struct kobject *kobj) 2443 { 2444 struct linux_cdev *cdev; 2445 struct kobject *parent; 2446 2447 cdev = container_of(kobj, struct linux_cdev, kobj); 2448 parent = kobj->parent; 2449 linux_destroy_dev(cdev); 2450 linux_cdev_deref(cdev); 2451 kobject_put(parent); 2452 } 2453 2454 static void 2455 linux_cdev_static_release(struct kobject *kobj) 2456 { 2457 struct cdev *cdev; 2458 struct linux_cdev *ldev; 2459 2460 ldev = container_of(kobj, struct linux_cdev, kobj); 2461 cdev = ldev->cdev; 2462 if (cdev != NULL) { 2463 destroy_dev(cdev); 2464 ldev->cdev = NULL; 2465 } 2466 kobject_put(kobj->parent); 2467 } 2468 2469 int 2470 linux_cdev_device_add(struct linux_cdev *ldev, struct device *dev) 2471 { 2472 int ret; 2473 2474 if (dev->devt != 0) { 2475 /* Set parent kernel object. */ 2476 ldev->kobj.parent = &dev->kobj; 2477 2478 /* 2479 * Unlike Linux we require the kobject of the 2480 * character device structure to have a valid name 2481 * before calling this function: 2482 */ 2483 if (ldev->kobj.name == NULL) 2484 return (-EINVAL); 2485 2486 ret = cdev_add(ldev, dev->devt, 1); 2487 if (ret) 2488 return (ret); 2489 } 2490 ret = device_add(dev); 2491 if (ret != 0 && dev->devt != 0) 2492 cdev_del(ldev); 2493 return (ret); 2494 } 2495 2496 void 2497 linux_cdev_device_del(struct linux_cdev *ldev, struct device *dev) 2498 { 2499 device_del(dev); 2500 2501 if (dev->devt != 0) 2502 cdev_del(ldev); 2503 } 2504 2505 static void 2506 linux_destroy_dev(struct linux_cdev *ldev) 2507 { 2508 2509 if (ldev->cdev == NULL) 2510 return; 2511 2512 MPASS((ldev->siref & LDEV_SI_DTR) == 0); 2513 MPASS(ldev->kobj.ktype == &linux_cdev_ktype); 2514 2515 atomic_set_int(&ldev->siref, LDEV_SI_DTR); 2516 while ((atomic_load_int(&ldev->siref) & ~LDEV_SI_DTR) != 0) 2517 pause("ldevdtr", hz / 4); 2518 2519 destroy_dev(ldev->cdev); 2520 ldev->cdev = NULL; 2521 } 2522 2523 const struct kobj_type linux_cdev_ktype = { 2524 .release = linux_cdev_release, 2525 }; 2526 2527 const struct kobj_type linux_cdev_static_ktype = { 2528 .release = linux_cdev_static_release, 2529 }; 2530 2531 static void 2532 linux_handle_ifnet_link_event(void *arg, struct ifnet *ifp, int linkstate) 2533 { 2534 struct notifier_block *nb; 2535 struct netdev_notifier_info ni; 2536 2537 nb = arg; 2538 ni.ifp = ifp; 2539 ni.dev = (struct net_device *)ifp; 2540 if (linkstate == LINK_STATE_UP) 2541 nb->notifier_call(nb, NETDEV_UP, &ni); 2542 else 2543 nb->notifier_call(nb, NETDEV_DOWN, &ni); 2544 } 2545 2546 static void 2547 linux_handle_ifnet_arrival_event(void *arg, struct ifnet *ifp) 2548 { 2549 struct notifier_block *nb; 2550 struct netdev_notifier_info ni; 2551 2552 nb = arg; 2553 ni.ifp = ifp; 2554 ni.dev = (struct net_device *)ifp; 2555 nb->notifier_call(nb, NETDEV_REGISTER, &ni); 2556 } 2557 2558 static void 2559 linux_handle_ifnet_departure_event(void *arg, struct ifnet *ifp) 2560 { 2561 struct notifier_block *nb; 2562 struct netdev_notifier_info ni; 2563 2564 nb = arg; 2565 ni.ifp = ifp; 2566 ni.dev = (struct net_device *)ifp; 2567 nb->notifier_call(nb, NETDEV_UNREGISTER, &ni); 2568 } 2569 2570 static void 2571 linux_handle_iflladdr_event(void *arg, struct ifnet *ifp) 2572 { 2573 struct notifier_block *nb; 2574 struct netdev_notifier_info ni; 2575 2576 nb = arg; 2577 ni.ifp = ifp; 2578 ni.dev = (struct net_device *)ifp; 2579 nb->notifier_call(nb, NETDEV_CHANGEADDR, &ni); 2580 } 2581 2582 static void 2583 linux_handle_ifaddr_event(void *arg, struct ifnet *ifp) 2584 { 2585 struct notifier_block *nb; 2586 struct netdev_notifier_info ni; 2587 2588 nb = arg; 2589 ni.ifp = ifp; 2590 ni.dev = (struct net_device *)ifp; 2591 nb->notifier_call(nb, NETDEV_CHANGEIFADDR, &ni); 2592 } 2593 2594 int 2595 register_netdevice_notifier(struct notifier_block *nb) 2596 { 2597 2598 nb->tags[NETDEV_UP] = EVENTHANDLER_REGISTER( 2599 ifnet_link_event, linux_handle_ifnet_link_event, nb, 0); 2600 nb->tags[NETDEV_REGISTER] = EVENTHANDLER_REGISTER( 2601 ifnet_arrival_event, linux_handle_ifnet_arrival_event, nb, 0); 2602 nb->tags[NETDEV_UNREGISTER] = EVENTHANDLER_REGISTER( 2603 ifnet_departure_event, linux_handle_ifnet_departure_event, nb, 0); 2604 nb->tags[NETDEV_CHANGEADDR] = EVENTHANDLER_REGISTER( 2605 iflladdr_event, linux_handle_iflladdr_event, nb, 0); 2606 2607 return (0); 2608 } 2609 2610 int 2611 register_inetaddr_notifier(struct notifier_block *nb) 2612 { 2613 2614 nb->tags[NETDEV_CHANGEIFADDR] = EVENTHANDLER_REGISTER( 2615 ifaddr_event, linux_handle_ifaddr_event, nb, 0); 2616 return (0); 2617 } 2618 2619 int 2620 unregister_netdevice_notifier(struct notifier_block *nb) 2621 { 2622 2623 EVENTHANDLER_DEREGISTER(ifnet_link_event, 2624 nb->tags[NETDEV_UP]); 2625 EVENTHANDLER_DEREGISTER(ifnet_arrival_event, 2626 nb->tags[NETDEV_REGISTER]); 2627 EVENTHANDLER_DEREGISTER(ifnet_departure_event, 2628 nb->tags[NETDEV_UNREGISTER]); 2629 EVENTHANDLER_DEREGISTER(iflladdr_event, 2630 nb->tags[NETDEV_CHANGEADDR]); 2631 2632 return (0); 2633 } 2634 2635 int 2636 unregister_inetaddr_notifier(struct notifier_block *nb) 2637 { 2638 2639 EVENTHANDLER_DEREGISTER(ifaddr_event, 2640 nb->tags[NETDEV_CHANGEIFADDR]); 2641 2642 return (0); 2643 } 2644 2645 struct list_sort_thunk { 2646 int (*cmp)(void *, struct list_head *, struct list_head *); 2647 void *priv; 2648 }; 2649 2650 static inline int 2651 linux_le_cmp(const void *d1, const void *d2, void *priv) 2652 { 2653 struct list_head *le1, *le2; 2654 struct list_sort_thunk *thunk; 2655 2656 thunk = priv; 2657 le1 = *(__DECONST(struct list_head **, d1)); 2658 le2 = *(__DECONST(struct list_head **, d2)); 2659 return ((thunk->cmp)(thunk->priv, le1, le2)); 2660 } 2661 2662 void 2663 list_sort(void *priv, struct list_head *head, int (*cmp)(void *priv, 2664 struct list_head *a, struct list_head *b)) 2665 { 2666 struct list_sort_thunk thunk; 2667 struct list_head **ar, *le; 2668 size_t count, i; 2669 2670 count = 0; 2671 list_for_each(le, head) 2672 count++; 2673 ar = malloc(sizeof(struct list_head *) * count, M_KMALLOC, M_WAITOK); 2674 i = 0; 2675 list_for_each(le, head) 2676 ar[i++] = le; 2677 thunk.cmp = cmp; 2678 thunk.priv = priv; 2679 qsort_r(ar, count, sizeof(struct list_head *), linux_le_cmp, &thunk); 2680 INIT_LIST_HEAD(head); 2681 for (i = 0; i < count; i++) 2682 list_add_tail(ar[i], head); 2683 free(ar, M_KMALLOC); 2684 } 2685 2686 #if defined(__i386__) || defined(__amd64__) 2687 int 2688 linux_wbinvd_on_all_cpus(void) 2689 { 2690 2691 pmap_invalidate_cache(); 2692 return (0); 2693 } 2694 #endif 2695 2696 int 2697 linux_on_each_cpu(void callback(void *), void *data) 2698 { 2699 2700 smp_rendezvous(smp_no_rendezvous_barrier, callback, 2701 smp_no_rendezvous_barrier, data); 2702 return (0); 2703 } 2704 2705 int 2706 linux_in_atomic(void) 2707 { 2708 2709 return ((curthread->td_pflags & TDP_NOFAULTING) != 0); 2710 } 2711 2712 struct linux_cdev * 2713 linux_find_cdev(const char *name, unsigned major, unsigned minor) 2714 { 2715 dev_t dev = MKDEV(major, minor); 2716 struct cdev *cdev; 2717 2718 dev_lock(); 2719 LIST_FOREACH(cdev, &linuxcdevsw.d_devs, si_list) { 2720 struct linux_cdev *ldev = cdev->si_drv1; 2721 if (ldev->dev == dev && 2722 strcmp(kobject_name(&ldev->kobj), name) == 0) { 2723 break; 2724 } 2725 } 2726 dev_unlock(); 2727 2728 return (cdev != NULL ? cdev->si_drv1 : NULL); 2729 } 2730 2731 int 2732 __register_chrdev(unsigned int major, unsigned int baseminor, 2733 unsigned int count, const char *name, 2734 const struct file_operations *fops) 2735 { 2736 struct linux_cdev *cdev; 2737 int ret = 0; 2738 int i; 2739 2740 for (i = baseminor; i < baseminor + count; i++) { 2741 cdev = cdev_alloc(); 2742 cdev->ops = fops; 2743 kobject_set_name(&cdev->kobj, name); 2744 2745 ret = cdev_add(cdev, makedev(major, i), 1); 2746 if (ret != 0) 2747 break; 2748 } 2749 return (ret); 2750 } 2751 2752 int 2753 __register_chrdev_p(unsigned int major, unsigned int baseminor, 2754 unsigned int count, const char *name, 2755 const struct file_operations *fops, uid_t uid, 2756 gid_t gid, int mode) 2757 { 2758 struct linux_cdev *cdev; 2759 int ret = 0; 2760 int i; 2761 2762 for (i = baseminor; i < baseminor + count; i++) { 2763 cdev = cdev_alloc(); 2764 cdev->ops = fops; 2765 kobject_set_name(&cdev->kobj, name); 2766 2767 ret = cdev_add_ext(cdev, makedev(major, i), uid, gid, mode); 2768 if (ret != 0) 2769 break; 2770 } 2771 return (ret); 2772 } 2773 2774 void 2775 __unregister_chrdev(unsigned int major, unsigned int baseminor, 2776 unsigned int count, const char *name) 2777 { 2778 struct linux_cdev *cdevp; 2779 int i; 2780 2781 for (i = baseminor; i < baseminor + count; i++) { 2782 cdevp = linux_find_cdev(name, major, i); 2783 if (cdevp != NULL) 2784 cdev_del(cdevp); 2785 } 2786 } 2787 2788 void 2789 linux_dump_stack(void) 2790 { 2791 #ifdef STACK 2792 struct stack st; 2793 2794 stack_save(&st); 2795 stack_print(&st); 2796 #endif 2797 } 2798 2799 int 2800 linuxkpi_net_ratelimit(void) 2801 { 2802 2803 return (ppsratecheck(&lkpi_net_lastlog, &lkpi_net_curpps, 2804 lkpi_net_maxpps)); 2805 } 2806 2807 struct io_mapping * 2808 io_mapping_create_wc(resource_size_t base, unsigned long size) 2809 { 2810 struct io_mapping *mapping; 2811 2812 mapping = kmalloc(sizeof(*mapping), GFP_KERNEL); 2813 if (mapping == NULL) 2814 return (NULL); 2815 return (io_mapping_init_wc(mapping, base, size)); 2816 } 2817 2818 /* We likely want a linuxkpi_device.c at some point. */ 2819 bool 2820 device_can_wakeup(struct device *dev) 2821 { 2822 2823 if (dev == NULL) 2824 return (false); 2825 /* 2826 * XXX-BZ iwlwifi queries it as part of enabling WoWLAN. 2827 * Normally this would be based on a bool in dev->power.XXX. 2828 * Check such as PCI PCIM_PCAP_*PME. We have no way to enable this yet. 2829 * We may get away by directly calling into bsddev for as long as 2830 * we can assume PCI only avoiding changing struct device breaking KBI. 2831 */ 2832 pr_debug("%s:%d: not enabled; see comment.\n", __func__, __LINE__); 2833 return (false); 2834 } 2835 2836 static void 2837 devm_device_group_remove(struct device *dev, void *p) 2838 { 2839 const struct attribute_group **dr = p; 2840 const struct attribute_group *group = *dr; 2841 2842 sysfs_remove_group(&dev->kobj, group); 2843 } 2844 2845 int 2846 lkpi_devm_device_add_group(struct device *dev, 2847 const struct attribute_group *group) 2848 { 2849 const struct attribute_group **dr; 2850 int ret; 2851 2852 dr = devres_alloc(devm_device_group_remove, sizeof(*dr), GFP_KERNEL); 2853 if (dr == NULL) 2854 return (-ENOMEM); 2855 2856 ret = sysfs_create_group(&dev->kobj, group); 2857 if (ret == 0) { 2858 *dr = group; 2859 devres_add(dev, dr); 2860 } else 2861 devres_free(dr); 2862 2863 return (ret); 2864 } 2865 2866 #if defined(__i386__) || defined(__amd64__) 2867 bool linux_cpu_has_clflush; 2868 struct cpuinfo_x86 boot_cpu_data; 2869 struct cpuinfo_x86 *__cpu_data; 2870 #endif 2871 2872 cpumask_t * 2873 lkpi_get_static_single_cpu_mask(int cpuid) 2874 { 2875 2876 KASSERT((cpuid >= 0 && cpuid <= mp_maxid), ("%s: invalid cpuid %d\n", 2877 __func__, cpuid)); 2878 KASSERT(!CPU_ABSENT(cpuid), ("%s: cpu with cpuid %d is absent\n", 2879 __func__, cpuid)); 2880 2881 return (static_single_cpu_mask[cpuid]); 2882 } 2883 2884 bool 2885 lkpi_xen_initial_domain(void) 2886 { 2887 #ifdef XENHVM 2888 return (xen_initial_domain()); 2889 #else 2890 return (false); 2891 #endif 2892 } 2893 2894 bool 2895 lkpi_xen_pv_domain(void) 2896 { 2897 #ifdef XENHVM 2898 return (xen_pv_domain()); 2899 #else 2900 return (false); 2901 #endif 2902 } 2903 2904 static void 2905 linux_compat_init(void *arg) 2906 { 2907 struct sysctl_oid *rootoid; 2908 int i; 2909 2910 #if defined(__i386__) || defined(__amd64__) 2911 static const uint32_t x86_vendors[X86_VENDOR_NUM] = { 2912 [X86_VENDOR_INTEL] = CPU_VENDOR_INTEL, 2913 [X86_VENDOR_CYRIX] = CPU_VENDOR_CYRIX, 2914 [X86_VENDOR_AMD] = CPU_VENDOR_AMD, 2915 [X86_VENDOR_UMC] = CPU_VENDOR_UMC, 2916 [X86_VENDOR_CENTAUR] = CPU_VENDOR_CENTAUR, 2917 [X86_VENDOR_TRANSMETA] = CPU_VENDOR_TRANSMETA, 2918 [X86_VENDOR_NSC] = CPU_VENDOR_NSC, 2919 [X86_VENDOR_HYGON] = CPU_VENDOR_HYGON, 2920 }; 2921 uint8_t x86_vendor = X86_VENDOR_UNKNOWN; 2922 2923 for (i = 0; i < X86_VENDOR_NUM; i++) { 2924 if (cpu_vendor_id != 0 && cpu_vendor_id == x86_vendors[i]) { 2925 x86_vendor = i; 2926 break; 2927 } 2928 } 2929 linux_cpu_has_clflush = (cpu_feature & CPUID_CLFSH); 2930 boot_cpu_data.x86_clflush_size = cpu_clflush_line_size; 2931 boot_cpu_data.x86_max_cores = mp_ncpus; 2932 boot_cpu_data.x86 = CPUID_TO_FAMILY(cpu_id); 2933 boot_cpu_data.x86_model = CPUID_TO_MODEL(cpu_id); 2934 boot_cpu_data.x86_vendor = x86_vendor; 2935 boot_cpu_data.x86_stepping = CPUID_TO_STEPPING(cpu_id); 2936 2937 __cpu_data = kmalloc_array(mp_maxid + 1, 2938 sizeof(*__cpu_data), M_WAITOK | M_ZERO); 2939 CPU_FOREACH(i) { 2940 __cpu_data[i].x86_clflush_size = cpu_clflush_line_size; 2941 __cpu_data[i].x86_max_cores = mp_ncpus; 2942 __cpu_data[i].x86 = CPUID_TO_FAMILY(cpu_id); 2943 __cpu_data[i].x86_model = CPUID_TO_MODEL(cpu_id); 2944 __cpu_data[i].x86_vendor = x86_vendor; 2945 } 2946 #endif 2947 rw_init(&linux_vma_lock, "lkpi-vma-lock"); 2948 2949 rootoid = SYSCTL_ADD_ROOT_NODE(NULL, 2950 OID_AUTO, "sys", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "sys"); 2951 kobject_init(&linux_class_root, &linux_class_ktype); 2952 kobject_set_name(&linux_class_root, "class"); 2953 linux_class_root.oidp = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(rootoid), 2954 OID_AUTO, "class", CTLFLAG_RD|CTLFLAG_MPSAFE, NULL, "class"); 2955 kobject_init(&linux_root_device.kobj, &linux_dev_ktype); 2956 kobject_set_name(&linux_root_device.kobj, "device"); 2957 linux_root_device.kobj.oidp = SYSCTL_ADD_NODE(NULL, 2958 SYSCTL_CHILDREN(rootoid), OID_AUTO, "device", 2959 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "device"); 2960 linux_root_device.bsddev = root_bus; 2961 linux_class_misc.name = "misc"; 2962 class_register(&linux_class_misc); 2963 INIT_LIST_HEAD(&pci_drivers); 2964 INIT_LIST_HEAD(&pci_devices); 2965 spin_lock_init(&pci_lock); 2966 init_waitqueue_head(&linux_bit_waitq); 2967 init_waitqueue_head(&linux_var_waitq); 2968 2969 CPU_COPY(&all_cpus, &cpu_online_mask); 2970 /* 2971 * Generate a single-CPU cpumask_t for each CPU (possibly) in the system. 2972 * CPUs are indexed from 0..(mp_maxid). The entry for cpuid 0 will only 2973 * have itself in the cpumask, cupid 1 only itself on entry 1, and so on. 2974 * This is used by cpumask_of() (and possibly others in the future) for, 2975 * e.g., drivers to pass hints to irq_set_affinity_hint(). 2976 */ 2977 static_single_cpu_mask = kmalloc_array(mp_maxid + 1, 2978 sizeof(static_single_cpu_mask), M_WAITOK | M_ZERO); 2979 2980 /* 2981 * When the number of CPUs reach a threshold, we start to save memory 2982 * given the sets are static by overlapping those having their single 2983 * bit set at same position in a bitset word. Asymptotically, this 2984 * regular scheme is in O(n²) whereas the overlapping one is in O(n) 2985 * only with n being the maximum number of CPUs, so the gain will become 2986 * huge quite quickly. The threshold for 64-bit architectures is 128 2987 * CPUs. 2988 */ 2989 if (mp_ncpus < (2 * _BITSET_BITS)) { 2990 cpumask_t *sscm_ptr; 2991 2992 /* 2993 * This represents 'mp_ncpus * __bitset_words(CPU_SETSIZE) * 2994 * (_BITSET_BITS / 8)' bytes (for comparison with the 2995 * overlapping scheme). 2996 */ 2997 static_single_cpu_mask_lcs = kmalloc_array(mp_ncpus, 2998 sizeof(*static_single_cpu_mask_lcs), 2999 M_WAITOK | M_ZERO); 3000 3001 sscm_ptr = static_single_cpu_mask_lcs; 3002 CPU_FOREACH(i) { 3003 static_single_cpu_mask[i] = sscm_ptr++; 3004 CPU_SET(i, static_single_cpu_mask[i]); 3005 } 3006 } else { 3007 /* Pointer to a bitset word. */ 3008 __typeof(((cpuset_t *)NULL)->__bits[0]) *bwp; 3009 3010 /* 3011 * Allocate memory for (static) spans of 'cpumask_t' ('cpuset_t' 3012 * really) with a single bit set that can be reused for all 3013 * single CPU masks by making them start at different offsets. 3014 * We need '__bitset_words(CPU_SETSIZE) - 1' bitset words before 3015 * the word having its single bit set, and the same amount 3016 * after. 3017 */ 3018 static_single_cpu_mask_lcs = mallocarray(_BITSET_BITS, 3019 (2 * __bitset_words(CPU_SETSIZE) - 1) * (_BITSET_BITS / 8), 3020 M_KMALLOC, M_WAITOK | M_ZERO); 3021 3022 /* 3023 * We rely below on cpuset_t and the bitset generic 3024 * implementation assigning words in the '__bits' array in the 3025 * same order of bits (i.e., little-endian ordering, not to be 3026 * confused with machine endianness, which concerns bits in 3027 * words and other integers). This is an imperfect test, but it 3028 * will detect a change to big-endian ordering. 3029 */ 3030 _Static_assert( 3031 __bitset_word(_BITSET_BITS + 1, _BITSET_BITS) == 1, 3032 "Assumes a bitset implementation that is little-endian " 3033 "on its words"); 3034 3035 /* Initialize the single bit of each static span. */ 3036 bwp = (__typeof(bwp))static_single_cpu_mask_lcs + 3037 (__bitset_words(CPU_SETSIZE) - 1); 3038 for (i = 0; i < _BITSET_BITS; i++) { 3039 CPU_SET(i, (cpuset_t *)bwp); 3040 bwp += (2 * __bitset_words(CPU_SETSIZE) - 1); 3041 } 3042 3043 /* 3044 * Finally set all CPU masks to the proper word in their 3045 * relevant span. 3046 */ 3047 CPU_FOREACH(i) { 3048 bwp = (__typeof(bwp))static_single_cpu_mask_lcs; 3049 /* Find the non-zero word of the relevant span. */ 3050 bwp += (2 * __bitset_words(CPU_SETSIZE) - 1) * 3051 (i % _BITSET_BITS) + 3052 __bitset_words(CPU_SETSIZE) - 1; 3053 /* Shift to find the CPU mask start. */ 3054 bwp -= (i / _BITSET_BITS); 3055 static_single_cpu_mask[i] = (cpuset_t *)bwp; 3056 } 3057 } 3058 3059 strlcpy(init_uts_ns.name.release, osrelease, sizeof(init_uts_ns.name.release)); 3060 } 3061 SYSINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_init, NULL); 3062 3063 static void 3064 linux_compat_uninit(void *arg) 3065 { 3066 linux_kobject_kfree_name(&linux_class_root); 3067 linux_kobject_kfree_name(&linux_root_device.kobj); 3068 linux_kobject_kfree_name(&linux_class_misc.kobj); 3069 3070 free(static_single_cpu_mask_lcs, M_KMALLOC); 3071 free(static_single_cpu_mask, M_KMALLOC); 3072 #if defined(__i386__) || defined(__amd64__) 3073 free(__cpu_data, M_KMALLOC); 3074 #endif 3075 3076 spin_lock_destroy(&pci_lock); 3077 rw_destroy(&linux_vma_lock); 3078 } 3079 SYSUNINIT(linux_compat, SI_SUB_DRIVERS, SI_ORDER_SECOND, linux_compat_uninit, NULL); 3080 3081 /* 3082 * NOTE: Linux frequently uses "unsigned long" for pointer to integer 3083 * conversion and vice versa, where in FreeBSD "uintptr_t" would be 3084 * used. Assert these types have the same size, else some parts of the 3085 * LinuxKPI may not work like expected: 3086 */ 3087 CTASSERT(sizeof(unsigned long) == sizeof(uintptr_t)); 3088