1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2016-2018 Intel Corporation. All rights reserved. */ 3 #include <linux/memremap.h> 4 #include <linux/pagemap.h> 5 #include <linux/module.h> 6 #include <linux/device.h> 7 #include <linux/pfn_t.h> 8 #include <linux/cdev.h> 9 #include <linux/slab.h> 10 #include <linux/dax.h> 11 #include <linux/fs.h> 12 #include <linux/mm.h> 13 #include <linux/mman.h> 14 #include "dax-private.h" 15 #include "bus.h" 16 17 static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma, 18 const char *func) 19 { 20 struct device *dev = &dev_dax->dev; 21 unsigned long mask; 22 23 if (!dax_alive(dev_dax->dax_dev)) 24 return -ENXIO; 25 26 /* prevent private mappings from being established */ 27 if ((vma->vm_flags & VM_MAYSHARE) != VM_MAYSHARE) { 28 dev_info_ratelimited(dev, 29 "%s: %s: fail, attempted private mapping\n", 30 current->comm, func); 31 return -EINVAL; 32 } 33 34 mask = dev_dax->align - 1; 35 if (vma->vm_start & mask || vma->vm_end & mask) { 36 dev_info_ratelimited(dev, 37 "%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n", 38 current->comm, func, vma->vm_start, vma->vm_end, 39 mask); 40 return -EINVAL; 41 } 42 43 if (!vma_is_dax(vma)) { 44 dev_info_ratelimited(dev, 45 "%s: %s: fail, vma is not DAX capable\n", 46 current->comm, func); 47 return -EINVAL; 48 } 49 50 return 0; 51 } 52 53 /* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */ 54 __weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff, 55 unsigned long size) 56 { 57 int i; 58 59 for (i = 0; i < dev_dax->nr_range; i++) { 60 struct dev_dax_range *dax_range = &dev_dax->ranges[i]; 61 struct range *range = &dax_range->range; 62 unsigned long long pgoff_end; 63 phys_addr_t phys; 64 65 pgoff_end = dax_range->pgoff + PHYS_PFN(range_len(range)) - 1; 66 if (pgoff < dax_range->pgoff || pgoff > pgoff_end) 67 continue; 68 phys = PFN_PHYS(pgoff - dax_range->pgoff) + range->start; 69 if (phys + size - 1 <= range->end) 70 return phys; 71 break; 72 } 73 return -1; 74 } 75 76 static void dax_set_mapping(struct vm_fault *vmf, pfn_t pfn, 77 unsigned long fault_size) 78 { 79 unsigned long i, nr_pages = fault_size / PAGE_SIZE; 80 struct file *filp = vmf->vma->vm_file; 81 struct dev_dax *dev_dax = filp->private_data; 82 pgoff_t pgoff; 83 84 /* mapping is only set on the head */ 85 if (dev_dax->pgmap->vmemmap_shift) 86 nr_pages = 1; 87 88 pgoff = linear_page_index(vmf->vma, 89 ALIGN_DOWN(vmf->address, fault_size)); 90 91 for (i = 0; i < nr_pages; i++) { 92 struct folio *folio = pfn_folio(pfn_t_to_pfn(pfn) + i); 93 94 if (folio->mapping) 95 continue; 96 97 folio->mapping = filp->f_mapping; 98 folio->index = pgoff + i; 99 } 100 } 101 102 static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax, 103 struct vm_fault *vmf) 104 { 105 struct device *dev = &dev_dax->dev; 106 phys_addr_t phys; 107 pfn_t pfn; 108 unsigned int fault_size = PAGE_SIZE; 109 110 if (check_vma(dev_dax, vmf->vma, __func__)) 111 return VM_FAULT_SIGBUS; 112 113 if (dev_dax->align > PAGE_SIZE) { 114 dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", 115 dev_dax->align, fault_size); 116 return VM_FAULT_SIGBUS; 117 } 118 119 if (fault_size != dev_dax->align) 120 return VM_FAULT_SIGBUS; 121 122 phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE); 123 if (phys == -1) { 124 dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", vmf->pgoff); 125 return VM_FAULT_SIGBUS; 126 } 127 128 pfn = phys_to_pfn_t(phys, 0); 129 130 dax_set_mapping(vmf, pfn, fault_size); 131 132 return vmf_insert_page_mkwrite(vmf, pfn_t_to_page(pfn), 133 vmf->flags & FAULT_FLAG_WRITE); 134 } 135 136 static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax, 137 struct vm_fault *vmf) 138 { 139 unsigned long pmd_addr = vmf->address & PMD_MASK; 140 struct device *dev = &dev_dax->dev; 141 phys_addr_t phys; 142 pgoff_t pgoff; 143 pfn_t pfn; 144 unsigned int fault_size = PMD_SIZE; 145 146 if (check_vma(dev_dax, vmf->vma, __func__)) 147 return VM_FAULT_SIGBUS; 148 149 if (dev_dax->align > PMD_SIZE) { 150 dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", 151 dev_dax->align, fault_size); 152 return VM_FAULT_SIGBUS; 153 } 154 155 if (fault_size < dev_dax->align) 156 return VM_FAULT_SIGBUS; 157 else if (fault_size > dev_dax->align) 158 return VM_FAULT_FALLBACK; 159 160 /* if we are outside of the VMA */ 161 if (pmd_addr < vmf->vma->vm_start || 162 (pmd_addr + PMD_SIZE) > vmf->vma->vm_end) 163 return VM_FAULT_SIGBUS; 164 165 pgoff = linear_page_index(vmf->vma, pmd_addr); 166 phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE); 167 if (phys == -1) { 168 dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff); 169 return VM_FAULT_SIGBUS; 170 } 171 172 pfn = phys_to_pfn_t(phys, 0); 173 174 dax_set_mapping(vmf, pfn, fault_size); 175 176 return vmf_insert_folio_pmd(vmf, page_folio(pfn_t_to_page(pfn)), 177 vmf->flags & FAULT_FLAG_WRITE); 178 } 179 180 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 181 static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax, 182 struct vm_fault *vmf) 183 { 184 unsigned long pud_addr = vmf->address & PUD_MASK; 185 struct device *dev = &dev_dax->dev; 186 phys_addr_t phys; 187 pgoff_t pgoff; 188 pfn_t pfn; 189 unsigned int fault_size = PUD_SIZE; 190 191 192 if (check_vma(dev_dax, vmf->vma, __func__)) 193 return VM_FAULT_SIGBUS; 194 195 if (dev_dax->align > PUD_SIZE) { 196 dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", 197 dev_dax->align, fault_size); 198 return VM_FAULT_SIGBUS; 199 } 200 201 if (fault_size < dev_dax->align) 202 return VM_FAULT_SIGBUS; 203 else if (fault_size > dev_dax->align) 204 return VM_FAULT_FALLBACK; 205 206 /* if we are outside of the VMA */ 207 if (pud_addr < vmf->vma->vm_start || 208 (pud_addr + PUD_SIZE) > vmf->vma->vm_end) 209 return VM_FAULT_SIGBUS; 210 211 pgoff = linear_page_index(vmf->vma, pud_addr); 212 phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE); 213 if (phys == -1) { 214 dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff); 215 return VM_FAULT_SIGBUS; 216 } 217 218 pfn = phys_to_pfn_t(phys, 0); 219 220 dax_set_mapping(vmf, pfn, fault_size); 221 222 return vmf_insert_folio_pud(vmf, page_folio(pfn_t_to_page(pfn)), 223 vmf->flags & FAULT_FLAG_WRITE); 224 } 225 #else 226 static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax, 227 struct vm_fault *vmf) 228 { 229 return VM_FAULT_FALLBACK; 230 } 231 #endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ 232 233 static vm_fault_t dev_dax_huge_fault(struct vm_fault *vmf, unsigned int order) 234 { 235 struct file *filp = vmf->vma->vm_file; 236 vm_fault_t rc = VM_FAULT_SIGBUS; 237 int id; 238 struct dev_dax *dev_dax = filp->private_data; 239 240 dev_dbg(&dev_dax->dev, "%s: op=%s addr=%#lx order=%d\n", current->comm, 241 (vmf->flags & FAULT_FLAG_WRITE) ? "write" : "read", 242 vmf->address & ~((1UL << (order + PAGE_SHIFT)) - 1), order); 243 244 id = dax_read_lock(); 245 if (order == 0) 246 rc = __dev_dax_pte_fault(dev_dax, vmf); 247 else if (order == PMD_ORDER) 248 rc = __dev_dax_pmd_fault(dev_dax, vmf); 249 else if (order == PUD_ORDER) 250 rc = __dev_dax_pud_fault(dev_dax, vmf); 251 else 252 rc = VM_FAULT_SIGBUS; 253 254 dax_read_unlock(id); 255 256 return rc; 257 } 258 259 static vm_fault_t dev_dax_fault(struct vm_fault *vmf) 260 { 261 return dev_dax_huge_fault(vmf, 0); 262 } 263 264 static int dev_dax_may_split(struct vm_area_struct *vma, unsigned long addr) 265 { 266 struct file *filp = vma->vm_file; 267 struct dev_dax *dev_dax = filp->private_data; 268 269 if (!IS_ALIGNED(addr, dev_dax->align)) 270 return -EINVAL; 271 return 0; 272 } 273 274 static unsigned long dev_dax_pagesize(struct vm_area_struct *vma) 275 { 276 struct file *filp = vma->vm_file; 277 struct dev_dax *dev_dax = filp->private_data; 278 279 return dev_dax->align; 280 } 281 282 static const struct vm_operations_struct dax_vm_ops = { 283 .fault = dev_dax_fault, 284 .huge_fault = dev_dax_huge_fault, 285 .may_split = dev_dax_may_split, 286 .pagesize = dev_dax_pagesize, 287 }; 288 289 static int dax_mmap(struct file *filp, struct vm_area_struct *vma) 290 { 291 struct dev_dax *dev_dax = filp->private_data; 292 int rc, id; 293 294 dev_dbg(&dev_dax->dev, "trace\n"); 295 296 /* 297 * We lock to check dax_dev liveness and will re-check at 298 * fault time. 299 */ 300 id = dax_read_lock(); 301 rc = check_vma(dev_dax, vma, __func__); 302 dax_read_unlock(id); 303 if (rc) 304 return rc; 305 306 vma->vm_ops = &dax_vm_ops; 307 vm_flags_set(vma, VM_HUGEPAGE); 308 return 0; 309 } 310 311 /* return an unmapped area aligned to the dax region specified alignment */ 312 static unsigned long dax_get_unmapped_area(struct file *filp, 313 unsigned long addr, unsigned long len, unsigned long pgoff, 314 unsigned long flags) 315 { 316 unsigned long off, off_end, off_align, len_align, addr_align, align; 317 struct dev_dax *dev_dax = filp ? filp->private_data : NULL; 318 319 if (!dev_dax || addr) 320 goto out; 321 322 align = dev_dax->align; 323 off = pgoff << PAGE_SHIFT; 324 off_end = off + len; 325 off_align = round_up(off, align); 326 327 if ((off_end <= off_align) || ((off_end - off_align) < align)) 328 goto out; 329 330 len_align = len + align; 331 if ((off + len_align) < off) 332 goto out; 333 334 addr_align = mm_get_unmapped_area(current->mm, filp, addr, len_align, 335 pgoff, flags); 336 if (!IS_ERR_VALUE(addr_align)) { 337 addr_align += (off - addr_align) & (align - 1); 338 return addr_align; 339 } 340 out: 341 return mm_get_unmapped_area(current->mm, filp, addr, len, pgoff, flags); 342 } 343 344 static const struct address_space_operations dev_dax_aops = { 345 .dirty_folio = noop_dirty_folio, 346 }; 347 348 static int dax_open(struct inode *inode, struct file *filp) 349 { 350 struct dax_device *dax_dev = inode_dax(inode); 351 struct inode *__dax_inode = dax_inode(dax_dev); 352 struct dev_dax *dev_dax = dax_get_private(dax_dev); 353 354 dev_dbg(&dev_dax->dev, "trace\n"); 355 inode->i_mapping = __dax_inode->i_mapping; 356 inode->i_mapping->host = __dax_inode; 357 inode->i_mapping->a_ops = &dev_dax_aops; 358 filp->f_mapping = inode->i_mapping; 359 filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping); 360 filp->f_sb_err = file_sample_sb_err(filp); 361 filp->private_data = dev_dax; 362 inode->i_flags = S_DAX; 363 364 return 0; 365 } 366 367 static int dax_release(struct inode *inode, struct file *filp) 368 { 369 struct dev_dax *dev_dax = filp->private_data; 370 371 dev_dbg(&dev_dax->dev, "trace\n"); 372 return 0; 373 } 374 375 static const struct file_operations dax_fops = { 376 .llseek = noop_llseek, 377 .owner = THIS_MODULE, 378 .open = dax_open, 379 .release = dax_release, 380 .get_unmapped_area = dax_get_unmapped_area, 381 .mmap = dax_mmap, 382 .fop_flags = FOP_MMAP_SYNC, 383 }; 384 385 static void dev_dax_cdev_del(void *cdev) 386 { 387 cdev_del(cdev); 388 } 389 390 static void dev_dax_kill(void *dev_dax) 391 { 392 kill_dev_dax(dev_dax); 393 } 394 395 static int dev_dax_probe(struct dev_dax *dev_dax) 396 { 397 struct dax_device *dax_dev = dev_dax->dax_dev; 398 struct device *dev = &dev_dax->dev; 399 struct dev_pagemap *pgmap; 400 struct inode *inode; 401 struct cdev *cdev; 402 void *addr; 403 int rc, i; 404 405 if (static_dev_dax(dev_dax)) { 406 if (dev_dax->nr_range > 1) { 407 dev_warn(dev, 408 "static pgmap / multi-range device conflict\n"); 409 return -EINVAL; 410 } 411 412 pgmap = dev_dax->pgmap; 413 } else { 414 if (dev_dax->pgmap) { 415 dev_warn(dev, 416 "dynamic-dax with pre-populated page map\n"); 417 return -EINVAL; 418 } 419 420 pgmap = devm_kzalloc(dev, 421 struct_size(pgmap, ranges, dev_dax->nr_range - 1), 422 GFP_KERNEL); 423 if (!pgmap) 424 return -ENOMEM; 425 426 pgmap->nr_range = dev_dax->nr_range; 427 dev_dax->pgmap = pgmap; 428 429 for (i = 0; i < dev_dax->nr_range; i++) { 430 struct range *range = &dev_dax->ranges[i].range; 431 pgmap->ranges[i] = *range; 432 } 433 } 434 435 for (i = 0; i < dev_dax->nr_range; i++) { 436 struct range *range = &dev_dax->ranges[i].range; 437 438 if (!devm_request_mem_region(dev, range->start, 439 range_len(range), dev_name(dev))) { 440 dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve range\n", 441 i, range->start, range->end); 442 return -EBUSY; 443 } 444 } 445 446 pgmap->type = MEMORY_DEVICE_GENERIC; 447 if (dev_dax->align > PAGE_SIZE) 448 pgmap->vmemmap_shift = 449 order_base_2(dev_dax->align >> PAGE_SHIFT); 450 addr = devm_memremap_pages(dev, pgmap); 451 if (IS_ERR(addr)) 452 return PTR_ERR(addr); 453 454 inode = dax_inode(dax_dev); 455 cdev = inode->i_cdev; 456 cdev_init(cdev, &dax_fops); 457 cdev->owner = dev->driver->owner; 458 cdev_set_parent(cdev, &dev->kobj); 459 rc = cdev_add(cdev, dev->devt, 1); 460 if (rc) 461 return rc; 462 463 rc = devm_add_action_or_reset(dev, dev_dax_cdev_del, cdev); 464 if (rc) 465 return rc; 466 467 run_dax(dax_dev); 468 return devm_add_action_or_reset(dev, dev_dax_kill, dev_dax); 469 } 470 471 static struct dax_device_driver device_dax_driver = { 472 .probe = dev_dax_probe, 473 .type = DAXDRV_DEVICE_TYPE, 474 }; 475 476 static int __init dax_init(void) 477 { 478 return dax_driver_register(&device_dax_driver); 479 } 480 481 static void __exit dax_exit(void) 482 { 483 dax_driver_unregister(&device_dax_driver); 484 } 485 486 MODULE_AUTHOR("Intel Corporation"); 487 MODULE_DESCRIPTION("Device DAX: direct access device driver"); 488 MODULE_LICENSE("GPL v2"); 489 module_init(dax_init); 490 module_exit(dax_exit); 491 MODULE_ALIAS_DAX_DEVICE(0); 492