1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2026 Micron Technology, Inc. */ 3 #include <linux/memremap.h> 4 #include <linux/pagemap.h> 5 #include <linux/module.h> 6 #include <linux/device.h> 7 #include <linux/cdev.h> 8 #include <linux/slab.h> 9 #include <linux/dax.h> 10 #include <linux/uio.h> 11 #include <linux/fs.h> 12 #include <linux/mm.h> 13 #include "dax-private.h" 14 #include "bus.h" 15 16 /* 17 * FS-DAX compatible devdax driver 18 * 19 * Unlike drivers/dax/device.c which pre-initializes compound folios based 20 * on device alignment (via vmemmap_shift), this driver leaves folios 21 * uninitialized similar to pmem. This allows fs-dax filesystems like famfs 22 * to work without needing special handling for pre-initialized folios. 23 * 24 * Key differences from device.c: 25 * - pgmap type is MEMORY_DEVICE_FS_DAX (not MEMORY_DEVICE_GENERIC) 26 * - vmemmap_shift is NOT set (folios remain order-0) 27 * - fs-dax can dynamically create compound folios as needed 28 * - No mmap support - all access is through fs-dax/iomap 29 */ 30 31 static void fsdev_write_dax(void *addr, struct page *page, 32 unsigned int off, unsigned int len) 33 { 34 while (len) { 35 void *mem = kmap_local_page(page); 36 unsigned int chunk = min_t(unsigned int, len, PAGE_SIZE - off); 37 38 memcpy_flushcache(addr, mem + off, chunk); 39 kunmap_local(mem); 40 len -= chunk; 41 off = 0; 42 page++; 43 addr += chunk; 44 } 45 } 46 47 static long __fsdev_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, 48 long nr_pages, enum dax_access_mode mode, void **kaddr, 49 unsigned long *pfn) 50 { 51 struct dev_dax *dev_dax = dax_get_private(dax_dev); 52 size_t size = nr_pages << PAGE_SHIFT; 53 size_t offset = pgoff << PAGE_SHIFT; 54 phys_addr_t phys; 55 56 phys = dax_pgoff_to_phys(dev_dax, pgoff, size); 57 if (phys == -1) { 58 dev_dbg(&dev_dax->dev, 59 "pgoff (%#lx) out of range\n", pgoff); 60 return -EFAULT; 61 } 62 63 if (kaddr) 64 *kaddr = __va(phys); 65 66 if (pfn) 67 *pfn = PHYS_PFN(phys); 68 69 /* 70 * Use cached_size which was computed at probe time. The size cannot 71 * change while the driver is bound (resize returns -EBUSY). 72 */ 73 return PHYS_PFN(min(size, dev_dax->cached_size - offset)); 74 } 75 76 static int fsdev_dax_zero_page_range(struct dax_device *dax_dev, 77 pgoff_t pgoff, size_t nr_pages) 78 { 79 void *kaddr; 80 long rc; 81 82 WARN_ONCE(nr_pages > 1, "%s: nr_pages > 1\n", __func__); 83 rc = __fsdev_dax_direct_access(dax_dev, pgoff, 1, DAX_ACCESS, 84 &kaddr, NULL); 85 if (rc < 0) 86 return rc; 87 fsdev_write_dax(kaddr, ZERO_PAGE(0), 0, PAGE_SIZE); 88 return 0; 89 } 90 91 static long fsdev_dax_direct_access(struct dax_device *dax_dev, 92 pgoff_t pgoff, long nr_pages, enum dax_access_mode mode, 93 void **kaddr, unsigned long *pfn) 94 { 95 return __fsdev_dax_direct_access(dax_dev, pgoff, nr_pages, mode, 96 kaddr, pfn); 97 } 98 99 static size_t fsdev_dax_recovery_write(struct dax_device *dax_dev, 100 pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i) 101 { 102 return _copy_from_iter_flushcache(addr, bytes, i); 103 } 104 105 static const struct dax_operations dev_dax_ops = { 106 .direct_access = fsdev_dax_direct_access, 107 .zero_page_range = fsdev_dax_zero_page_range, 108 .recovery_write = fsdev_dax_recovery_write, 109 }; 110 111 static void fsdev_cdev_del(void *cdev) 112 { 113 cdev_del(cdev); 114 } 115 116 static void fsdev_kill(void *dev_dax) 117 { 118 kill_dev_dax(dev_dax); 119 } 120 121 static void fsdev_clear_ops(void *data) 122 { 123 struct dev_dax *dev_dax = data; 124 125 dax_set_ops(dev_dax->dax_dev, NULL); 126 } 127 128 static void fsdev_clear_pgmap_ops(void *data) 129 { 130 struct dev_pagemap *pgmap = data; 131 132 /* 133 * fsdev installs pgmap->ops and ->owner at probe. For a static device 134 * the pgmap is shared and long-lived (owned by the dax bus), so 135 * leaving fsdev's ops behind on unbind would let a later 136 * memory_failure -- after rebind to another driver, or after this 137 * module is unloaded -- dispatch through a stale or freed 138 * ->memory_failure handler. Clear them so the pgmap carries no fsdev 139 * state once we are unbound. 140 */ 141 pgmap->ops = NULL; 142 pgmap->owner = NULL; 143 } 144 145 /* 146 * Page map operations for FS-DAX mode 147 * Similar to fsdax_pagemap_ops in drivers/nvdimm/pmem.c 148 * 149 * Note: folio_free callback is not needed for MEMORY_DEVICE_FS_DAX. 150 * The core mm code in free_zone_device_folio() handles the wake_up_var() 151 * directly for this memory type. 152 */ 153 static u64 fsdev_pfn_to_offset(struct dev_dax *dev_dax, unsigned long pfn) 154 { 155 phys_addr_t phys = PFN_PHYS(pfn); 156 u64 offset = 0; 157 158 for (int i = 0; i < dev_dax->nr_range; i++) { 159 struct range *range = &dev_dax->ranges[i].range; 160 161 if (phys >= range->start && phys <= range->end) 162 return offset + (phys - range->start); 163 offset += range_len(range); 164 } 165 return -1ULL; 166 } 167 168 static int fsdev_pagemap_memory_failure(struct dev_pagemap *pgmap, 169 unsigned long pfn, unsigned long nr_pages, int mf_flags) 170 { 171 struct dev_dax *dev_dax = pgmap->owner; 172 u64 offset = fsdev_pfn_to_offset(dev_dax, pfn); 173 u64 len = nr_pages << PAGE_SHIFT; 174 175 return dax_holder_notify_failure(dev_dax->dax_dev, offset, 176 len, mf_flags); 177 } 178 179 static const struct dev_pagemap_ops fsdev_pagemap_ops = { 180 .memory_failure = fsdev_pagemap_memory_failure, 181 }; 182 183 /* 184 * Clear any stale folio state from pages in the given range. 185 * This is necessary because device_dax pre-initializes compound folios 186 * based on vmemmap_shift, and that state may persist after driver unbind. 187 * Since fsdev_dax uses MEMORY_DEVICE_FS_DAX without vmemmap_shift, fs-dax 188 * expects to find clean order-0 folios that it can build into compound 189 * folios on demand. 190 * 191 * At probe time, no filesystem should be mounted yet, so all mappings 192 * are stale and must be cleared along with compound state. 193 */ 194 static void fsdev_clear_folio_state(struct dev_dax *dev_dax) 195 { 196 for (int i = 0; i < dev_dax->nr_range; i++) { 197 struct range *range = &dev_dax->ranges[i].range; 198 unsigned long pfn = PHYS_PFN(range->start); 199 unsigned long end_pfn = PHYS_PFN(range->end) + 1; 200 201 while (pfn < end_pfn) { 202 struct folio *folio = pfn_folio(pfn); 203 int order = dax_folio_reset_order(folio); 204 205 pfn += 1UL << order; 206 } 207 } 208 } 209 210 static void fsdev_clear_folio_state_action(void *data) 211 { 212 fsdev_clear_folio_state(data); 213 } 214 215 static int fsdev_open(struct inode *inode, struct file *filp) 216 { 217 struct dax_device *dax_dev = inode_dax(inode); 218 struct dev_dax *dev_dax = dax_get_private(dax_dev); 219 220 filp->private_data = dev_dax; 221 222 return 0; 223 } 224 225 static int fsdev_release(struct inode *inode, struct file *filp) 226 { 227 return 0; 228 } 229 230 static const struct file_operations fsdev_fops = { 231 .llseek = noop_llseek, 232 .owner = THIS_MODULE, 233 .open = fsdev_open, 234 .release = fsdev_release, 235 }; 236 237 /** 238 * fsdev_dax_from_file - get dax_device from an open file 239 * @file: open device file 240 * 241 * Returns a dax_device pointer if @file refers to a fsdev_dax device. 242 * Otherwise return NULL. 243 * 244 * Caller must put_dax() the returned device when done. 245 */ 246 struct dax_device *fsdev_dax_from_file(struct file *file) 247 { 248 struct dax_device *dax_dev; 249 250 if (file->f_op != &fsdev_fops) 251 return NULL; 252 253 dax_dev = inode_dax(file_inode(file)); 254 ihold(dax_inode(dax_dev)); 255 256 return dax_dev; 257 } 258 EXPORT_SYMBOL_GPL(fsdev_dax_from_file); 259 260 /* 261 * Acquire the dev_pagemap for probe: the static (pre-populated) one if 262 * present, or a devm-allocated one for the dynamic case. Note that 263 * dev_dax->pgmap is not set here; fsdev_dax_probe() sets it only once 264 * probe succeeds, so a failed probe never leaves a dangling pointer 265 * to a devres-freed pgmap. 266 */ 267 static struct dev_pagemap *fsdev_acquire_pgmap(struct dev_dax *dev_dax) 268 { 269 struct device *dev = &dev_dax->dev; 270 struct dev_pagemap *pgmap; 271 size_t pgmap_size; 272 273 if (static_dev_dax(dev_dax)) { 274 if (dev_dax->nr_range > 1) { 275 dev_warn(dev, 276 "static pgmap / multi-range device conflict\n"); 277 return ERR_PTR(-EINVAL); 278 } 279 280 pgmap = dev_dax->pgmap; 281 pgmap->vmemmap_shift = 0; 282 return pgmap; 283 } 284 285 if (dev_dax->pgmap) { 286 dev_warn(dev, "dynamic-dax with pre-populated page map\n"); 287 return ERR_PTR(-EINVAL); 288 } 289 290 pgmap_size = struct_size(pgmap, ranges, dev_dax->nr_range - 1); 291 pgmap = devm_kzalloc(dev, pgmap_size, GFP_KERNEL); 292 if (!pgmap) 293 return ERR_PTR(-ENOMEM); 294 295 pgmap->nr_range = dev_dax->nr_range; 296 for (int i = 0; i < dev_dax->nr_range; i++) 297 pgmap->ranges[i] = dev_dax->ranges[i].range; 298 299 return pgmap; 300 } 301 302 static int fsdev_dax_probe(struct dev_dax *dev_dax) 303 { 304 struct dax_device *dax_dev = dev_dax->dax_dev; 305 struct device *dev = &dev_dax->dev; 306 struct dev_pagemap *pgmap; 307 struct inode *inode; 308 u64 data_offset = 0; 309 struct cdev *cdev; 310 void *addr; 311 int rc, i; 312 313 pgmap = fsdev_acquire_pgmap(dev_dax); 314 if (IS_ERR(pgmap)) 315 return PTR_ERR(pgmap); 316 317 for (i = 0; i < dev_dax->nr_range; i++) { 318 struct range *range = &dev_dax->ranges[i].range; 319 320 if (!devm_request_mem_region(dev, range->start, 321 range_len(range), dev_name(dev))) { 322 dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve range\n", 323 i, range->start, range->end); 324 return -EBUSY; 325 } 326 } 327 328 /* Cache size now; it cannot change while driver is bound */ 329 dev_dax->cached_size = 0; 330 for (i = 0; i < dev_dax->nr_range; i++) 331 dev_dax->cached_size += range_len(&dev_dax->ranges[i].range); 332 333 /* 334 * Use MEMORY_DEVICE_FS_DAX without setting vmemmap_shift, leaving 335 * folios at order-0. Unlike device.c (MEMORY_DEVICE_GENERIC), this 336 * lets fs-dax dynamically build compound folios as needed, similar 337 * to pmem behavior. 338 */ 339 pgmap->type = MEMORY_DEVICE_FS_DAX; 340 pgmap->ops = &fsdev_pagemap_ops; 341 pgmap->owner = dev_dax; 342 343 addr = devm_memremap_pages(dev, pgmap); 344 if (IS_ERR(addr)) 345 return PTR_ERR(addr); 346 347 /* Drop fsdev's pgmap->ops/owner on unbind so no stale ops survive. */ 348 rc = devm_add_action_or_reset(dev, fsdev_clear_pgmap_ops, pgmap); 349 if (rc) 350 return rc; 351 352 /* 353 * Clear any stale compound folio state left over from a previous 354 * driver (e.g., device_dax with vmemmap_shift). Also register this 355 * as a devm action so folio state is cleared on unbind, ensuring 356 * clean pages for subsequent drivers (e.g., kmem for system-ram). 357 */ 358 fsdev_clear_folio_state(dev_dax); 359 rc = devm_add_action_or_reset(dev, fsdev_clear_folio_state_action, 360 dev_dax); 361 if (rc) 362 return rc; 363 364 /* Detect whether the data is at a non-zero offset into the memory */ 365 if (pgmap->range.start != dev_dax->ranges[0].range.start) { 366 u64 phys = dev_dax->ranges[0].range.start; 367 u64 pgmap_phys = pgmap[0].range.start; 368 369 if (pgmap_phys > phys) { 370 dev_err(dev, "pgmap start %#llx exceeds data start %#llx\n", 371 pgmap_phys, phys); 372 return -EINVAL; 373 } 374 data_offset = phys - pgmap_phys; 375 376 pr_debug("%s: offset detected phys=%llx pgmap_phys=%llx offset=%llx\n", 377 __func__, phys, pgmap_phys, data_offset); 378 } 379 380 inode = dax_inode(dax_dev); 381 cdev = inode->i_cdev; 382 cdev_init(cdev, &fsdev_fops); 383 cdev->owner = dev->driver->owner; 384 cdev_set_parent(cdev, &dev->kobj); 385 rc = cdev_add(cdev, dev->devt, 1); 386 if (rc) 387 return rc; 388 389 rc = devm_add_action_or_reset(dev, fsdev_cdev_del, cdev); 390 if (rc) 391 return rc; 392 393 /* Set the dax operations for fs-dax access path */ 394 rc = dax_set_ops(dax_dev, &dev_dax_ops); 395 if (rc) 396 return rc; 397 398 rc = devm_add_action_or_reset(dev, fsdev_clear_ops, dev_dax); 399 if (rc) 400 return rc; 401 402 run_dax(dax_dev); 403 rc = devm_add_action_or_reset(dev, fsdev_kill, dev_dax); 404 if (rc) 405 return rc; 406 407 /* Probe can no longer fail; expose the pgmap via dev_dax */ 408 dev_dax->pgmap = pgmap; 409 return 0; 410 } 411 412 static struct dax_device_driver fsdev_dax_driver = { 413 .probe = fsdev_dax_probe, 414 .type = DAXDRV_FSDEV_TYPE, 415 }; 416 417 static int __init dax_init(void) 418 { 419 return dax_driver_register(&fsdev_dax_driver); 420 } 421 422 static void __exit dax_exit(void) 423 { 424 dax_driver_unregister(&fsdev_dax_driver); 425 } 426 427 MODULE_AUTHOR("John Groves"); 428 MODULE_DESCRIPTION("FS-DAX Device: fs-dax compatible devdax driver"); 429 MODULE_LICENSE("GPL"); 430 module_init(dax_init); 431 module_exit(dax_exit); 432 MODULE_ALIAS_DAX_DEVICE(0); 433