1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2026 Micron Technology, Inc. */ 3 #include <linux/memremap.h> 4 #include <linux/pagemap.h> 5 #include <linux/module.h> 6 #include <linux/device.h> 7 #include <linux/cdev.h> 8 #include <linux/slab.h> 9 #include <linux/dax.h> 10 #include <linux/uio.h> 11 #include <linux/fs.h> 12 #include <linux/mm.h> 13 #include "dax-private.h" 14 #include "bus.h" 15 16 /* 17 * FS-DAX compatible devdax driver 18 * 19 * Unlike drivers/dax/device.c which pre-initializes compound folios based 20 * on device alignment (via vmemmap_shift), this driver leaves folios 21 * uninitialized similar to pmem. This allows fs-dax filesystems like famfs 22 * to work without needing special handling for pre-initialized folios. 23 * 24 * Key differences from device.c: 25 * - pgmap type is MEMORY_DEVICE_FS_DAX (not MEMORY_DEVICE_GENERIC) 26 * - vmemmap_shift is NOT set (folios remain order-0) 27 * - fs-dax can dynamically create compound folios as needed 28 * - No mmap support - all access is through fs-dax/iomap 29 */ 30 31 static void fsdev_write_dax(void *addr, struct page *page, 32 unsigned int off, unsigned int len) 33 { 34 while (len) { 35 void *mem = kmap_local_page(page); 36 unsigned int chunk = min_t(unsigned int, len, PAGE_SIZE - off); 37 38 memcpy_flushcache(addr, mem + off, chunk); 39 kunmap_local(mem); 40 len -= chunk; 41 off = 0; 42 page++; 43 addr += chunk; 44 } 45 } 46 47 static long __fsdev_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, 48 long nr_pages, enum dax_access_mode mode, void **kaddr, 49 unsigned long *pfn) 50 { 51 struct dev_dax *dev_dax = dax_get_private(dax_dev); 52 size_t size = nr_pages << PAGE_SHIFT; 53 size_t offset = pgoff << PAGE_SHIFT; 54 phys_addr_t phys; 55 56 phys = dax_pgoff_to_phys(dev_dax, pgoff, size); 57 if (phys == -1) { 58 dev_dbg(&dev_dax->dev, 59 "pgoff (%#lx) out of range\n", pgoff); 60 return -EFAULT; 61 } 62 63 if (kaddr) 64 *kaddr = __va(phys); 65 66 if (pfn) 67 *pfn = PHYS_PFN(phys); 68 69 /* 70 * Use cached_size which was computed at probe time. The size cannot 71 * change while the driver is bound (resize returns -EBUSY). 72 */ 73 return PHYS_PFN(min(size, dev_dax->cached_size - offset)); 74 } 75 76 static int fsdev_dax_zero_page_range(struct dax_device *dax_dev, 77 pgoff_t pgoff, size_t nr_pages) 78 { 79 void *kaddr; 80 long rc; 81 82 WARN_ONCE(nr_pages > 1, "%s: nr_pages > 1\n", __func__); 83 rc = __fsdev_dax_direct_access(dax_dev, pgoff, 1, DAX_ACCESS, 84 &kaddr, NULL); 85 if (rc < 0) 86 return rc; 87 fsdev_write_dax(kaddr, ZERO_PAGE(0), 0, PAGE_SIZE); 88 return 0; 89 } 90 91 static long fsdev_dax_direct_access(struct dax_device *dax_dev, 92 pgoff_t pgoff, long nr_pages, enum dax_access_mode mode, 93 void **kaddr, unsigned long *pfn) 94 { 95 return __fsdev_dax_direct_access(dax_dev, pgoff, nr_pages, mode, 96 kaddr, pfn); 97 } 98 99 static size_t fsdev_dax_recovery_write(struct dax_device *dax_dev, 100 pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i) 101 { 102 return _copy_from_iter_flushcache(addr, bytes, i); 103 } 104 105 static const struct dax_operations dev_dax_ops = { 106 .direct_access = fsdev_dax_direct_access, 107 .zero_page_range = fsdev_dax_zero_page_range, 108 .recovery_write = fsdev_dax_recovery_write, 109 }; 110 111 static void fsdev_cdev_del(void *cdev) 112 { 113 cdev_del(cdev); 114 } 115 116 static void fsdev_kill(void *dev_dax) 117 { 118 kill_dev_dax(dev_dax); 119 } 120 121 static void fsdev_clear_ops(void *data) 122 { 123 struct dev_dax *dev_dax = data; 124 125 dax_set_ops(dev_dax->dax_dev, NULL); 126 } 127 128 static void fsdev_clear_pgmap_ops(void *data) 129 { 130 struct dev_pagemap *pgmap = data; 131 132 /* 133 * fsdev installs pgmap->ops and ->owner at probe. For a static device 134 * the pgmap is shared and long-lived (owned by the dax bus), so 135 * leaving fsdev's ops behind on unbind would let a later 136 * memory_failure -- after rebind to another driver, or after this 137 * module is unloaded -- dispatch through a stale or freed 138 * ->memory_failure handler. Clear them so the pgmap carries no fsdev 139 * state once we are unbound. 140 */ 141 pgmap->ops = NULL; 142 pgmap->owner = NULL; 143 } 144 145 /* 146 * Page map operations for FS-DAX mode 147 * Similar to fsdax_pagemap_ops in drivers/nvdimm/pmem.c 148 * 149 * Note: folio_free callback is not needed for MEMORY_DEVICE_FS_DAX. 150 * The core mm code in free_zone_device_folio() handles the wake_up_var() 151 * directly for this memory type. 152 */ 153 static u64 fsdev_pfn_to_offset(struct dev_dax *dev_dax, unsigned long pfn) 154 { 155 phys_addr_t phys = PFN_PHYS(pfn); 156 u64 offset = 0; 157 158 for (int i = 0; i < dev_dax->nr_range; i++) { 159 struct range *range = &dev_dax->ranges[i].range; 160 161 if (phys >= range->start && phys <= range->end) 162 return offset + (phys - range->start); 163 offset += range_len(range); 164 } 165 return -1ULL; 166 } 167 168 static int fsdev_pagemap_memory_failure(struct dev_pagemap *pgmap, 169 unsigned long pfn, unsigned long nr_pages, int mf_flags) 170 { 171 struct dev_dax *dev_dax = pgmap->owner; 172 u64 offset = fsdev_pfn_to_offset(dev_dax, pfn); 173 u64 len = nr_pages << PAGE_SHIFT; 174 175 return dax_holder_notify_failure(dev_dax->dax_dev, offset, 176 len, mf_flags); 177 } 178 179 static const struct dev_pagemap_ops fsdev_pagemap_ops = { 180 .memory_failure = fsdev_pagemap_memory_failure, 181 }; 182 183 /* 184 * Clear any stale folio state from pages in the given range. 185 * This is necessary because device_dax pre-initializes compound folios 186 * based on vmemmap_shift, and that state may persist after driver unbind. 187 * Since fsdev_dax uses MEMORY_DEVICE_FS_DAX without vmemmap_shift, fs-dax 188 * expects to find clean order-0 folios that it can build into compound 189 * folios on demand. 190 * 191 * At probe time, no filesystem should be mounted yet, so all mappings 192 * are stale and must be cleared along with compound state. 193 */ 194 static void fsdev_clear_folio_state(struct dev_dax *dev_dax) 195 { 196 for (int i = 0; i < dev_dax->nr_range; i++) { 197 struct range *range = &dev_dax->ranges[i].range; 198 unsigned long pfn = PHYS_PFN(range->start); 199 unsigned long end_pfn = PHYS_PFN(range->end) + 1; 200 201 while (pfn < end_pfn) { 202 struct folio *folio = pfn_folio(pfn); 203 int order = dax_folio_reset_order(folio); 204 205 pfn += 1UL << order; 206 } 207 } 208 } 209 210 static void fsdev_clear_folio_state_action(void *data) 211 { 212 fsdev_clear_folio_state(data); 213 } 214 215 static int fsdev_open(struct inode *inode, struct file *filp) 216 { 217 struct dax_device *dax_dev = inode_dax(inode); 218 struct dev_dax *dev_dax = dax_get_private(dax_dev); 219 220 filp->private_data = dev_dax; 221 222 return 0; 223 } 224 225 static int fsdev_release(struct inode *inode, struct file *filp) 226 { 227 return 0; 228 } 229 230 static const struct file_operations fsdev_fops = { 231 .llseek = noop_llseek, 232 .owner = THIS_MODULE, 233 .open = fsdev_open, 234 .release = fsdev_release, 235 }; 236 237 /* 238 * Acquire the dev_pagemap for probe: the static (pre-populated) one if 239 * present, or a devm-allocated one for the dynamic case. Note that 240 * dev_dax->pgmap is not set here; fsdev_dax_probe() sets it only once 241 * probe succeeds, so a failed probe never leaves a dangling pointer 242 * to a devres-freed pgmap. 243 */ 244 static struct dev_pagemap *fsdev_acquire_pgmap(struct dev_dax *dev_dax) 245 { 246 struct device *dev = &dev_dax->dev; 247 struct dev_pagemap *pgmap; 248 size_t pgmap_size; 249 250 if (static_dev_dax(dev_dax)) { 251 if (dev_dax->nr_range > 1) { 252 dev_warn(dev, 253 "static pgmap / multi-range device conflict\n"); 254 return ERR_PTR(-EINVAL); 255 } 256 257 pgmap = dev_dax->pgmap; 258 pgmap->vmemmap_shift = 0; 259 return pgmap; 260 } 261 262 if (dev_dax->pgmap) { 263 dev_warn(dev, "dynamic-dax with pre-populated page map\n"); 264 return ERR_PTR(-EINVAL); 265 } 266 267 pgmap_size = struct_size(pgmap, ranges, dev_dax->nr_range - 1); 268 pgmap = devm_kzalloc(dev, pgmap_size, GFP_KERNEL); 269 if (!pgmap) 270 return ERR_PTR(-ENOMEM); 271 272 pgmap->nr_range = dev_dax->nr_range; 273 for (int i = 0; i < dev_dax->nr_range; i++) 274 pgmap->ranges[i] = dev_dax->ranges[i].range; 275 276 return pgmap; 277 } 278 279 static int fsdev_dax_probe(struct dev_dax *dev_dax) 280 { 281 struct dax_device *dax_dev = dev_dax->dax_dev; 282 struct device *dev = &dev_dax->dev; 283 struct dev_pagemap *pgmap; 284 struct inode *inode; 285 u64 data_offset = 0; 286 struct cdev *cdev; 287 void *addr; 288 int rc, i; 289 290 pgmap = fsdev_acquire_pgmap(dev_dax); 291 if (IS_ERR(pgmap)) 292 return PTR_ERR(pgmap); 293 294 for (i = 0; i < dev_dax->nr_range; i++) { 295 struct range *range = &dev_dax->ranges[i].range; 296 297 if (!devm_request_mem_region(dev, range->start, 298 range_len(range), dev_name(dev))) { 299 dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve range\n", 300 i, range->start, range->end); 301 return -EBUSY; 302 } 303 } 304 305 /* Cache size now; it cannot change while driver is bound */ 306 dev_dax->cached_size = 0; 307 for (i = 0; i < dev_dax->nr_range; i++) 308 dev_dax->cached_size += range_len(&dev_dax->ranges[i].range); 309 310 /* 311 * Use MEMORY_DEVICE_FS_DAX without setting vmemmap_shift, leaving 312 * folios at order-0. Unlike device.c (MEMORY_DEVICE_GENERIC), this 313 * lets fs-dax dynamically build compound folios as needed, similar 314 * to pmem behavior. 315 */ 316 pgmap->type = MEMORY_DEVICE_FS_DAX; 317 pgmap->ops = &fsdev_pagemap_ops; 318 pgmap->owner = dev_dax; 319 320 addr = devm_memremap_pages(dev, pgmap); 321 if (IS_ERR(addr)) 322 return PTR_ERR(addr); 323 324 /* Drop fsdev's pgmap->ops/owner on unbind so no stale ops survive. */ 325 rc = devm_add_action_or_reset(dev, fsdev_clear_pgmap_ops, pgmap); 326 if (rc) 327 return rc; 328 329 /* 330 * Clear any stale compound folio state left over from a previous 331 * driver (e.g., device_dax with vmemmap_shift). Also register this 332 * as a devm action so folio state is cleared on unbind, ensuring 333 * clean pages for subsequent drivers (e.g., kmem for system-ram). 334 */ 335 fsdev_clear_folio_state(dev_dax); 336 rc = devm_add_action_or_reset(dev, fsdev_clear_folio_state_action, 337 dev_dax); 338 if (rc) 339 return rc; 340 341 /* Detect whether the data is at a non-zero offset into the memory */ 342 if (pgmap->range.start != dev_dax->ranges[0].range.start) { 343 u64 phys = dev_dax->ranges[0].range.start; 344 u64 pgmap_phys = pgmap[0].range.start; 345 346 if (pgmap_phys > phys) { 347 dev_err(dev, "pgmap start %#llx exceeds data start %#llx\n", 348 pgmap_phys, phys); 349 return -EINVAL; 350 } 351 data_offset = phys - pgmap_phys; 352 353 pr_debug("%s: offset detected phys=%llx pgmap_phys=%llx offset=%llx\n", 354 __func__, phys, pgmap_phys, data_offset); 355 } 356 357 inode = dax_inode(dax_dev); 358 cdev = inode->i_cdev; 359 cdev_init(cdev, &fsdev_fops); 360 cdev->owner = dev->driver->owner; 361 cdev_set_parent(cdev, &dev->kobj); 362 rc = cdev_add(cdev, dev->devt, 1); 363 if (rc) 364 return rc; 365 366 rc = devm_add_action_or_reset(dev, fsdev_cdev_del, cdev); 367 if (rc) 368 return rc; 369 370 /* Set the dax operations for fs-dax access path */ 371 rc = dax_set_ops(dax_dev, &dev_dax_ops); 372 if (rc) 373 return rc; 374 375 rc = devm_add_action_or_reset(dev, fsdev_clear_ops, dev_dax); 376 if (rc) 377 return rc; 378 379 run_dax(dax_dev); 380 rc = devm_add_action_or_reset(dev, fsdev_kill, dev_dax); 381 if (rc) 382 return rc; 383 384 /* Probe can no longer fail; expose the pgmap via dev_dax */ 385 dev_dax->pgmap = pgmap; 386 return 0; 387 } 388 389 static struct dax_device_driver fsdev_dax_driver = { 390 .probe = fsdev_dax_probe, 391 .type = DAXDRV_FSDEV_TYPE, 392 }; 393 394 static int __init dax_init(void) 395 { 396 return dax_driver_register(&fsdev_dax_driver); 397 } 398 399 static void __exit dax_exit(void) 400 { 401 dax_driver_unregister(&fsdev_dax_driver); 402 } 403 404 MODULE_AUTHOR("John Groves"); 405 MODULE_DESCRIPTION("FS-DAX Device: fs-dax compatible devdax driver"); 406 MODULE_LICENSE("GPL"); 407 module_init(dax_init); 408 module_exit(dax_exit); 409 MODULE_ALIAS_DAX_DEVICE(0); 410