1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_DAX_H 3 #define _LINUX_DAX_H 4 5 #include <linux/fs.h> 6 #include <linux/mm.h> 7 #include <linux/radix-tree.h> 8 9 typedef unsigned long dax_entry_t; 10 11 struct dax_device; 12 struct gendisk; 13 struct iomap_ops; 14 struct iomap_iter; 15 struct iomap; 16 17 enum dax_access_mode { 18 DAX_ACCESS, 19 DAX_RECOVERY_WRITE, 20 }; 21 22 struct dax_operations { 23 /* 24 * direct_access: translate a device-relative 25 * logical-page-offset into an absolute physical pfn. Return the 26 * number of pages available for DAX at that pfn. 27 */ 28 long (*direct_access)(struct dax_device *, pgoff_t, long, 29 enum dax_access_mode, void **, unsigned long *); 30 /* zero_page_range: required operation. Zero page range */ 31 int (*zero_page_range)(struct dax_device *, pgoff_t, size_t); 32 /* 33 * recovery_write: recover a poisoned range by DAX device driver 34 * capable of clearing poison. 35 */ 36 size_t (*recovery_write)(struct dax_device *dax_dev, pgoff_t pgoff, 37 void *addr, size_t bytes, struct iov_iter *iter); 38 }; 39 40 struct dax_holder_operations { 41 /* 42 * notify_failure - notify memory failure into inner holder device 43 * @dax_dev: the dax device which contains the holder 44 * @offset: offset on this dax device where memory failure occurs 45 * @len: length of this memory failure event 46 * @flags: action flags for memory failure handler 47 */ 48 int (*notify_failure)(struct dax_device *dax_dev, u64 offset, 49 u64 len, int mf_flags); 50 }; 51 52 #if IS_ENABLED(CONFIG_DAX) 53 struct dax_device *alloc_dax(void *private, const struct dax_operations *ops); 54 void *dax_holder(struct dax_device *dax_dev); 55 void put_dax(struct dax_device *dax_dev); 56 void kill_dax(struct dax_device *dax_dev); 57 void dax_write_cache(struct dax_device *dax_dev, bool wc); 58 bool dax_write_cache_enabled(struct dax_device *dax_dev); 59 bool dax_synchronous(struct dax_device *dax_dev); 60 void set_dax_nocache(struct dax_device *dax_dev); 61 void set_dax_nomc(struct dax_device *dax_dev); 62 void set_dax_synchronous(struct dax_device *dax_dev); 63 size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff, 64 void *addr, size_t bytes, struct iov_iter *i); 65 /* 66 * Check if given mapping is supported by the file / underlying device. 67 */ 68 static inline bool daxdev_mapping_supported(vm_flags_t vm_flags, 69 const struct inode *inode, 70 struct dax_device *dax_dev) 71 { 72 if (!(vm_flags & VM_SYNC)) 73 return true; 74 if (!IS_DAX(inode)) 75 return false; 76 return dax_synchronous(dax_dev); 77 } 78 #else 79 static inline void *dax_holder(struct dax_device *dax_dev) 80 { 81 return NULL; 82 } 83 static inline struct dax_device *alloc_dax(void *private, 84 const struct dax_operations *ops) 85 { 86 return ERR_PTR(-EOPNOTSUPP); 87 } 88 static inline void put_dax(struct dax_device *dax_dev) 89 { 90 } 91 static inline void kill_dax(struct dax_device *dax_dev) 92 { 93 } 94 static inline void dax_write_cache(struct dax_device *dax_dev, bool wc) 95 { 96 } 97 static inline bool dax_write_cache_enabled(struct dax_device *dax_dev) 98 { 99 return false; 100 } 101 static inline bool dax_synchronous(struct dax_device *dax_dev) 102 { 103 return true; 104 } 105 static inline void set_dax_nocache(struct dax_device *dax_dev) 106 { 107 } 108 static inline void set_dax_nomc(struct dax_device *dax_dev) 109 { 110 } 111 static inline void set_dax_synchronous(struct dax_device *dax_dev) 112 { 113 } 114 static inline bool daxdev_mapping_supported(vm_flags_t vm_flags, 115 const struct inode *inode, 116 struct dax_device *dax_dev) 117 { 118 return !(vm_flags & VM_SYNC); 119 } 120 static inline size_t dax_recovery_write(struct dax_device *dax_dev, 121 pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i) 122 { 123 return 0; 124 } 125 #endif 126 127 struct writeback_control; 128 #if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX) 129 int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk); 130 void dax_remove_host(struct gendisk *disk); 131 struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off, 132 void *holder, const struct dax_holder_operations *ops); 133 void fs_put_dax(struct dax_device *dax_dev, void *holder); 134 #else 135 static inline int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk) 136 { 137 return 0; 138 } 139 static inline void dax_remove_host(struct gendisk *disk) 140 { 141 } 142 static inline struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, 143 u64 *start_off, void *holder, 144 const struct dax_holder_operations *ops) 145 { 146 return NULL; 147 } 148 static inline void fs_put_dax(struct dax_device *dax_dev, void *holder) 149 { 150 } 151 #endif /* CONFIG_BLOCK && CONFIG_FS_DAX */ 152 153 #if IS_ENABLED(CONFIG_FS_DAX) 154 int dax_writeback_mapping_range(struct address_space *mapping, 155 struct dax_device *dax_dev, struct writeback_control *wbc); 156 157 struct page *dax_layout_busy_page(struct address_space *mapping); 158 struct page *dax_layout_busy_page_range(struct address_space *mapping, loff_t start, loff_t end); 159 dax_entry_t dax_lock_folio(struct folio *folio); 160 void dax_unlock_folio(struct folio *folio, dax_entry_t cookie); 161 dax_entry_t dax_lock_mapping_entry(struct address_space *mapping, 162 unsigned long index, struct page **page); 163 void dax_unlock_mapping_entry(struct address_space *mapping, 164 unsigned long index, dax_entry_t cookie); 165 #else 166 static inline struct page *dax_layout_busy_page(struct address_space *mapping) 167 { 168 return NULL; 169 } 170 171 static inline struct page *dax_layout_busy_page_range(struct address_space *mapping, pgoff_t start, pgoff_t nr_pages) 172 { 173 return NULL; 174 } 175 176 static inline int dax_writeback_mapping_range(struct address_space *mapping, 177 struct dax_device *dax_dev, struct writeback_control *wbc) 178 { 179 return -EOPNOTSUPP; 180 } 181 182 static inline dax_entry_t dax_lock_folio(struct folio *folio) 183 { 184 if (IS_DAX(folio->mapping->host)) 185 return ~0UL; 186 return 0; 187 } 188 189 static inline void dax_unlock_folio(struct folio *folio, dax_entry_t cookie) 190 { 191 } 192 193 static inline dax_entry_t dax_lock_mapping_entry(struct address_space *mapping, 194 unsigned long index, struct page **page) 195 { 196 return 0; 197 } 198 199 static inline void dax_unlock_mapping_entry(struct address_space *mapping, 200 unsigned long index, dax_entry_t cookie) 201 { 202 } 203 #endif 204 205 int dax_file_unshare(struct inode *inode, loff_t pos, loff_t len, 206 const struct iomap_ops *ops); 207 int dax_zero_range(struct inode *inode, loff_t pos, loff_t len, bool *did_zero, 208 const struct iomap_ops *ops); 209 int dax_truncate_page(struct inode *inode, loff_t pos, bool *did_zero, 210 const struct iomap_ops *ops); 211 212 static inline bool dax_page_is_idle(struct page *page) 213 { 214 return page && page_ref_count(page) == 0; 215 } 216 217 #if IS_ENABLED(CONFIG_DAX) 218 int dax_read_lock(void); 219 void dax_read_unlock(int id); 220 #else 221 static inline int dax_read_lock(void) 222 { 223 return 0; 224 } 225 226 static inline void dax_read_unlock(int id) 227 { 228 } 229 #endif /* CONFIG_DAX */ 230 231 #if !IS_ENABLED(CONFIG_FS_DAX) 232 static inline int __must_check dax_break_layout(struct inode *inode, 233 loff_t start, loff_t end, void (cb)(struct inode *)) 234 { 235 return 0; 236 } 237 238 static inline void dax_break_layout_final(struct inode *inode) 239 { 240 } 241 #endif 242 243 bool dax_alive(struct dax_device *dax_dev); 244 void *dax_get_private(struct dax_device *dax_dev); 245 long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages, 246 enum dax_access_mode mode, void **kaddr, unsigned long *pfn); 247 size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, 248 size_t bytes, struct iov_iter *i); 249 size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, 250 size_t bytes, struct iov_iter *i); 251 int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, 252 size_t nr_pages); 253 int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off, u64 len, 254 int mf_flags); 255 void dax_flush(struct dax_device *dax_dev, void *addr, size_t size); 256 257 ssize_t dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter, 258 const struct iomap_ops *ops); 259 vm_fault_t dax_iomap_fault(struct vm_fault *vmf, unsigned int order, 260 unsigned long *pfnp, int *errp, 261 const struct iomap_ops *ops); 262 vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf, 263 unsigned int order, unsigned long pfn); 264 int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index); 265 void dax_delete_mapping_range(struct address_space *mapping, 266 loff_t start, loff_t end); 267 int dax_invalidate_mapping_entry_sync(struct address_space *mapping, 268 pgoff_t index); 269 int __must_check dax_break_layout(struct inode *inode, loff_t start, 270 loff_t end, void (cb)(struct inode *)); 271 static inline int __must_check dax_break_layout_inode(struct inode *inode, 272 void (cb)(struct inode *)) 273 { 274 return dax_break_layout(inode, 0, LLONG_MAX, cb); 275 } 276 void dax_break_layout_final(struct inode *inode); 277 int dax_dedupe_file_range_compare(struct inode *src, loff_t srcoff, 278 struct inode *dest, loff_t destoff, 279 loff_t len, bool *is_same, 280 const struct iomap_ops *ops); 281 int dax_remap_file_range_prep(struct file *file_in, loff_t pos_in, 282 struct file *file_out, loff_t pos_out, 283 loff_t *len, unsigned int remap_flags, 284 const struct iomap_ops *ops); 285 static inline bool dax_mapping(struct address_space *mapping) 286 { 287 return mapping->host && IS_DAX(mapping->host); 288 } 289 290 /* 291 * Due to dax's memory and block duo personalities, hwpoison reporting 292 * takes into consideration which personality is presently visible. 293 * When dax acts like a block device, such as in block IO, an encounter of 294 * dax hwpoison is reported as -EIO. 295 * When dax acts like memory, such as in page fault, a detection of hwpoison 296 * is reported as -EHWPOISON which leads to VM_FAULT_HWPOISON. 297 */ 298 static inline int dax_mem2blk_err(int err) 299 { 300 return (err == -EHWPOISON) ? -EIO : err; 301 } 302 303 #ifdef CONFIG_DEV_DAX_HMEM_DEVICES 304 void hmem_register_resource(int target_nid, struct resource *r); 305 #else 306 static inline void hmem_register_resource(int target_nid, struct resource *r) 307 { 308 } 309 #endif 310 311 typedef int (*walk_hmem_fn)(struct device *dev, int target_nid, 312 const struct resource *res); 313 int walk_hmem_resources(struct device *dev, walk_hmem_fn fn); 314 #endif 315