1 /* SPDX-License-Identifier: MIT */ 2 #ifndef _DRM_PAGEMAP_H_ 3 #define _DRM_PAGEMAP_H_ 4 5 #include <linux/bits.h> 6 #include <linux/dma-direction.h> 7 #include <linux/hmm.h> 8 #include <linux/memremap.h> 9 #include <linux/types.h> 10 11 #define NR_PAGES(order) (1U << (order)) 12 13 struct dma_fence; 14 struct drm_pagemap; 15 struct drm_pagemap_cache; 16 struct drm_pagemap_dev_hold; 17 struct drm_pagemap_zdd; 18 struct device; 19 20 /** 21 * enum drm_interconnect_protocol - Used to identify an interconnect protocol. 22 * 23 * @DRM_INTERCONNECT_SYSTEM: DMA map is system pages 24 * @DRM_INTERCONNECT_DRIVER: DMA map is driver defined 25 */ 26 enum drm_interconnect_protocol { 27 DRM_INTERCONNECT_SYSTEM, 28 DRM_INTERCONNECT_DRIVER, 29 /* A driver can add private values beyond DRM_INTERCONNECT_DRIVER */ 30 }; 31 32 /** 33 * struct drm_pagemap_addr - Address representation. 34 * @addr: The dma address or driver-defined address for driver private interconnects. 35 * @proto: The interconnect protocol. 36 * @order: The page order of the device mapping. (Size is PAGE_SIZE << order). 37 * @dir: The DMA direction. 38 * 39 * Note: There is room for improvement here. We should be able to pack into 40 * 64 bits. 41 */ 42 struct drm_pagemap_addr { 43 dma_addr_t addr; 44 u64 proto : 54; 45 u64 order : 8; 46 u64 dir : 2; 47 }; 48 49 /** 50 * drm_pagemap_addr_encode() - Encode a dma address with metadata 51 * @addr: The dma address or driver-defined address for driver private interconnects. 52 * @proto: The interconnect protocol. 53 * @order: The page order of the dma mapping. (Size is PAGE_SIZE << order). 54 * @dir: The DMA direction. 55 * 56 * Return: A struct drm_pagemap_addr encoding the above information. 57 */ 58 static inline struct drm_pagemap_addr 59 drm_pagemap_addr_encode(dma_addr_t addr, 60 enum drm_interconnect_protocol proto, 61 unsigned int order, 62 enum dma_data_direction dir) 63 { 64 return (struct drm_pagemap_addr) { 65 .addr = addr, 66 .proto = proto, 67 .order = order, 68 .dir = dir, 69 }; 70 } 71 72 /** 73 * struct drm_pagemap_ops: Ops for a drm-pagemap. 74 */ 75 struct drm_pagemap_ops { 76 /** 77 * @device_map: Map for device access or provide a virtual address suitable for 78 * 79 * @dpagemap: The struct drm_pagemap for the page. 80 * @dev: The device mapper. 81 * @page: The page to map. 82 * @order: The page order of the device mapping. (Size is PAGE_SIZE << order). 83 * @dir: The transfer direction. 84 */ 85 struct drm_pagemap_addr (*device_map)(struct drm_pagemap *dpagemap, 86 struct device *dev, 87 struct page *page, 88 unsigned int order, 89 enum dma_data_direction dir); 90 91 /** 92 * @device_unmap: Unmap a device address previously obtained using @device_map. 93 * 94 * @dpagemap: The struct drm_pagemap for the mapping. 95 * @dev: The device unmapper. 96 * @addr: The device address obtained when mapping. 97 */ 98 void (*device_unmap)(struct drm_pagemap *dpagemap, 99 struct device *dev, 100 const struct drm_pagemap_addr *addr); 101 102 /** 103 * @populate_mm: Populate part of the mm with @dpagemap memory, 104 * migrating existing data. 105 * @dpagemap: The struct drm_pagemap managing the memory. 106 * @start: The virtual start address in @mm 107 * @end: The virtual end address in @mm 108 * @mm: Pointer to a live mm. The caller must have an mmget() 109 * reference. 110 * 111 * The caller will have the mm lock at least in read mode. 112 * Note that there is no guarantee that the memory is resident 113 * after the function returns, it's best effort only. 114 * When the mm is not using the memory anymore, 115 * it will be released. The struct drm_pagemap might have a 116 * mechanism in place to reclaim the memory and the data will 117 * then be migrated. Typically to system memory. 118 * The implementation should hold sufficient runtime power- 119 * references while pages are used in an address space and 120 * should ideally guard against hardware device unbind in 121 * a way such that device pages are migrated back to system 122 * followed by device page removal. The implementation should 123 * return -ENODEV after device removal. 124 * 125 * Return: 0 if successful. Negative error code on error. 126 */ 127 int (*populate_mm)(struct drm_pagemap *dpagemap, 128 unsigned long start, unsigned long end, 129 struct mm_struct *mm, 130 unsigned long timeslice_ms); 131 /** 132 * @destroy: Destroy the drm_pagemap and associated resources. 133 * @dpagemap: The drm_pagemap to destroy. 134 * @is_atomic_or_reclaim: The function may be called from 135 * atomic- or reclaim context. 136 * 137 * The implementation should take care not to attempt to 138 * destroy resources that may already have been destroyed 139 * using devm_ callbacks, since this function may be called 140 * after the underlying struct device has been unbound. 141 * If the implementation defers the execution to a work item 142 * to avoid locking issues, then it must make sure the work 143 * items are flushed before module exit. If the destroy call 144 * happens after the provider's pci_remove() callback has 145 * been executed, a module reference and drm device reference is 146 * held across the destroy callback. 147 */ 148 void (*destroy)(struct drm_pagemap *dpagemap, 149 bool is_atomic_or_reclaim); 150 }; 151 152 /** 153 * struct drm_pagemap: Additional information for a struct dev_pagemap 154 * used for device p2p handshaking. 155 * @ops: The struct drm_pagemap_ops. 156 * @ref: Reference count. 157 * @drm: The struct drm device owning the device-private memory. 158 * @pagemap: Pointer to the underlying dev_pagemap. 159 * @dev_hold: Pointer to a struct drm_pagemap_dev_hold for 160 * device referencing. 161 * @cache: Back-pointer to the &struct drm_pagemap_cache used for this 162 * &struct drm_pagemap. May be NULL if no cache is used. 163 * @shrink_link: Link into the shrinker's list of drm_pagemaps. Only 164 * used if also using a pagemap cache. 165 */ 166 struct drm_pagemap { 167 const struct drm_pagemap_ops *ops; 168 struct kref ref; 169 struct drm_device *drm; 170 struct dev_pagemap *pagemap; 171 struct drm_pagemap_dev_hold *dev_hold; 172 struct drm_pagemap_cache *cache; 173 struct list_head shrink_link; 174 }; 175 176 struct drm_pagemap_devmem; 177 178 /** 179 * struct drm_pagemap_devmem_ops - Operations structure for GPU SVM device memory 180 * 181 * This structure defines the operations for GPU Shared Virtual Memory (SVM) 182 * device memory. These operations are provided by the GPU driver to manage device memory 183 * allocations and perform operations such as migration between device memory and system 184 * RAM. 185 */ 186 struct drm_pagemap_devmem_ops { 187 /** 188 * @devmem_release: Release device memory allocation (optional) 189 * @devmem_allocation: device memory allocation 190 * 191 * Release device memory allocation and drop a reference to device 192 * memory allocation. 193 */ 194 void (*devmem_release)(struct drm_pagemap_devmem *devmem_allocation); 195 196 /** 197 * @populate_devmem_pfn: Populate device memory PFN (required for migration) 198 * @devmem_allocation: device memory allocation 199 * @npages: Number of pages to populate 200 * @pfn: Array of page frame numbers to populate 201 * 202 * Populate device memory page frame numbers (PFN). 203 * 204 * Return: 0 on success, a negative error code on failure. 205 */ 206 int (*populate_devmem_pfn)(struct drm_pagemap_devmem *devmem_allocation, 207 unsigned long npages, unsigned long *pfn); 208 209 /** 210 * @copy_to_devmem: Copy to device memory (required for migration) 211 * @pages: Pointer to array of device memory pages (destination) 212 * @pagemap_addr: Pointer to array of DMA information (source) 213 * @npages: Number of pages to copy 214 * @pre_migrate_fence: dma-fence to wait for before migration start. 215 * May be NULL. 216 * 217 * Copy pages to device memory. If the order of a @pagemap_addr entry 218 * is greater than 0, the entry is populated but subsequent entries 219 * within the range of that order are not populated. 220 * 221 * Return: 0 on success, a negative error code on failure. 222 */ 223 int (*copy_to_devmem)(struct page **pages, 224 struct drm_pagemap_addr *pagemap_addr, 225 unsigned long npages, 226 struct dma_fence *pre_migrate_fence); 227 228 /** 229 * @copy_to_ram: Copy to system RAM (required for migration) 230 * @pages: Pointer to array of device memory pages (source) 231 * @pagemap_addr: Pointer to array of DMA information (destination) 232 * @npages: Number of pages to copy 233 * @pre_migrate_fence: dma-fence to wait for before migration start. 234 * May be NULL. 235 * 236 * Copy pages to system RAM. If the order of a @pagemap_addr entry 237 * is greater than 0, the entry is populated but subsequent entries 238 * within the range of that order are not populated. 239 * 240 * Return: 0 on success, a negative error code on failure. 241 */ 242 int (*copy_to_ram)(struct page **pages, 243 struct drm_pagemap_addr *pagemap_addr, 244 unsigned long npages, 245 struct dma_fence *pre_migrate_fence); 246 }; 247 248 #if IS_ENABLED(CONFIG_ZONE_DEVICE) 249 250 int drm_pagemap_init(struct drm_pagemap *dpagemap, 251 struct dev_pagemap *pagemap, 252 struct drm_device *drm, 253 const struct drm_pagemap_ops *ops); 254 255 struct drm_pagemap *drm_pagemap_create(struct drm_device *drm, 256 struct dev_pagemap *pagemap, 257 const struct drm_pagemap_ops *ops); 258 259 struct drm_pagemap *drm_pagemap_page_to_dpagemap(struct page *page); 260 261 void drm_pagemap_put(struct drm_pagemap *dpagemap); 262 263 #else 264 265 static inline struct drm_pagemap *drm_pagemap_page_to_dpagemap(struct page *page) 266 { 267 return NULL; 268 } 269 270 static inline void drm_pagemap_put(struct drm_pagemap *dpagemap) 271 { 272 } 273 274 #endif /* IS_ENABLED(CONFIG_ZONE_DEVICE) */ 275 276 /** 277 * drm_pagemap_get() - Obtain a reference on a struct drm_pagemap 278 * @dpagemap: Pointer to the struct drm_pagemap, or NULL. 279 * 280 * Return: Pointer to the struct drm_pagemap, or NULL. 281 */ 282 static inline struct drm_pagemap * 283 drm_pagemap_get(struct drm_pagemap *dpagemap) 284 { 285 if (likely(dpagemap)) 286 kref_get(&dpagemap->ref); 287 288 return dpagemap; 289 } 290 291 /** 292 * drm_pagemap_get_unless_zero() - Obtain a reference on a struct drm_pagemap 293 * unless the current reference count is zero. 294 * @dpagemap: Pointer to the drm_pagemap or NULL. 295 * 296 * Return: A pointer to @dpagemap if the reference count was successfully 297 * incremented. NULL if @dpagemap was NULL, or its refcount was 0. 298 */ 299 static inline struct drm_pagemap * __must_check 300 drm_pagemap_get_unless_zero(struct drm_pagemap *dpagemap) 301 { 302 return (dpagemap && kref_get_unless_zero(&dpagemap->ref)) ? dpagemap : NULL; 303 } 304 305 /** 306 * struct drm_pagemap_devmem - Structure representing a GPU SVM device memory allocation 307 * 308 * @dev: Pointer to the device structure which device memory allocation belongs to 309 * @mm: Pointer to the mm_struct for the address space 310 * @detached: device memory allocations is detached from device pages 311 * @ops: Pointer to the operations structure for GPU SVM device memory 312 * @dpagemap: The struct drm_pagemap of the pages this allocation belongs to. 313 * @size: Size of device memory allocation 314 * @timeslice_expiration: Timeslice expiration in jiffies 315 * @pre_migrate_fence: Fence to wait for or pipeline behind before migration starts. 316 * (May be NULL). 317 */ 318 struct drm_pagemap_devmem { 319 struct device *dev; 320 struct mm_struct *mm; 321 struct completion detached; 322 const struct drm_pagemap_devmem_ops *ops; 323 struct drm_pagemap *dpagemap; 324 size_t size; 325 u64 timeslice_expiration; 326 struct dma_fence *pre_migrate_fence; 327 }; 328 329 /** 330 * struct drm_pagemap_migrate_details - Details to govern migration. 331 * @timeslice_ms: The time requested for the migrated pagemap pages to 332 * be present in @mm before being allowed to be migrated back. 333 * @can_migrate_same_pagemap: Whether the copy function can migrate 334 * device pages within a single drm_pagemap. 335 */ 336 struct drm_pagemap_migrate_details { 337 unsigned long timeslice_ms; 338 u32 can_migrate_same_pagemap : 1; 339 }; 340 341 #if IS_ENABLED(CONFIG_ZONE_DEVICE) 342 343 #define DRM_PAGEMAP_ZDD_FLAG_MIGRATED BIT(0) 344 #define DRM_PAGEMAP_ZDD_FLAG_MASK DRM_PAGEMAP_ZDD_FLAG_MIGRATED 345 346 int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation, 347 struct mm_struct *mm, 348 unsigned long start, unsigned long end, 349 const struct drm_pagemap_migrate_details *mdetails); 350 351 int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation); 352 353 const struct dev_pagemap_ops *drm_pagemap_pagemap_ops_get(void); 354 355 void drm_pagemap_devmem_init(struct drm_pagemap_devmem *devmem_allocation, 356 struct device *dev, struct mm_struct *mm, 357 const struct drm_pagemap_devmem_ops *ops, 358 struct drm_pagemap *dpagemap, size_t size, 359 struct dma_fence *pre_migrate_fence); 360 361 int drm_pagemap_populate_mm(struct drm_pagemap *dpagemap, 362 unsigned long start, unsigned long end, 363 struct mm_struct *mm, 364 unsigned long timeslice_ms); 365 366 void drm_pagemap_destroy(struct drm_pagemap *dpagemap, bool is_atomic_or_reclaim); 367 368 int drm_pagemap_reinit(struct drm_pagemap *dpagemap); 369 370 /** 371 * drm_pagemap_page_zone_device_data() - Page to zone_device_data 372 * @page: Pointer to the page 373 * 374 * Return: Page's zone_device_data 375 */ 376 static inline struct drm_pagemap_zdd *drm_pagemap_page_zone_device_data(struct page *page) 377 { 378 struct folio *folio = page_folio(page); 379 380 return (struct drm_pagemap_zdd *) 381 ((unsigned long)folio_zone_device_data(folio) & 382 ~DRM_PAGEMAP_ZDD_FLAG_MASK); 383 } 384 385 #else 386 387 static inline struct drm_pagemap_zdd *drm_pagemap_page_zone_device_data(struct page *page) 388 { 389 return NULL; 390 } 391 392 #endif /* IS_ENABLED(CONFIG_ZONE_DEVICE) */ 393 394 #endif 395