1 /* 2 * linux/drivers/video/fb_defio.c 3 * 4 * Copyright (C) 2006 Jaya Kumar 5 * 6 * This file is subject to the terms and conditions of the GNU General Public 7 * License. See the file COPYING in the main directory of this archive 8 * for more details. 9 */ 10 11 #include <linux/module.h> 12 #include <linux/kernel.h> 13 #include <linux/errno.h> 14 #include <linux/export.h> 15 #include <linux/string.h> 16 #include <linux/mm.h> 17 #include <linux/vmalloc.h> 18 #include <linux/delay.h> 19 #include <linux/interrupt.h> 20 #include <linux/fb.h> 21 #include <linux/list.h> 22 23 /* to support deferred IO */ 24 #include <linux/rmap.h> 25 #include <linux/pagemap.h> 26 27 struct address_space; 28 29 /* 30 * struct fb_deferred_io_state 31 */ 32 33 struct fb_deferred_io_state { 34 struct kref ref; 35 36 int open_count; /* number of opened files; protected by fb_info lock */ 37 struct address_space *mapping; /* page cache object for fb device */ 38 39 struct mutex lock; /* mutex that protects the pageref list */ 40 /* fields protected by lock */ 41 struct fb_info *info; 42 struct list_head pagereflist; /* list of pagerefs for touched pages */ 43 unsigned long npagerefs; 44 struct fb_deferred_io_pageref *pagerefs; 45 }; 46 47 static struct fb_deferred_io_state *fb_deferred_io_state_alloc(unsigned long len) 48 { 49 struct fb_deferred_io_state *fbdefio_state; 50 struct fb_deferred_io_pageref *pagerefs; 51 unsigned long npagerefs; 52 53 fbdefio_state = kzalloc_obj(*fbdefio_state); 54 if (!fbdefio_state) 55 return NULL; 56 57 npagerefs = DIV_ROUND_UP(len, PAGE_SIZE); 58 59 /* alloc a page ref for each page of the display memory */ 60 pagerefs = kvzalloc_objs(*pagerefs, npagerefs); 61 if (!pagerefs) 62 goto err_kfree; 63 fbdefio_state->npagerefs = npagerefs; 64 fbdefio_state->pagerefs = pagerefs; 65 66 kref_init(&fbdefio_state->ref); 67 mutex_init(&fbdefio_state->lock); 68 69 INIT_LIST_HEAD(&fbdefio_state->pagereflist); 70 71 return fbdefio_state; 72 73 err_kfree: 74 kfree(fbdefio_state); 75 return NULL; 76 } 77 78 static void fb_deferred_io_state_release(struct fb_deferred_io_state *fbdefio_state) 79 { 80 WARN_ON(!list_empty(&fbdefio_state->pagereflist)); 81 mutex_destroy(&fbdefio_state->lock); 82 kvfree(fbdefio_state->pagerefs); 83 84 kfree(fbdefio_state); 85 } 86 87 static void fb_deferred_io_state_get(struct fb_deferred_io_state *fbdefio_state) 88 { 89 kref_get(&fbdefio_state->ref); 90 } 91 92 static void __fb_deferred_io_state_release(struct kref *ref) 93 { 94 struct fb_deferred_io_state *fbdefio_state = 95 container_of(ref, struct fb_deferred_io_state, ref); 96 97 fb_deferred_io_state_release(fbdefio_state); 98 } 99 100 static void fb_deferred_io_state_put(struct fb_deferred_io_state *fbdefio_state) 101 { 102 kref_put(&fbdefio_state->ref, __fb_deferred_io_state_release); 103 } 104 105 /* 106 * struct vm_operations_struct 107 */ 108 109 static void fb_deferred_io_vm_open(struct vm_area_struct *vma) 110 { 111 struct fb_deferred_io_state *fbdefio_state = vma->vm_private_data; 112 113 WARN_ON_ONCE(!try_module_get(THIS_MODULE)); 114 fb_deferred_io_state_get(fbdefio_state); 115 } 116 117 static void fb_deferred_io_vm_close(struct vm_area_struct *vma) 118 { 119 struct fb_deferred_io_state *fbdefio_state = vma->vm_private_data; 120 121 fb_deferred_io_state_put(fbdefio_state); 122 module_put(THIS_MODULE); 123 } 124 125 static struct page *fb_deferred_io_get_page(struct fb_info *info, unsigned long offs) 126 { 127 struct fb_deferred_io *fbdefio = info->fbdefio; 128 const void *screen_buffer = info->screen_buffer; 129 struct page *page = NULL; 130 131 if (fbdefio->get_page) 132 return fbdefio->get_page(info, offs); 133 134 if (is_vmalloc_addr(screen_buffer + offs)) 135 page = vmalloc_to_page(screen_buffer + offs); 136 else if (info->fix.smem_start) 137 page = pfn_to_page((info->fix.smem_start + offs) >> PAGE_SHIFT); 138 139 if (page) 140 get_page(page); 141 142 return page; 143 } 144 145 static struct fb_deferred_io_pageref * 146 fb_deferred_io_pageref_lookup(struct fb_deferred_io_state *fbdefio_state, unsigned long offset, 147 struct page *page) 148 { 149 struct fb_info *info = fbdefio_state->info; 150 unsigned long pgoff = offset >> PAGE_SHIFT; 151 struct fb_deferred_io_pageref *pageref; 152 153 if (fb_WARN_ON_ONCE(info, pgoff >= fbdefio_state->npagerefs)) 154 return NULL; /* incorrect allocation size */ 155 156 /* 1:1 mapping between pageref and page offset */ 157 pageref = &fbdefio_state->pagerefs[pgoff]; 158 159 if (pageref->page) 160 goto out; 161 162 pageref->page = page; 163 pageref->offset = pgoff << PAGE_SHIFT; 164 INIT_LIST_HEAD(&pageref->list); 165 166 out: 167 if (fb_WARN_ON_ONCE(info, pageref->page != page)) 168 return NULL; /* inconsistent state */ 169 return pageref; 170 } 171 172 static struct fb_deferred_io_pageref *fb_deferred_io_pageref_get(struct fb_info *info, 173 unsigned long offset, 174 struct page *page) 175 { 176 struct fb_deferred_io *fbdefio = info->fbdefio; 177 struct fb_deferred_io_state *fbdefio_state = info->fbdefio_state; 178 struct list_head *pos = &fbdefio_state->pagereflist; 179 struct fb_deferred_io_pageref *pageref, *cur; 180 181 pageref = fb_deferred_io_pageref_lookup(fbdefio_state, offset, page); 182 if (!pageref) 183 return NULL; 184 185 /* 186 * This check is to catch the case where a new process could start 187 * writing to the same page through a new PTE. This new access 188 * can cause a call to .page_mkwrite even if the original process' 189 * PTE is marked writable. 190 */ 191 if (!list_empty(&pageref->list)) 192 goto pageref_already_added; 193 194 if (unlikely(fbdefio->sort_pagereflist)) { 195 /* 196 * We loop through the list of pagerefs before adding in 197 * order to keep the pagerefs sorted. This has significant 198 * overhead of O(n^2) with n being the number of written 199 * pages. If possible, drivers should try to work with 200 * unsorted page lists instead. 201 */ 202 list_for_each_entry(cur, &fbdefio_state->pagereflist, list) { 203 if (cur->offset > pageref->offset) 204 break; 205 } 206 pos = &cur->list; 207 } 208 209 list_add_tail(&pageref->list, pos); 210 211 pageref_already_added: 212 return pageref; 213 } 214 215 static void fb_deferred_io_pageref_put(struct fb_deferred_io_pageref *pageref, 216 struct fb_info *info) 217 { 218 list_del_init(&pageref->list); 219 } 220 221 /* this is to find and return the vmalloc-ed fb pages */ 222 static vm_fault_t fb_deferred_io_fault(struct vm_fault *vmf) 223 { 224 struct fb_info *info; 225 unsigned long offset; 226 struct page *page; 227 vm_fault_t ret; 228 struct fb_deferred_io_state *fbdefio_state = vmf->vma->vm_private_data; 229 230 mutex_lock(&fbdefio_state->lock); 231 232 info = fbdefio_state->info; 233 if (!info) { 234 ret = VM_FAULT_SIGBUS; /* our device is gone */ 235 goto err_mutex_unlock; 236 } 237 238 offset = vmf->pgoff << PAGE_SHIFT; 239 if (offset >= info->fix.smem_len) { 240 ret = VM_FAULT_SIGBUS; 241 goto err_mutex_unlock; 242 } 243 244 page = fb_deferred_io_get_page(info, offset); 245 if (!page) { 246 ret = VM_FAULT_SIGBUS; 247 goto err_mutex_unlock; 248 } 249 250 if (!vmf->vma->vm_file) 251 fb_err(info, "no mapping available\n"); 252 253 fb_WARN_ON_ONCE(info, !fbdefio_state->mapping); 254 255 mutex_unlock(&fbdefio_state->lock); 256 257 vmf->page = page; 258 259 return 0; 260 261 err_mutex_unlock: 262 mutex_unlock(&fbdefio_state->lock); 263 return ret; 264 } 265 266 int fb_deferred_io_fsync(struct file *file, loff_t start, loff_t end, int datasync) 267 { 268 struct fb_info *info = file->private_data; 269 struct inode *inode = file_inode(file); 270 int err = file_write_and_wait_range(file, start, end); 271 if (err) 272 return err; 273 274 /* Skip if deferred io is compiled-in but disabled on this fbdev */ 275 if (!info->fbdefio) 276 return 0; 277 278 inode_lock(inode); 279 flush_delayed_work(&info->deferred_work); 280 inode_unlock(inode); 281 282 return 0; 283 } 284 EXPORT_SYMBOL_GPL(fb_deferred_io_fsync); 285 286 /* 287 * Adds a page to the dirty list. Call this from struct 288 * vm_operations_struct.page_mkwrite. 289 */ 290 static vm_fault_t fb_deferred_io_track_page(struct fb_deferred_io_state *fbdefio_state, 291 unsigned long offset, struct page *page) 292 { 293 struct fb_info *info; 294 struct fb_deferred_io *fbdefio; 295 struct fb_deferred_io_pageref *pageref; 296 vm_fault_t ret; 297 298 /* protect against the workqueue changing the page list */ 299 mutex_lock(&fbdefio_state->lock); 300 301 info = fbdefio_state->info; 302 if (!info) { 303 ret = VM_FAULT_SIGBUS; /* our device is gone */ 304 goto err_mutex_unlock; 305 } 306 307 fbdefio = info->fbdefio; 308 309 pageref = fb_deferred_io_pageref_get(info, offset, page); 310 if (WARN_ON_ONCE(!pageref)) { 311 ret = VM_FAULT_OOM; 312 goto err_mutex_unlock; 313 } 314 315 /* 316 * We want the page to remain locked from ->page_mkwrite until 317 * the PTE is marked dirty to avoid mapping_wrprotect_range() 318 * being called before the PTE is updated, which would leave 319 * the page ignored by defio. 320 * Do this by locking the page here and informing the caller 321 * about it with VM_FAULT_LOCKED. 322 */ 323 lock_page(pageref->page); 324 325 mutex_unlock(&fbdefio_state->lock); 326 327 /* come back after delay to process the deferred IO */ 328 schedule_delayed_work(&info->deferred_work, fbdefio->delay); 329 return VM_FAULT_LOCKED; 330 331 err_mutex_unlock: 332 mutex_unlock(&fbdefio_state->lock); 333 return ret; 334 } 335 336 static vm_fault_t fb_deferred_io_page_mkwrite(struct fb_deferred_io_state *fbdefio_state, 337 struct vm_fault *vmf) 338 { 339 unsigned long offset = vmf->pgoff << PAGE_SHIFT; 340 struct page *page = vmf->page; 341 342 file_update_time(vmf->vma->vm_file); 343 344 return fb_deferred_io_track_page(fbdefio_state, offset, page); 345 } 346 347 static vm_fault_t fb_deferred_io_mkwrite(struct vm_fault *vmf) 348 { 349 struct fb_deferred_io_state *fbdefio_state = vmf->vma->vm_private_data; 350 351 return fb_deferred_io_page_mkwrite(fbdefio_state, vmf); 352 } 353 354 static const struct vm_operations_struct fb_deferred_io_vm_ops = { 355 .open = fb_deferred_io_vm_open, 356 .close = fb_deferred_io_vm_close, 357 .fault = fb_deferred_io_fault, 358 .page_mkwrite = fb_deferred_io_mkwrite, 359 }; 360 361 static const struct address_space_operations fb_deferred_io_aops = { 362 .dirty_folio = noop_dirty_folio, 363 }; 364 365 int fb_deferred_io_mmap(struct fb_info *info, struct vm_area_struct *vma) 366 { 367 vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); 368 369 if (!try_module_get(THIS_MODULE)) 370 return -EINVAL; 371 372 vma->vm_ops = &fb_deferred_io_vm_ops; 373 vm_flags_set(vma, VM_DONTEXPAND | VM_DONTDUMP); 374 if (!(info->flags & FBINFO_VIRTFB)) 375 vm_flags_set(vma, VM_IO); 376 vma->vm_private_data = info->fbdefio_state; 377 378 fb_deferred_io_state_get(info->fbdefio_state); /* released in vma->vm_ops->close() */ 379 380 return 0; 381 } 382 EXPORT_SYMBOL_GPL(fb_deferred_io_mmap); 383 384 /* workqueue callback */ 385 static void fb_deferred_io_work(struct work_struct *work) 386 { 387 struct fb_info *info = container_of(work, struct fb_info, deferred_work.work); 388 struct fb_deferred_io_pageref *pageref, *next; 389 struct fb_deferred_io *fbdefio = info->fbdefio; 390 struct fb_deferred_io_state *fbdefio_state = info->fbdefio_state; 391 392 /* here we wrprotect the page's mappings, then do all deferred IO. */ 393 mutex_lock(&fbdefio_state->lock); 394 #ifdef CONFIG_MMU 395 list_for_each_entry(pageref, &fbdefio_state->pagereflist, list) { 396 struct page *page = pageref->page; 397 pgoff_t pgoff = pageref->offset >> PAGE_SHIFT; 398 399 mapping_wrprotect_range(fbdefio_state->mapping, pgoff, 400 page_to_pfn(page), 1); 401 } 402 #endif 403 404 /* driver's callback with pagereflist */ 405 fbdefio->deferred_io(info, &fbdefio_state->pagereflist); 406 407 /* clear the list */ 408 list_for_each_entry_safe(pageref, next, &fbdefio_state->pagereflist, list) 409 fb_deferred_io_pageref_put(pageref, info); 410 411 mutex_unlock(&fbdefio_state->lock); 412 } 413 414 int fb_deferred_io_init(struct fb_info *info) 415 { 416 struct fb_deferred_io *fbdefio = info->fbdefio; 417 struct fb_deferred_io_state *fbdefio_state; 418 419 BUG_ON(!fbdefio); 420 421 if (WARN_ON(!info->fix.smem_len)) 422 return -EINVAL; 423 424 fbdefio_state = fb_deferred_io_state_alloc(info->fix.smem_len); 425 if (!fbdefio_state) 426 return -ENOMEM; 427 fbdefio_state->info = info; 428 429 INIT_DELAYED_WORK(&info->deferred_work, fb_deferred_io_work); 430 if (fbdefio->delay == 0) /* set a default of 1 s */ 431 fbdefio->delay = HZ; 432 433 info->fbdefio_state = fbdefio_state; 434 435 return 0; 436 } 437 EXPORT_SYMBOL_GPL(fb_deferred_io_init); 438 439 void fb_deferred_io_open(struct fb_info *info, 440 struct inode *inode, 441 struct file *file) 442 { 443 struct fb_deferred_io_state *fbdefio_state = info->fbdefio_state; 444 445 fbdefio_state->mapping = file->f_mapping; 446 file->f_mapping->a_ops = &fb_deferred_io_aops; 447 fbdefio_state->open_count++; 448 } 449 EXPORT_SYMBOL_GPL(fb_deferred_io_open); 450 451 static void fb_deferred_io_lastclose(struct fb_info *info) 452 { 453 flush_delayed_work(&info->deferred_work); 454 } 455 456 void fb_deferred_io_release(struct fb_info *info) 457 { 458 struct fb_deferred_io_state *fbdefio_state = info->fbdefio_state; 459 460 if (!--fbdefio_state->open_count) 461 fb_deferred_io_lastclose(info); 462 } 463 EXPORT_SYMBOL_GPL(fb_deferred_io_release); 464 465 void fb_deferred_io_cleanup(struct fb_info *info) 466 { 467 struct fb_deferred_io_state *fbdefio_state = info->fbdefio_state; 468 469 fb_deferred_io_lastclose(info); 470 471 info->fbdefio_state = NULL; 472 473 mutex_lock(&fbdefio_state->lock); 474 fbdefio_state->info = NULL; 475 mutex_unlock(&fbdefio_state->lock); 476 477 fb_deferred_io_state_put(fbdefio_state); 478 } 479 EXPORT_SYMBOL_GPL(fb_deferred_io_cleanup); 480