11da177e4SLinus Torvalds #include <linux/module.h> 21da177e4SLinus Torvalds #include <linux/highmem.h> 31da177e4SLinus Torvalds #include <asm/tlbflush.h> 41da177e4SLinus Torvalds 51da177e4SLinus Torvalds void *__kmap(struct page *page) 61da177e4SLinus Torvalds { 71da177e4SLinus Torvalds void *addr; 81da177e4SLinus Torvalds 91da177e4SLinus Torvalds might_sleep(); 101da177e4SLinus Torvalds if (!PageHighMem(page)) 111da177e4SLinus Torvalds return page_address(page); 121da177e4SLinus Torvalds addr = kmap_high(page); 131da177e4SLinus Torvalds flush_tlb_one((unsigned long)addr); 141da177e4SLinus Torvalds 151da177e4SLinus Torvalds return addr; 161da177e4SLinus Torvalds } 171da177e4SLinus Torvalds 181da177e4SLinus Torvalds void __kunmap(struct page *page) 191da177e4SLinus Torvalds { 20*b72b7092SRalf Baechle BUG_ON(in_interrupt()); 211da177e4SLinus Torvalds if (!PageHighMem(page)) 221da177e4SLinus Torvalds return; 231da177e4SLinus Torvalds kunmap_high(page); 241da177e4SLinus Torvalds } 251da177e4SLinus Torvalds 261da177e4SLinus Torvalds /* 271da177e4SLinus Torvalds * kmap_atomic/kunmap_atomic is significantly faster than kmap/kunmap because 281da177e4SLinus Torvalds * no global lock is needed and because the kmap code must perform a global TLB 291da177e4SLinus Torvalds * invalidation when the kmap pool wraps. 301da177e4SLinus Torvalds * 311da177e4SLinus Torvalds * However when holding an atomic kmap is is not legal to sleep, so atomic 321da177e4SLinus Torvalds * kmaps are appropriate for short, tight code paths only. 331da177e4SLinus Torvalds */ 341da177e4SLinus Torvalds 351da177e4SLinus Torvalds void *__kmap_atomic(struct page *page, enum km_type type) 361da177e4SLinus Torvalds { 371da177e4SLinus Torvalds enum fixed_addresses idx; 381da177e4SLinus Torvalds unsigned long vaddr; 391da177e4SLinus Torvalds 401da177e4SLinus Torvalds /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */ 41a866374aSPeter Zijlstra pagefault_disable(); 421da177e4SLinus Torvalds if (!PageHighMem(page)) 431da177e4SLinus Torvalds return page_address(page); 441da177e4SLinus Torvalds 451da177e4SLinus Torvalds idx = type + KM_TYPE_NR*smp_processor_id(); 461da177e4SLinus Torvalds vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); 471da177e4SLinus Torvalds #ifdef CONFIG_DEBUG_HIGHMEM 48*b72b7092SRalf Baechle BUG_ON(!pte_none(*(kmap_pte - idx))); 491da177e4SLinus Torvalds #endif 501da177e4SLinus Torvalds set_pte(kmap_pte-idx, mk_pte(page, kmap_prot)); 511da177e4SLinus Torvalds local_flush_tlb_one((unsigned long)vaddr); 521da177e4SLinus Torvalds 531da177e4SLinus Torvalds return (void*) vaddr; 541da177e4SLinus Torvalds } 551da177e4SLinus Torvalds 561da177e4SLinus Torvalds void __kunmap_atomic(void *kvaddr, enum km_type type) 571da177e4SLinus Torvalds { 581da177e4SLinus Torvalds #ifdef CONFIG_DEBUG_HIGHMEM 591da177e4SLinus Torvalds unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK; 601da177e4SLinus Torvalds enum fixed_addresses idx = type + KM_TYPE_NR*smp_processor_id(); 611da177e4SLinus Torvalds 621da177e4SLinus Torvalds if (vaddr < FIXADDR_START) { // FIXME 63a866374aSPeter Zijlstra pagefault_enable(); 641da177e4SLinus Torvalds return; 651da177e4SLinus Torvalds } 661da177e4SLinus Torvalds 67*b72b7092SRalf Baechle BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx)); 681da177e4SLinus Torvalds 691da177e4SLinus Torvalds /* 701da177e4SLinus Torvalds * force other mappings to Oops if they'll try to access 711da177e4SLinus Torvalds * this pte without first remap it 721da177e4SLinus Torvalds */ 731da177e4SLinus Torvalds pte_clear(&init_mm, vaddr, kmap_pte-idx); 741da177e4SLinus Torvalds local_flush_tlb_one(vaddr); 751da177e4SLinus Torvalds #endif 761da177e4SLinus Torvalds 77a866374aSPeter Zijlstra pagefault_enable(); 781da177e4SLinus Torvalds } 791da177e4SLinus Torvalds 8060080265SRalf Baechle /* 8160080265SRalf Baechle * This is the same as kmap_atomic() but can map memory that doesn't 8260080265SRalf Baechle * have a struct page associated with it. 8360080265SRalf Baechle */ 8460080265SRalf Baechle void *kmap_atomic_pfn(unsigned long pfn, enum km_type type) 8560080265SRalf Baechle { 8660080265SRalf Baechle enum fixed_addresses idx; 8760080265SRalf Baechle unsigned long vaddr; 8860080265SRalf Baechle 89a866374aSPeter Zijlstra pagefault_disable(); 9060080265SRalf Baechle 9160080265SRalf Baechle idx = type + KM_TYPE_NR*smp_processor_id(); 9260080265SRalf Baechle vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); 9360080265SRalf Baechle set_pte(kmap_pte-idx, pfn_pte(pfn, kmap_prot)); 9460080265SRalf Baechle flush_tlb_one(vaddr); 9560080265SRalf Baechle 9660080265SRalf Baechle return (void*) vaddr; 9760080265SRalf Baechle } 9860080265SRalf Baechle 991da177e4SLinus Torvalds struct page *__kmap_atomic_to_page(void *ptr) 1001da177e4SLinus Torvalds { 1011da177e4SLinus Torvalds unsigned long idx, vaddr = (unsigned long)ptr; 1021da177e4SLinus Torvalds pte_t *pte; 1031da177e4SLinus Torvalds 1041da177e4SLinus Torvalds if (vaddr < FIXADDR_START) 1051da177e4SLinus Torvalds return virt_to_page(ptr); 1061da177e4SLinus Torvalds 1071da177e4SLinus Torvalds idx = virt_to_fix(vaddr); 1081da177e4SLinus Torvalds pte = kmap_pte - (idx - FIX_KMAP_BEGIN); 1091da177e4SLinus Torvalds return pte_page(*pte); 1101da177e4SLinus Torvalds } 1111da177e4SLinus Torvalds 1121da177e4SLinus Torvalds EXPORT_SYMBOL(__kmap); 1131da177e4SLinus Torvalds EXPORT_SYMBOL(__kunmap); 1141da177e4SLinus Torvalds EXPORT_SYMBOL(__kmap_atomic); 1151da177e4SLinus Torvalds EXPORT_SYMBOL(__kunmap_atomic); 116