1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * High memory handling common code and variables. 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * (C) 1999 Andrea Arcangeli, SuSE GmbH, andrea@suse.de 61da177e4SLinus Torvalds * Gerhard Wichert, Siemens AG, Gerhard.Wichert@pdb.siemens.de 71da177e4SLinus Torvalds * 81da177e4SLinus Torvalds * 91da177e4SLinus Torvalds * Redesigned the x86 32-bit VM architecture to deal with 101da177e4SLinus Torvalds * 64-bit physical space. With current x86 CPUs this 111da177e4SLinus Torvalds * means up to 64 Gigabytes physical RAM. 121da177e4SLinus Torvalds * 131da177e4SLinus Torvalds * Rewrote high memory support to move the page cache into 141da177e4SLinus Torvalds * high memory. Implemented permanent (schedulable) kmaps 151da177e4SLinus Torvalds * based on Linus' idea. 161da177e4SLinus Torvalds * 171da177e4SLinus Torvalds * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com> 181da177e4SLinus Torvalds */ 191da177e4SLinus Torvalds 201da177e4SLinus Torvalds #include <linux/mm.h> 21b95f1b31SPaul Gortmaker #include <linux/export.h> 221da177e4SLinus Torvalds #include <linux/swap.h> 231da177e4SLinus Torvalds #include <linux/bio.h> 241da177e4SLinus Torvalds #include <linux/pagemap.h> 251da177e4SLinus Torvalds #include <linux/mempool.h> 261da177e4SLinus Torvalds #include <linux/blkdev.h> 271da177e4SLinus Torvalds #include <linux/init.h> 281da177e4SLinus Torvalds #include <linux/hash.h> 291da177e4SLinus Torvalds #include <linux/highmem.h> 30eac79005SJason Wessel #include <linux/kgdb.h> 311da177e4SLinus Torvalds #include <asm/tlbflush.h> 32186525bdSIngo Molnar #include <linux/vmalloc.h> 33a8e23a29SPeter Zijlstra 341da177e4SLinus Torvalds /* 351da177e4SLinus Torvalds * Virtual_count is not a pure "count". 361da177e4SLinus Torvalds * 0 means that it is not mapped, and has not been mapped 371da177e4SLinus Torvalds * since a TLB flush - it is usable. 381da177e4SLinus Torvalds * 1 means that there are no users, but it has been mapped 391da177e4SLinus Torvalds * since the last TLB flush - so we can't use it. 401da177e4SLinus Torvalds * n means that there are (n-1) current users of it. 411da177e4SLinus Torvalds */ 421da177e4SLinus Torvalds #ifdef CONFIG_HIGHMEM 43260b2367SAl Viro 4415de36a4SMax Filippov /* 4515de36a4SMax Filippov * Architecture with aliasing data cache may define the following family of 4615de36a4SMax Filippov * helper functions in its asm/highmem.h to control cache color of virtual 4715de36a4SMax Filippov * addresses where physical memory pages are mapped by kmap. 4815de36a4SMax Filippov */ 4915de36a4SMax Filippov #ifndef get_pkmap_color 5015de36a4SMax Filippov 5115de36a4SMax Filippov /* 5215de36a4SMax Filippov * Determine color of virtual address where the page should be mapped. 5315de36a4SMax Filippov */ 5415de36a4SMax Filippov static inline unsigned int get_pkmap_color(struct page *page) 5515de36a4SMax Filippov { 5615de36a4SMax Filippov return 0; 5715de36a4SMax Filippov } 5815de36a4SMax Filippov #define get_pkmap_color get_pkmap_color 5915de36a4SMax Filippov 6015de36a4SMax Filippov /* 6115de36a4SMax Filippov * Get next index for mapping inside PKMAP region for page with given color. 6215de36a4SMax Filippov */ 6315de36a4SMax Filippov static inline unsigned int get_next_pkmap_nr(unsigned int color) 6415de36a4SMax Filippov { 6515de36a4SMax Filippov static unsigned int last_pkmap_nr; 6615de36a4SMax Filippov 6715de36a4SMax Filippov last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK; 6815de36a4SMax Filippov return last_pkmap_nr; 6915de36a4SMax Filippov } 7015de36a4SMax Filippov 7115de36a4SMax Filippov /* 7215de36a4SMax Filippov * Determine if page index inside PKMAP region (pkmap_nr) of given color 7315de36a4SMax Filippov * has wrapped around PKMAP region end. When this happens an attempt to 7415de36a4SMax Filippov * flush all unused PKMAP slots is made. 7515de36a4SMax Filippov */ 7615de36a4SMax Filippov static inline int no_more_pkmaps(unsigned int pkmap_nr, unsigned int color) 7715de36a4SMax Filippov { 7815de36a4SMax Filippov return pkmap_nr == 0; 7915de36a4SMax Filippov } 8015de36a4SMax Filippov 8115de36a4SMax Filippov /* 8215de36a4SMax Filippov * Get the number of PKMAP entries of the given color. If no free slot is 8315de36a4SMax Filippov * found after checking that many entries, kmap will sleep waiting for 8415de36a4SMax Filippov * someone to call kunmap and free PKMAP slot. 8515de36a4SMax Filippov */ 8615de36a4SMax Filippov static inline int get_pkmap_entries_count(unsigned int color) 8715de36a4SMax Filippov { 8815de36a4SMax Filippov return LAST_PKMAP; 8915de36a4SMax Filippov } 9015de36a4SMax Filippov 9115de36a4SMax Filippov /* 9215de36a4SMax Filippov * Get head of a wait queue for PKMAP entries of the given color. 9315de36a4SMax Filippov * Wait queues for different mapping colors should be independent to avoid 9415de36a4SMax Filippov * unnecessary wakeups caused by freeing of slots of other colors. 9515de36a4SMax Filippov */ 9615de36a4SMax Filippov static inline wait_queue_head_t *get_pkmap_wait_queue_head(unsigned int color) 9715de36a4SMax Filippov { 9815de36a4SMax Filippov static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait); 9915de36a4SMax Filippov 10015de36a4SMax Filippov return &pkmap_map_wait; 10115de36a4SMax Filippov } 10215de36a4SMax Filippov #endif 10315de36a4SMax Filippov 104ca79b0c2SArun KS atomic_long_t _totalhigh_pages __read_mostly; 105ca79b0c2SArun KS EXPORT_SYMBOL(_totalhigh_pages); 1063e4d3af5SPeter Zijlstra 10713f876baSThomas Gleixner unsigned int __nr_free_highpages(void) 108c1f60a5aSChristoph Lameter { 10933499bfeSJoonsoo Kim struct zone *zone; 110c1f60a5aSChristoph Lameter unsigned int pages = 0; 111c1f60a5aSChristoph Lameter 11233499bfeSJoonsoo Kim for_each_populated_zone(zone) { 11333499bfeSJoonsoo Kim if (is_highmem(zone)) 11433499bfeSJoonsoo Kim pages += zone_page_state(zone, NR_FREE_PAGES); 1152a1e274aSMel Gorman } 116c1f60a5aSChristoph Lameter 117c1f60a5aSChristoph Lameter return pages; 118c1f60a5aSChristoph Lameter } 119c1f60a5aSChristoph Lameter 1201da177e4SLinus Torvalds static int pkmap_count[LAST_PKMAP]; 1211da177e4SLinus Torvalds static __cacheline_aligned_in_smp DEFINE_SPINLOCK(kmap_lock); 1221da177e4SLinus Torvalds 1231da177e4SLinus Torvalds pte_t *pkmap_page_table; 1241da177e4SLinus Torvalds 1253297e760SNicolas Pitre /* 1263297e760SNicolas Pitre * Most architectures have no use for kmap_high_get(), so let's abstract 1273297e760SNicolas Pitre * the disabling of IRQ out of the locking in that case to save on a 1283297e760SNicolas Pitre * potential useless overhead. 1293297e760SNicolas Pitre */ 1303297e760SNicolas Pitre #ifdef ARCH_NEEDS_KMAP_HIGH_GET 1313297e760SNicolas Pitre #define lock_kmap() spin_lock_irq(&kmap_lock) 1323297e760SNicolas Pitre #define unlock_kmap() spin_unlock_irq(&kmap_lock) 1333297e760SNicolas Pitre #define lock_kmap_any(flags) spin_lock_irqsave(&kmap_lock, flags) 1343297e760SNicolas Pitre #define unlock_kmap_any(flags) spin_unlock_irqrestore(&kmap_lock, flags) 1353297e760SNicolas Pitre #else 1363297e760SNicolas Pitre #define lock_kmap() spin_lock(&kmap_lock) 1373297e760SNicolas Pitre #define unlock_kmap() spin_unlock(&kmap_lock) 1383297e760SNicolas Pitre #define lock_kmap_any(flags) \ 1393297e760SNicolas Pitre do { spin_lock(&kmap_lock); (void)(flags); } while (0) 1403297e760SNicolas Pitre #define unlock_kmap_any(flags) \ 1413297e760SNicolas Pitre do { spin_unlock(&kmap_lock); (void)(flags); } while (0) 1423297e760SNicolas Pitre #endif 1433297e760SNicolas Pitre 14413f876baSThomas Gleixner struct page *__kmap_to_page(void *vaddr) 1455a178119SMel Gorman { 1465a178119SMel Gorman unsigned long addr = (unsigned long)vaddr; 1475a178119SMel Gorman 148498c2280SWill Deacon if (addr >= PKMAP_ADDR(0) && addr < PKMAP_ADDR(LAST_PKMAP)) { 1494de22c05SJoonsoo Kim int i = PKMAP_NR(addr); 1509727688dSsongqiang 1515a178119SMel Gorman return pte_page(pkmap_page_table[i]); 1525a178119SMel Gorman } 1535a178119SMel Gorman 1545a178119SMel Gorman return virt_to_page(addr); 1555a178119SMel Gorman } 15613f876baSThomas Gleixner EXPORT_SYMBOL(__kmap_to_page); 1575a178119SMel Gorman 1581da177e4SLinus Torvalds static void flush_all_zero_pkmaps(void) 1591da177e4SLinus Torvalds { 1601da177e4SLinus Torvalds int i; 1615843d9a4SNick Piggin int need_flush = 0; 1621da177e4SLinus Torvalds 1631da177e4SLinus Torvalds flush_cache_kmaps(); 1641da177e4SLinus Torvalds 1651da177e4SLinus Torvalds for (i = 0; i < LAST_PKMAP; i++) { 1661da177e4SLinus Torvalds struct page *page; 1671da177e4SLinus Torvalds 1681da177e4SLinus Torvalds /* 1691da177e4SLinus Torvalds * zero means we don't have anything to do, 1701da177e4SLinus Torvalds * >1 means that it is still in use. Only 1711da177e4SLinus Torvalds * a count of 1 means that it is free but 1721da177e4SLinus Torvalds * needs to be unmapped 1731da177e4SLinus Torvalds */ 1741da177e4SLinus Torvalds if (pkmap_count[i] != 1) 1751da177e4SLinus Torvalds continue; 1761da177e4SLinus Torvalds pkmap_count[i] = 0; 1771da177e4SLinus Torvalds 1781da177e4SLinus Torvalds /* sanity check */ 17975babcacSEric Sesterhenn BUG_ON(pte_none(pkmap_page_table[i])); 1801da177e4SLinus Torvalds 1811da177e4SLinus Torvalds /* 1821da177e4SLinus Torvalds * Don't need an atomic fetch-and-clear op here; 1831da177e4SLinus Torvalds * no-one has the page mapped, and cannot get at 1841da177e4SLinus Torvalds * its virtual address (and hence PTE) without first 1851da177e4SLinus Torvalds * getting the kmap_lock (which is held here). 1861da177e4SLinus Torvalds * So no dangers, even with speculative execution. 1871da177e4SLinus Torvalds */ 1881da177e4SLinus Torvalds page = pte_page(pkmap_page_table[i]); 189eb2db439SJoonsoo Kim pte_clear(&init_mm, PKMAP_ADDR(i), &pkmap_page_table[i]); 1901da177e4SLinus Torvalds 1911da177e4SLinus Torvalds set_page_address(page, NULL); 1925843d9a4SNick Piggin need_flush = 1; 1931da177e4SLinus Torvalds } 1945843d9a4SNick Piggin if (need_flush) 1951da177e4SLinus Torvalds flush_tlb_kernel_range(PKMAP_ADDR(0), PKMAP_ADDR(LAST_PKMAP)); 1961da177e4SLinus Torvalds } 1971da177e4SLinus Torvalds 19813f876baSThomas Gleixner void __kmap_flush_unused(void) 199ce6234b5SJeremy Fitzhardinge { 2003297e760SNicolas Pitre lock_kmap(); 201ce6234b5SJeremy Fitzhardinge flush_all_zero_pkmaps(); 2023297e760SNicolas Pitre unlock_kmap(); 203ce6234b5SJeremy Fitzhardinge } 204ce6234b5SJeremy Fitzhardinge 2051da177e4SLinus Torvalds static inline unsigned long map_new_virtual(struct page *page) 2061da177e4SLinus Torvalds { 2071da177e4SLinus Torvalds unsigned long vaddr; 2081da177e4SLinus Torvalds int count; 20915de36a4SMax Filippov unsigned int last_pkmap_nr; 21015de36a4SMax Filippov unsigned int color = get_pkmap_color(page); 2111da177e4SLinus Torvalds 2121da177e4SLinus Torvalds start: 21315de36a4SMax Filippov count = get_pkmap_entries_count(color); 2141da177e4SLinus Torvalds /* Find an empty entry */ 2151da177e4SLinus Torvalds for (;;) { 21615de36a4SMax Filippov last_pkmap_nr = get_next_pkmap_nr(color); 21715de36a4SMax Filippov if (no_more_pkmaps(last_pkmap_nr, color)) { 2181da177e4SLinus Torvalds flush_all_zero_pkmaps(); 21915de36a4SMax Filippov count = get_pkmap_entries_count(color); 2201da177e4SLinus Torvalds } 2211da177e4SLinus Torvalds if (!pkmap_count[last_pkmap_nr]) 2221da177e4SLinus Torvalds break; /* Found a usable entry */ 2231da177e4SLinus Torvalds if (--count) 2241da177e4SLinus Torvalds continue; 2251da177e4SLinus Torvalds 2261da177e4SLinus Torvalds /* 2271da177e4SLinus Torvalds * Sleep for somebody else to unmap their entries 2281da177e4SLinus Torvalds */ 2291da177e4SLinus Torvalds { 2301da177e4SLinus Torvalds DECLARE_WAITQUEUE(wait, current); 23115de36a4SMax Filippov wait_queue_head_t *pkmap_map_wait = 23215de36a4SMax Filippov get_pkmap_wait_queue_head(color); 2331da177e4SLinus Torvalds 2341da177e4SLinus Torvalds __set_current_state(TASK_UNINTERRUPTIBLE); 23515de36a4SMax Filippov add_wait_queue(pkmap_map_wait, &wait); 2363297e760SNicolas Pitre unlock_kmap(); 2371da177e4SLinus Torvalds schedule(); 23815de36a4SMax Filippov remove_wait_queue(pkmap_map_wait, &wait); 2393297e760SNicolas Pitre lock_kmap(); 2401da177e4SLinus Torvalds 2411da177e4SLinus Torvalds /* Somebody else might have mapped it while we slept */ 2421da177e4SLinus Torvalds if (page_address(page)) 2431da177e4SLinus Torvalds return (unsigned long)page_address(page); 2441da177e4SLinus Torvalds 2451da177e4SLinus Torvalds /* Re-start */ 2461da177e4SLinus Torvalds goto start; 2471da177e4SLinus Torvalds } 2481da177e4SLinus Torvalds } 2491da177e4SLinus Torvalds vaddr = PKMAP_ADDR(last_pkmap_nr); 2501da177e4SLinus Torvalds set_pte_at(&init_mm, vaddr, 2511da177e4SLinus Torvalds &(pkmap_page_table[last_pkmap_nr]), mk_pte(page, kmap_prot)); 2521da177e4SLinus Torvalds 2531da177e4SLinus Torvalds pkmap_count[last_pkmap_nr] = 1; 2541da177e4SLinus Torvalds set_page_address(page, (void *)vaddr); 2551da177e4SLinus Torvalds 2561da177e4SLinus Torvalds return vaddr; 2571da177e4SLinus Torvalds } 2581da177e4SLinus Torvalds 25977f6078aSRandy Dunlap /** 26077f6078aSRandy Dunlap * kmap_high - map a highmem page into memory 26177f6078aSRandy Dunlap * @page: &struct page to map 26277f6078aSRandy Dunlap * 26377f6078aSRandy Dunlap * Returns the page's virtual memory address. 26477f6078aSRandy Dunlap * 26577f6078aSRandy Dunlap * We cannot call this from interrupts, as it may block. 26677f6078aSRandy Dunlap */ 267920c7a5dSHarvey Harrison void *kmap_high(struct page *page) 2681da177e4SLinus Torvalds { 2691da177e4SLinus Torvalds unsigned long vaddr; 2701da177e4SLinus Torvalds 2711da177e4SLinus Torvalds /* 2721da177e4SLinus Torvalds * For highmem pages, we can't trust "virtual" until 2731da177e4SLinus Torvalds * after we have the lock. 2741da177e4SLinus Torvalds */ 2753297e760SNicolas Pitre lock_kmap(); 2761da177e4SLinus Torvalds vaddr = (unsigned long)page_address(page); 2771da177e4SLinus Torvalds if (!vaddr) 2781da177e4SLinus Torvalds vaddr = map_new_virtual(page); 2791da177e4SLinus Torvalds pkmap_count[PKMAP_NR(vaddr)]++; 28075babcacSEric Sesterhenn BUG_ON(pkmap_count[PKMAP_NR(vaddr)] < 2); 2813297e760SNicolas Pitre unlock_kmap(); 2821da177e4SLinus Torvalds return (void *) vaddr; 2831da177e4SLinus Torvalds } 2841da177e4SLinus Torvalds EXPORT_SYMBOL(kmap_high); 2851da177e4SLinus Torvalds 2863297e760SNicolas Pitre #ifdef ARCH_NEEDS_KMAP_HIGH_GET 2873297e760SNicolas Pitre /** 2883297e760SNicolas Pitre * kmap_high_get - pin a highmem page into memory 2893297e760SNicolas Pitre * @page: &struct page to pin 2903297e760SNicolas Pitre * 2913297e760SNicolas Pitre * Returns the page's current virtual memory address, or NULL if no mapping 2925e39df56SUwe Kleine-König * exists. If and only if a non null address is returned then a 2933297e760SNicolas Pitre * matching call to kunmap_high() is necessary. 2943297e760SNicolas Pitre * 2953297e760SNicolas Pitre * This can be called from any context. 2963297e760SNicolas Pitre */ 2973297e760SNicolas Pitre void *kmap_high_get(struct page *page) 2983297e760SNicolas Pitre { 2993297e760SNicolas Pitre unsigned long vaddr, flags; 3003297e760SNicolas Pitre 3013297e760SNicolas Pitre lock_kmap_any(flags); 3023297e760SNicolas Pitre vaddr = (unsigned long)page_address(page); 3033297e760SNicolas Pitre if (vaddr) { 3043297e760SNicolas Pitre BUG_ON(pkmap_count[PKMAP_NR(vaddr)] < 1); 3053297e760SNicolas Pitre pkmap_count[PKMAP_NR(vaddr)]++; 3063297e760SNicolas Pitre } 3073297e760SNicolas Pitre unlock_kmap_any(flags); 3083297e760SNicolas Pitre return (void *) vaddr; 3093297e760SNicolas Pitre } 3103297e760SNicolas Pitre #endif 3113297e760SNicolas Pitre 31277f6078aSRandy Dunlap /** 3134e9dc5dfSLi Haifeng * kunmap_high - unmap a highmem page into memory 31477f6078aSRandy Dunlap * @page: &struct page to unmap 3153297e760SNicolas Pitre * 3163297e760SNicolas Pitre * If ARCH_NEEDS_KMAP_HIGH_GET is not defined then this may be called 3173297e760SNicolas Pitre * only from user context. 31877f6078aSRandy Dunlap */ 319920c7a5dSHarvey Harrison void kunmap_high(struct page *page) 3201da177e4SLinus Torvalds { 3211da177e4SLinus Torvalds unsigned long vaddr; 3221da177e4SLinus Torvalds unsigned long nr; 3233297e760SNicolas Pitre unsigned long flags; 3241da177e4SLinus Torvalds int need_wakeup; 32515de36a4SMax Filippov unsigned int color = get_pkmap_color(page); 32615de36a4SMax Filippov wait_queue_head_t *pkmap_map_wait; 3271da177e4SLinus Torvalds 3283297e760SNicolas Pitre lock_kmap_any(flags); 3291da177e4SLinus Torvalds vaddr = (unsigned long)page_address(page); 33075babcacSEric Sesterhenn BUG_ON(!vaddr); 3311da177e4SLinus Torvalds nr = PKMAP_NR(vaddr); 3321da177e4SLinus Torvalds 3331da177e4SLinus Torvalds /* 3341da177e4SLinus Torvalds * A count must never go down to zero 3351da177e4SLinus Torvalds * without a TLB flush! 3361da177e4SLinus Torvalds */ 3371da177e4SLinus Torvalds need_wakeup = 0; 3381da177e4SLinus Torvalds switch (--pkmap_count[nr]) { 3391da177e4SLinus Torvalds case 0: 3401da177e4SLinus Torvalds BUG(); 3411da177e4SLinus Torvalds case 1: 3421da177e4SLinus Torvalds /* 3431da177e4SLinus Torvalds * Avoid an unnecessary wake_up() function call. 3441da177e4SLinus Torvalds * The common case is pkmap_count[] == 1, but 3451da177e4SLinus Torvalds * no waiters. 3461da177e4SLinus Torvalds * The tasks queued in the wait-queue are guarded 3471da177e4SLinus Torvalds * by both the lock in the wait-queue-head and by 3481da177e4SLinus Torvalds * the kmap_lock. As the kmap_lock is held here, 3491da177e4SLinus Torvalds * no need for the wait-queue-head's lock. Simply 3501da177e4SLinus Torvalds * test if the queue is empty. 3511da177e4SLinus Torvalds */ 35215de36a4SMax Filippov pkmap_map_wait = get_pkmap_wait_queue_head(color); 35315de36a4SMax Filippov need_wakeup = waitqueue_active(pkmap_map_wait); 3541da177e4SLinus Torvalds } 3553297e760SNicolas Pitre unlock_kmap_any(flags); 3561da177e4SLinus Torvalds 3571da177e4SLinus Torvalds /* do wake-up, if needed, race-free outside of the spin lock */ 3581da177e4SLinus Torvalds if (need_wakeup) 35915de36a4SMax Filippov wake_up(pkmap_map_wait); 3601da177e4SLinus Torvalds } 3611da177e4SLinus Torvalds EXPORT_SYMBOL(kunmap_high); 3620060ef3bSMatthew Wilcox (Oracle) 3630060ef3bSMatthew Wilcox (Oracle) #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3640060ef3bSMatthew Wilcox (Oracle) void zero_user_segments(struct page *page, unsigned start1, unsigned end1, 3650060ef3bSMatthew Wilcox (Oracle) unsigned start2, unsigned end2) 3660060ef3bSMatthew Wilcox (Oracle) { 3670060ef3bSMatthew Wilcox (Oracle) unsigned int i; 3680060ef3bSMatthew Wilcox (Oracle) 3690060ef3bSMatthew Wilcox (Oracle) BUG_ON(end1 > page_size(page) || end2 > page_size(page)); 3700060ef3bSMatthew Wilcox (Oracle) 371184cee51SOGAWA Hirofumi if (start1 >= end1) 372184cee51SOGAWA Hirofumi start1 = end1 = 0; 373184cee51SOGAWA Hirofumi if (start2 >= end2) 374184cee51SOGAWA Hirofumi start2 = end2 = 0; 375184cee51SOGAWA Hirofumi 3760060ef3bSMatthew Wilcox (Oracle) for (i = 0; i < compound_nr(page); i++) { 3770060ef3bSMatthew Wilcox (Oracle) void *kaddr = NULL; 3780060ef3bSMatthew Wilcox (Oracle) 3790060ef3bSMatthew Wilcox (Oracle) if (start1 >= PAGE_SIZE) { 3800060ef3bSMatthew Wilcox (Oracle) start1 -= PAGE_SIZE; 3810060ef3bSMatthew Wilcox (Oracle) end1 -= PAGE_SIZE; 3820060ef3bSMatthew Wilcox (Oracle) } else { 3830060ef3bSMatthew Wilcox (Oracle) unsigned this_end = min_t(unsigned, end1, PAGE_SIZE); 3840060ef3bSMatthew Wilcox (Oracle) 385184cee51SOGAWA Hirofumi if (end1 > start1) { 386*d2c20e51SIra Weiny kaddr = kmap_local_page(page + i); 3870060ef3bSMatthew Wilcox (Oracle) memset(kaddr + start1, 0, this_end - start1); 388184cee51SOGAWA Hirofumi } 3890060ef3bSMatthew Wilcox (Oracle) end1 -= this_end; 3900060ef3bSMatthew Wilcox (Oracle) start1 = 0; 3910060ef3bSMatthew Wilcox (Oracle) } 3920060ef3bSMatthew Wilcox (Oracle) 3930060ef3bSMatthew Wilcox (Oracle) if (start2 >= PAGE_SIZE) { 3940060ef3bSMatthew Wilcox (Oracle) start2 -= PAGE_SIZE; 3950060ef3bSMatthew Wilcox (Oracle) end2 -= PAGE_SIZE; 3960060ef3bSMatthew Wilcox (Oracle) } else { 3970060ef3bSMatthew Wilcox (Oracle) unsigned this_end = min_t(unsigned, end2, PAGE_SIZE); 3980060ef3bSMatthew Wilcox (Oracle) 399184cee51SOGAWA Hirofumi if (end2 > start2) { 400184cee51SOGAWA Hirofumi if (!kaddr) 401*d2c20e51SIra Weiny kaddr = kmap_local_page(page + i); 4020060ef3bSMatthew Wilcox (Oracle) memset(kaddr + start2, 0, this_end - start2); 403184cee51SOGAWA Hirofumi } 4040060ef3bSMatthew Wilcox (Oracle) end2 -= this_end; 4050060ef3bSMatthew Wilcox (Oracle) start2 = 0; 4060060ef3bSMatthew Wilcox (Oracle) } 4070060ef3bSMatthew Wilcox (Oracle) 4080060ef3bSMatthew Wilcox (Oracle) if (kaddr) { 409*d2c20e51SIra Weiny kunmap_local(kaddr); 4100060ef3bSMatthew Wilcox (Oracle) flush_dcache_page(page + i); 4110060ef3bSMatthew Wilcox (Oracle) } 4120060ef3bSMatthew Wilcox (Oracle) 4130060ef3bSMatthew Wilcox (Oracle) if (!end1 && !end2) 4140060ef3bSMatthew Wilcox (Oracle) break; 4150060ef3bSMatthew Wilcox (Oracle) } 4160060ef3bSMatthew Wilcox (Oracle) 4170060ef3bSMatthew Wilcox (Oracle) BUG_ON((start1 | start2 | end1 | end2) != 0); 4180060ef3bSMatthew Wilcox (Oracle) } 4190060ef3bSMatthew Wilcox (Oracle) EXPORT_SYMBOL(zero_user_segments); 4200060ef3bSMatthew Wilcox (Oracle) #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 421955cc774SIra Weiny #endif /* CONFIG_HIGHMEM */ 4221da177e4SLinus Torvalds 423298fa1adSThomas Gleixner #ifdef CONFIG_KMAP_LOCAL 424298fa1adSThomas Gleixner 425298fa1adSThomas Gleixner #include <asm/kmap_size.h> 426298fa1adSThomas Gleixner 427389755c2SThomas Gleixner /* 4286e799cb6SThomas Gleixner * With DEBUG_KMAP_LOCAL the stack depth is doubled and every second 429389755c2SThomas Gleixner * slot is unused which acts as a guard page 430389755c2SThomas Gleixner */ 4316e799cb6SThomas Gleixner #ifdef CONFIG_DEBUG_KMAP_LOCAL 432389755c2SThomas Gleixner # define KM_INCR 2 433389755c2SThomas Gleixner #else 434389755c2SThomas Gleixner # define KM_INCR 1 435389755c2SThomas Gleixner #endif 436389755c2SThomas Gleixner 437298fa1adSThomas Gleixner static inline int kmap_local_idx_push(void) 438298fa1adSThomas Gleixner { 439ea0eafeaSChangbin Du WARN_ON_ONCE(in_hardirq() && !irqs_disabled()); 4405fbda3ecSThomas Gleixner current->kmap_ctrl.idx += KM_INCR; 4415fbda3ecSThomas Gleixner BUG_ON(current->kmap_ctrl.idx >= KM_MAX_IDX); 4425fbda3ecSThomas Gleixner return current->kmap_ctrl.idx - 1; 443298fa1adSThomas Gleixner } 444298fa1adSThomas Gleixner 445298fa1adSThomas Gleixner static inline int kmap_local_idx(void) 446298fa1adSThomas Gleixner { 4475fbda3ecSThomas Gleixner return current->kmap_ctrl.idx - 1; 448298fa1adSThomas Gleixner } 449298fa1adSThomas Gleixner 450298fa1adSThomas Gleixner static inline void kmap_local_idx_pop(void) 451298fa1adSThomas Gleixner { 4525fbda3ecSThomas Gleixner current->kmap_ctrl.idx -= KM_INCR; 4535fbda3ecSThomas Gleixner BUG_ON(current->kmap_ctrl.idx < 0); 454298fa1adSThomas Gleixner } 455298fa1adSThomas Gleixner 456298fa1adSThomas Gleixner #ifndef arch_kmap_local_post_map 457298fa1adSThomas Gleixner # define arch_kmap_local_post_map(vaddr, pteval) do { } while (0) 458298fa1adSThomas Gleixner #endif 4593c1016b5SThomas Gleixner 460298fa1adSThomas Gleixner #ifndef arch_kmap_local_pre_unmap 461298fa1adSThomas Gleixner # define arch_kmap_local_pre_unmap(vaddr) do { } while (0) 462298fa1adSThomas Gleixner #endif 463298fa1adSThomas Gleixner 464298fa1adSThomas Gleixner #ifndef arch_kmap_local_post_unmap 465298fa1adSThomas Gleixner # define arch_kmap_local_post_unmap(vaddr) do { } while (0) 466298fa1adSThomas Gleixner #endif 467298fa1adSThomas Gleixner 468298fa1adSThomas Gleixner #ifndef arch_kmap_local_map_idx 469298fa1adSThomas Gleixner #define arch_kmap_local_map_idx(idx, pfn) kmap_local_calc_idx(idx) 470298fa1adSThomas Gleixner #endif 471298fa1adSThomas Gleixner 472298fa1adSThomas Gleixner #ifndef arch_kmap_local_unmap_idx 473298fa1adSThomas Gleixner #define arch_kmap_local_unmap_idx(idx, vaddr) kmap_local_calc_idx(idx) 474298fa1adSThomas Gleixner #endif 475298fa1adSThomas Gleixner 476298fa1adSThomas Gleixner #ifndef arch_kmap_local_high_get 477298fa1adSThomas Gleixner static inline void *arch_kmap_local_high_get(struct page *page) 478298fa1adSThomas Gleixner { 479298fa1adSThomas Gleixner return NULL; 480298fa1adSThomas Gleixner } 481298fa1adSThomas Gleixner #endif 482298fa1adSThomas Gleixner 483a1dce7fdSThomas Gleixner #ifndef arch_kmap_local_set_pte 484a1dce7fdSThomas Gleixner #define arch_kmap_local_set_pte(mm, vaddr, ptep, ptev) \ 485a1dce7fdSThomas Gleixner set_pte_at(mm, vaddr, ptep, ptev) 486a1dce7fdSThomas Gleixner #endif 487a1dce7fdSThomas Gleixner 488298fa1adSThomas Gleixner /* Unmap a local mapping which was obtained by kmap_high_get() */ 4892a656cadSThomas Gleixner static inline bool kmap_high_unmap_local(unsigned long vaddr) 490298fa1adSThomas Gleixner { 491298fa1adSThomas Gleixner #ifdef ARCH_NEEDS_KMAP_HIGH_GET 4922a656cadSThomas Gleixner if (vaddr >= PKMAP_ADDR(0) && vaddr < PKMAP_ADDR(LAST_PKMAP)) { 493298fa1adSThomas Gleixner kunmap_high(pte_page(pkmap_page_table[PKMAP_NR(vaddr)])); 4942a656cadSThomas Gleixner return true; 4952a656cadSThomas Gleixner } 496298fa1adSThomas Gleixner #endif 4972a656cadSThomas Gleixner return false; 498298fa1adSThomas Gleixner } 499298fa1adSThomas Gleixner 500298fa1adSThomas Gleixner static inline int kmap_local_calc_idx(int idx) 501298fa1adSThomas Gleixner { 502298fa1adSThomas Gleixner return idx + KM_MAX_IDX * smp_processor_id(); 503298fa1adSThomas Gleixner } 504298fa1adSThomas Gleixner 505298fa1adSThomas Gleixner static pte_t *__kmap_pte; 506298fa1adSThomas Gleixner 507298fa1adSThomas Gleixner static pte_t *kmap_get_pte(void) 508298fa1adSThomas Gleixner { 509298fa1adSThomas Gleixner if (!__kmap_pte) 510298fa1adSThomas Gleixner __kmap_pte = virt_to_kpte(__fix_to_virt(FIX_KMAP_BEGIN)); 511298fa1adSThomas Gleixner return __kmap_pte; 512298fa1adSThomas Gleixner } 513298fa1adSThomas Gleixner 514298fa1adSThomas Gleixner void *__kmap_local_pfn_prot(unsigned long pfn, pgprot_t prot) 515298fa1adSThomas Gleixner { 516298fa1adSThomas Gleixner pte_t pteval, *kmap_pte = kmap_get_pte(); 517298fa1adSThomas Gleixner unsigned long vaddr; 518298fa1adSThomas Gleixner int idx; 519298fa1adSThomas Gleixner 520f3ba3c71SThomas Gleixner /* 521f3ba3c71SThomas Gleixner * Disable migration so resulting virtual address is stable 522f0953a1bSIngo Molnar * across preemption. 523f3ba3c71SThomas Gleixner */ 524f3ba3c71SThomas Gleixner migrate_disable(); 525298fa1adSThomas Gleixner preempt_disable(); 526298fa1adSThomas Gleixner idx = arch_kmap_local_map_idx(kmap_local_idx_push(), pfn); 527298fa1adSThomas Gleixner vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); 528298fa1adSThomas Gleixner BUG_ON(!pte_none(*(kmap_pte - idx))); 529298fa1adSThomas Gleixner pteval = pfn_pte(pfn, prot); 530a1dce7fdSThomas Gleixner arch_kmap_local_set_pte(&init_mm, vaddr, kmap_pte - idx, pteval); 531298fa1adSThomas Gleixner arch_kmap_local_post_map(vaddr, pteval); 5325fbda3ecSThomas Gleixner current->kmap_ctrl.pteval[kmap_local_idx()] = pteval; 533298fa1adSThomas Gleixner preempt_enable(); 534298fa1adSThomas Gleixner 535298fa1adSThomas Gleixner return (void *)vaddr; 536298fa1adSThomas Gleixner } 537298fa1adSThomas Gleixner EXPORT_SYMBOL_GPL(__kmap_local_pfn_prot); 538298fa1adSThomas Gleixner 539298fa1adSThomas Gleixner void *__kmap_local_page_prot(struct page *page, pgprot_t prot) 540298fa1adSThomas Gleixner { 541298fa1adSThomas Gleixner void *kmap; 542298fa1adSThomas Gleixner 5430e91a0c6SThomas Gleixner /* 5440e91a0c6SThomas Gleixner * To broaden the usage of the actual kmap_local() machinery always map 5450e91a0c6SThomas Gleixner * pages when debugging is enabled and the architecture has no problems 5460e91a0c6SThomas Gleixner * with alias mappings. 5470e91a0c6SThomas Gleixner */ 5480e91a0c6SThomas Gleixner if (!IS_ENABLED(CONFIG_DEBUG_KMAP_LOCAL_FORCE_MAP) && !PageHighMem(page)) 549298fa1adSThomas Gleixner return page_address(page); 550298fa1adSThomas Gleixner 551298fa1adSThomas Gleixner /* Try kmap_high_get() if architecture has it enabled */ 552298fa1adSThomas Gleixner kmap = arch_kmap_local_high_get(page); 553298fa1adSThomas Gleixner if (kmap) 554298fa1adSThomas Gleixner return kmap; 555298fa1adSThomas Gleixner 556298fa1adSThomas Gleixner return __kmap_local_pfn_prot(page_to_pfn(page), prot); 557298fa1adSThomas Gleixner } 558298fa1adSThomas Gleixner EXPORT_SYMBOL(__kmap_local_page_prot); 559298fa1adSThomas Gleixner 560298fa1adSThomas Gleixner void kunmap_local_indexed(void *vaddr) 561298fa1adSThomas Gleixner { 562298fa1adSThomas Gleixner unsigned long addr = (unsigned long) vaddr & PAGE_MASK; 563298fa1adSThomas Gleixner pte_t *kmap_pte = kmap_get_pte(); 564298fa1adSThomas Gleixner int idx; 565298fa1adSThomas Gleixner 566298fa1adSThomas Gleixner if (addr < __fix_to_virt(FIX_KMAP_END) || 567298fa1adSThomas Gleixner addr > __fix_to_virt(FIX_KMAP_BEGIN)) { 5680e91a0c6SThomas Gleixner if (IS_ENABLED(CONFIG_DEBUG_KMAP_LOCAL_FORCE_MAP)) { 5690e91a0c6SThomas Gleixner /* This _should_ never happen! See above. */ 5700e91a0c6SThomas Gleixner WARN_ON_ONCE(1); 5710e91a0c6SThomas Gleixner return; 5720e91a0c6SThomas Gleixner } 5732a656cadSThomas Gleixner /* 5742a656cadSThomas Gleixner * Handle mappings which were obtained by kmap_high_get() 5752a656cadSThomas Gleixner * first as the virtual address of such mappings is below 5762a656cadSThomas Gleixner * PAGE_OFFSET. Warn for all other addresses which are in 5772a656cadSThomas Gleixner * the user space part of the virtual address space. 5782a656cadSThomas Gleixner */ 5792a656cadSThomas Gleixner if (!kmap_high_unmap_local(addr)) 580298fa1adSThomas Gleixner WARN_ON_ONCE(addr < PAGE_OFFSET); 581298fa1adSThomas Gleixner return; 582298fa1adSThomas Gleixner } 583298fa1adSThomas Gleixner 584298fa1adSThomas Gleixner preempt_disable(); 585298fa1adSThomas Gleixner idx = arch_kmap_local_unmap_idx(kmap_local_idx(), addr); 586298fa1adSThomas Gleixner WARN_ON_ONCE(addr != __fix_to_virt(FIX_KMAP_BEGIN + idx)); 587298fa1adSThomas Gleixner 588298fa1adSThomas Gleixner arch_kmap_local_pre_unmap(addr); 589298fa1adSThomas Gleixner pte_clear(&init_mm, addr, kmap_pte - idx); 590298fa1adSThomas Gleixner arch_kmap_local_post_unmap(addr); 5915fbda3ecSThomas Gleixner current->kmap_ctrl.pteval[kmap_local_idx()] = __pte(0); 592298fa1adSThomas Gleixner kmap_local_idx_pop(); 593298fa1adSThomas Gleixner preempt_enable(); 594f3ba3c71SThomas Gleixner migrate_enable(); 595298fa1adSThomas Gleixner } 596298fa1adSThomas Gleixner EXPORT_SYMBOL(kunmap_local_indexed); 5975fbda3ecSThomas Gleixner 5985fbda3ecSThomas Gleixner /* 5995fbda3ecSThomas Gleixner * Invoked before switch_to(). This is safe even when during or after 6005fbda3ecSThomas Gleixner * clearing the maps an interrupt which needs a kmap_local happens because 6015fbda3ecSThomas Gleixner * the task::kmap_ctrl.idx is not modified by the unmapping code so a 6025fbda3ecSThomas Gleixner * nested kmap_local will use the next unused index and restore the index 6035fbda3ecSThomas Gleixner * on unmap. The already cleared kmaps of the outgoing task are irrelevant 6045fbda3ecSThomas Gleixner * because the interrupt context does not know about them. The same applies 6055fbda3ecSThomas Gleixner * when scheduling back in for an interrupt which happens before the 6065fbda3ecSThomas Gleixner * restore is complete. 6075fbda3ecSThomas Gleixner */ 6085fbda3ecSThomas Gleixner void __kmap_local_sched_out(void) 6095fbda3ecSThomas Gleixner { 6105fbda3ecSThomas Gleixner struct task_struct *tsk = current; 6115fbda3ecSThomas Gleixner pte_t *kmap_pte = kmap_get_pte(); 6125fbda3ecSThomas Gleixner int i; 6135fbda3ecSThomas Gleixner 6145fbda3ecSThomas Gleixner /* Clear kmaps */ 6155fbda3ecSThomas Gleixner for (i = 0; i < tsk->kmap_ctrl.idx; i++) { 6165fbda3ecSThomas Gleixner pte_t pteval = tsk->kmap_ctrl.pteval[i]; 6175fbda3ecSThomas Gleixner unsigned long addr; 6185fbda3ecSThomas Gleixner int idx; 6195fbda3ecSThomas Gleixner 6205fbda3ecSThomas Gleixner /* With debug all even slots are unmapped and act as guard */ 621487cfadeSIra Weiny if (IS_ENABLED(CONFIG_DEBUG_KMAP_LOCAL) && !(i & 0x01)) { 6225fbda3ecSThomas Gleixner WARN_ON_ONCE(!pte_none(pteval)); 6235fbda3ecSThomas Gleixner continue; 6245fbda3ecSThomas Gleixner } 6255fbda3ecSThomas Gleixner if (WARN_ON_ONCE(pte_none(pteval))) 6265fbda3ecSThomas Gleixner continue; 6275fbda3ecSThomas Gleixner 6285fbda3ecSThomas Gleixner /* 6295fbda3ecSThomas Gleixner * This is a horrible hack for XTENSA to calculate the 6305fbda3ecSThomas Gleixner * coloured PTE index. Uses the PFN encoded into the pteval 6315fbda3ecSThomas Gleixner * and the map index calculation because the actual mapped 6325fbda3ecSThomas Gleixner * virtual address is not stored in task::kmap_ctrl. 6335fbda3ecSThomas Gleixner * For any sane architecture this is optimized out. 6345fbda3ecSThomas Gleixner */ 6355fbda3ecSThomas Gleixner idx = arch_kmap_local_map_idx(i, pte_pfn(pteval)); 6365fbda3ecSThomas Gleixner 6375fbda3ecSThomas Gleixner addr = __fix_to_virt(FIX_KMAP_BEGIN + idx); 6385fbda3ecSThomas Gleixner arch_kmap_local_pre_unmap(addr); 6395fbda3ecSThomas Gleixner pte_clear(&init_mm, addr, kmap_pte - idx); 6405fbda3ecSThomas Gleixner arch_kmap_local_post_unmap(addr); 6415fbda3ecSThomas Gleixner } 6425fbda3ecSThomas Gleixner } 6435fbda3ecSThomas Gleixner 6445fbda3ecSThomas Gleixner void __kmap_local_sched_in(void) 6455fbda3ecSThomas Gleixner { 6465fbda3ecSThomas Gleixner struct task_struct *tsk = current; 6475fbda3ecSThomas Gleixner pte_t *kmap_pte = kmap_get_pte(); 6485fbda3ecSThomas Gleixner int i; 6495fbda3ecSThomas Gleixner 6505fbda3ecSThomas Gleixner /* Restore kmaps */ 6515fbda3ecSThomas Gleixner for (i = 0; i < tsk->kmap_ctrl.idx; i++) { 6525fbda3ecSThomas Gleixner pte_t pteval = tsk->kmap_ctrl.pteval[i]; 6535fbda3ecSThomas Gleixner unsigned long addr; 6545fbda3ecSThomas Gleixner int idx; 6555fbda3ecSThomas Gleixner 6565fbda3ecSThomas Gleixner /* With debug all even slots are unmapped and act as guard */ 657487cfadeSIra Weiny if (IS_ENABLED(CONFIG_DEBUG_KMAP_LOCAL) && !(i & 0x01)) { 6585fbda3ecSThomas Gleixner WARN_ON_ONCE(!pte_none(pteval)); 6595fbda3ecSThomas Gleixner continue; 6605fbda3ecSThomas Gleixner } 6615fbda3ecSThomas Gleixner if (WARN_ON_ONCE(pte_none(pteval))) 6625fbda3ecSThomas Gleixner continue; 6635fbda3ecSThomas Gleixner 6645fbda3ecSThomas Gleixner /* See comment in __kmap_local_sched_out() */ 6655fbda3ecSThomas Gleixner idx = arch_kmap_local_map_idx(i, pte_pfn(pteval)); 6665fbda3ecSThomas Gleixner addr = __fix_to_virt(FIX_KMAP_BEGIN + idx); 6675fbda3ecSThomas Gleixner set_pte_at(&init_mm, addr, kmap_pte - idx, pteval); 6685fbda3ecSThomas Gleixner arch_kmap_local_post_map(addr, pteval); 6695fbda3ecSThomas Gleixner } 6705fbda3ecSThomas Gleixner } 6715fbda3ecSThomas Gleixner 6725fbda3ecSThomas Gleixner void kmap_local_fork(struct task_struct *tsk) 6735fbda3ecSThomas Gleixner { 6745fbda3ecSThomas Gleixner if (WARN_ON_ONCE(tsk->kmap_ctrl.idx)) 6755fbda3ecSThomas Gleixner memset(&tsk->kmap_ctrl, 0, sizeof(tsk->kmap_ctrl)); 6765fbda3ecSThomas Gleixner } 6775fbda3ecSThomas Gleixner 678298fa1adSThomas Gleixner #endif 679298fa1adSThomas Gleixner 6801da177e4SLinus Torvalds #if defined(HASHED_PAGE_VIRTUAL) 6811da177e4SLinus Torvalds 6821da177e4SLinus Torvalds #define PA_HASH_ORDER 7 6831da177e4SLinus Torvalds 6841da177e4SLinus Torvalds /* 6851da177e4SLinus Torvalds * Describes one page->virtual association 6861da177e4SLinus Torvalds */ 6871da177e4SLinus Torvalds struct page_address_map { 6881da177e4SLinus Torvalds struct page *page; 6891da177e4SLinus Torvalds void *virtual; 6901da177e4SLinus Torvalds struct list_head list; 6911da177e4SLinus Torvalds }; 6921da177e4SLinus Torvalds 693a354e2c8SJoonsoo Kim static struct page_address_map page_address_maps[LAST_PKMAP]; 6941da177e4SLinus Torvalds 6951da177e4SLinus Torvalds /* 6961da177e4SLinus Torvalds * Hash table bucket 6971da177e4SLinus Torvalds */ 6981da177e4SLinus Torvalds static struct page_address_slot { 6991da177e4SLinus Torvalds struct list_head lh; /* List of page_address_maps */ 7001da177e4SLinus Torvalds spinlock_t lock; /* Protect this bucket's list */ 7011da177e4SLinus Torvalds } ____cacheline_aligned_in_smp page_address_htable[1<<PA_HASH_ORDER]; 7021da177e4SLinus Torvalds 703f9918794SIan Campbell static struct page_address_slot *page_slot(const struct page *page) 7041da177e4SLinus Torvalds { 7051da177e4SLinus Torvalds return &page_address_htable[hash_ptr(page, PA_HASH_ORDER)]; 7061da177e4SLinus Torvalds } 7071da177e4SLinus Torvalds 70877f6078aSRandy Dunlap /** 70977f6078aSRandy Dunlap * page_address - get the mapped virtual address of a page 71077f6078aSRandy Dunlap * @page: &struct page to get the virtual address of 71177f6078aSRandy Dunlap * 71277f6078aSRandy Dunlap * Returns the page's virtual address. 71377f6078aSRandy Dunlap */ 714f9918794SIan Campbell void *page_address(const struct page *page) 7151da177e4SLinus Torvalds { 7161da177e4SLinus Torvalds unsigned long flags; 7171da177e4SLinus Torvalds void *ret; 7181da177e4SLinus Torvalds struct page_address_slot *pas; 7191da177e4SLinus Torvalds 7201da177e4SLinus Torvalds if (!PageHighMem(page)) 7211da177e4SLinus Torvalds return lowmem_page_address(page); 7221da177e4SLinus Torvalds 7231da177e4SLinus Torvalds pas = page_slot(page); 7241da177e4SLinus Torvalds ret = NULL; 7251da177e4SLinus Torvalds spin_lock_irqsave(&pas->lock, flags); 7261da177e4SLinus Torvalds if (!list_empty(&pas->lh)) { 7271da177e4SLinus Torvalds struct page_address_map *pam; 7281da177e4SLinus Torvalds 7291da177e4SLinus Torvalds list_for_each_entry(pam, &pas->lh, list) { 7301da177e4SLinus Torvalds if (pam->page == page) { 7311da177e4SLinus Torvalds ret = pam->virtual; 7321da177e4SLinus Torvalds goto done; 7331da177e4SLinus Torvalds } 7341da177e4SLinus Torvalds } 7351da177e4SLinus Torvalds } 7361da177e4SLinus Torvalds done: 7371da177e4SLinus Torvalds spin_unlock_irqrestore(&pas->lock, flags); 7381da177e4SLinus Torvalds return ret; 7391da177e4SLinus Torvalds } 7401da177e4SLinus Torvalds EXPORT_SYMBOL(page_address); 7411da177e4SLinus Torvalds 74277f6078aSRandy Dunlap /** 74377f6078aSRandy Dunlap * set_page_address - set a page's virtual address 74477f6078aSRandy Dunlap * @page: &struct page to set 74577f6078aSRandy Dunlap * @virtual: virtual address to use 74677f6078aSRandy Dunlap */ 7471da177e4SLinus Torvalds void set_page_address(struct page *page, void *virtual) 7481da177e4SLinus Torvalds { 7491da177e4SLinus Torvalds unsigned long flags; 7501da177e4SLinus Torvalds struct page_address_slot *pas; 7511da177e4SLinus Torvalds struct page_address_map *pam; 7521da177e4SLinus Torvalds 7531da177e4SLinus Torvalds BUG_ON(!PageHighMem(page)); 7541da177e4SLinus Torvalds 7551da177e4SLinus Torvalds pas = page_slot(page); 7561da177e4SLinus Torvalds if (virtual) { /* Add */ 757a354e2c8SJoonsoo Kim pam = &page_address_maps[PKMAP_NR((unsigned long)virtual)]; 7581da177e4SLinus Torvalds pam->page = page; 7591da177e4SLinus Torvalds pam->virtual = virtual; 7601da177e4SLinus Torvalds 7611da177e4SLinus Torvalds spin_lock_irqsave(&pas->lock, flags); 7621da177e4SLinus Torvalds list_add_tail(&pam->list, &pas->lh); 7631da177e4SLinus Torvalds spin_unlock_irqrestore(&pas->lock, flags); 7641da177e4SLinus Torvalds } else { /* Remove */ 7651da177e4SLinus Torvalds spin_lock_irqsave(&pas->lock, flags); 7661da177e4SLinus Torvalds list_for_each_entry(pam, &pas->lh, list) { 7671da177e4SLinus Torvalds if (pam->page == page) { 7681da177e4SLinus Torvalds list_del(&pam->list); 7691da177e4SLinus Torvalds spin_unlock_irqrestore(&pas->lock, flags); 7701da177e4SLinus Torvalds goto done; 7711da177e4SLinus Torvalds } 7721da177e4SLinus Torvalds } 7731da177e4SLinus Torvalds spin_unlock_irqrestore(&pas->lock, flags); 7741da177e4SLinus Torvalds } 7751da177e4SLinus Torvalds done: 7761da177e4SLinus Torvalds return; 7771da177e4SLinus Torvalds } 7781da177e4SLinus Torvalds 7791da177e4SLinus Torvalds void __init page_address_init(void) 7801da177e4SLinus Torvalds { 7811da177e4SLinus Torvalds int i; 7821da177e4SLinus Torvalds 7831da177e4SLinus Torvalds for (i = 0; i < ARRAY_SIZE(page_address_htable); i++) { 7841da177e4SLinus Torvalds INIT_LIST_HEAD(&page_address_htable[i].lh); 7851da177e4SLinus Torvalds spin_lock_init(&page_address_htable[i].lock); 7861da177e4SLinus Torvalds } 7871da177e4SLinus Torvalds } 7881da177e4SLinus Torvalds 789955cc774SIra Weiny #endif /* defined(HASHED_PAGE_VIRTUAL) */ 790