11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * High memory handling common code and variables. 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * (C) 1999 Andrea Arcangeli, SuSE GmbH, andrea@suse.de 51da177e4SLinus Torvalds * Gerhard Wichert, Siemens AG, Gerhard.Wichert@pdb.siemens.de 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * 81da177e4SLinus Torvalds * Redesigned the x86 32-bit VM architecture to deal with 91da177e4SLinus Torvalds * 64-bit physical space. With current x86 CPUs this 101da177e4SLinus Torvalds * means up to 64 Gigabytes physical RAM. 111da177e4SLinus Torvalds * 121da177e4SLinus Torvalds * Rewrote high memory support to move the page cache into 131da177e4SLinus Torvalds * high memory. Implemented permanent (schedulable) kmaps 141da177e4SLinus Torvalds * based on Linus' idea. 151da177e4SLinus Torvalds * 161da177e4SLinus Torvalds * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com> 171da177e4SLinus Torvalds */ 181da177e4SLinus Torvalds 191da177e4SLinus Torvalds #include <linux/mm.h> 20b95f1b31SPaul Gortmaker #include <linux/export.h> 211da177e4SLinus Torvalds #include <linux/swap.h> 221da177e4SLinus Torvalds #include <linux/bio.h> 231da177e4SLinus Torvalds #include <linux/pagemap.h> 241da177e4SLinus Torvalds #include <linux/mempool.h> 251da177e4SLinus Torvalds #include <linux/blkdev.h> 261da177e4SLinus Torvalds #include <linux/init.h> 271da177e4SLinus Torvalds #include <linux/hash.h> 281da177e4SLinus Torvalds #include <linux/highmem.h> 29eac79005SJason Wessel #include <linux/kgdb.h> 301da177e4SLinus Torvalds #include <asm/tlbflush.h> 311da177e4SLinus Torvalds 32a8e23a29SPeter Zijlstra 33a8e23a29SPeter Zijlstra #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32) 34a8e23a29SPeter Zijlstra DEFINE_PER_CPU(int, __kmap_atomic_idx); 35a8e23a29SPeter Zijlstra #endif 36a8e23a29SPeter Zijlstra 371da177e4SLinus Torvalds /* 381da177e4SLinus Torvalds * Virtual_count is not a pure "count". 391da177e4SLinus Torvalds * 0 means that it is not mapped, and has not been mapped 401da177e4SLinus Torvalds * since a TLB flush - it is usable. 411da177e4SLinus Torvalds * 1 means that there are no users, but it has been mapped 421da177e4SLinus Torvalds * since the last TLB flush - so we can't use it. 431da177e4SLinus Torvalds * n means that there are (n-1) current users of it. 441da177e4SLinus Torvalds */ 451da177e4SLinus Torvalds #ifdef CONFIG_HIGHMEM 46260b2367SAl Viro 47c1f60a5aSChristoph Lameter unsigned long totalhigh_pages __read_mostly; 48db7a94d6SDavid S. Miller EXPORT_SYMBOL(totalhigh_pages); 49c1f60a5aSChristoph Lameter 503e4d3af5SPeter Zijlstra 513e4d3af5SPeter Zijlstra EXPORT_PER_CPU_SYMBOL(__kmap_atomic_idx); 523e4d3af5SPeter Zijlstra 53c1f60a5aSChristoph Lameter unsigned int nr_free_highpages (void) 54c1f60a5aSChristoph Lameter { 55c1f60a5aSChristoph Lameter pg_data_t *pgdat; 56c1f60a5aSChristoph Lameter unsigned int pages = 0; 57c1f60a5aSChristoph Lameter 582a1e274aSMel Gorman for_each_online_pgdat(pgdat) { 59d23ad423SChristoph Lameter pages += zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM], 60d23ad423SChristoph Lameter NR_FREE_PAGES); 612a1e274aSMel Gorman if (zone_movable_is_highmem()) 622a1e274aSMel Gorman pages += zone_page_state( 632a1e274aSMel Gorman &pgdat->node_zones[ZONE_MOVABLE], 642a1e274aSMel Gorman NR_FREE_PAGES); 652a1e274aSMel Gorman } 66c1f60a5aSChristoph Lameter 67c1f60a5aSChristoph Lameter return pages; 68c1f60a5aSChristoph Lameter } 69c1f60a5aSChristoph Lameter 701da177e4SLinus Torvalds static int pkmap_count[LAST_PKMAP]; 711da177e4SLinus Torvalds static unsigned int last_pkmap_nr; 721da177e4SLinus Torvalds static __cacheline_aligned_in_smp DEFINE_SPINLOCK(kmap_lock); 731da177e4SLinus Torvalds 741da177e4SLinus Torvalds pte_t * pkmap_page_table; 751da177e4SLinus Torvalds 761da177e4SLinus Torvalds static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait); 771da177e4SLinus Torvalds 783297e760SNicolas Pitre /* 793297e760SNicolas Pitre * Most architectures have no use for kmap_high_get(), so let's abstract 803297e760SNicolas Pitre * the disabling of IRQ out of the locking in that case to save on a 813297e760SNicolas Pitre * potential useless overhead. 823297e760SNicolas Pitre */ 833297e760SNicolas Pitre #ifdef ARCH_NEEDS_KMAP_HIGH_GET 843297e760SNicolas Pitre #define lock_kmap() spin_lock_irq(&kmap_lock) 853297e760SNicolas Pitre #define unlock_kmap() spin_unlock_irq(&kmap_lock) 863297e760SNicolas Pitre #define lock_kmap_any(flags) spin_lock_irqsave(&kmap_lock, flags) 873297e760SNicolas Pitre #define unlock_kmap_any(flags) spin_unlock_irqrestore(&kmap_lock, flags) 883297e760SNicolas Pitre #else 893297e760SNicolas Pitre #define lock_kmap() spin_lock(&kmap_lock) 903297e760SNicolas Pitre #define unlock_kmap() spin_unlock(&kmap_lock) 913297e760SNicolas Pitre #define lock_kmap_any(flags) \ 923297e760SNicolas Pitre do { spin_lock(&kmap_lock); (void)(flags); } while (0) 933297e760SNicolas Pitre #define unlock_kmap_any(flags) \ 943297e760SNicolas Pitre do { spin_unlock(&kmap_lock); (void)(flags); } while (0) 953297e760SNicolas Pitre #endif 963297e760SNicolas Pitre 975a178119SMel Gorman struct page *kmap_to_page(void *vaddr) 985a178119SMel Gorman { 995a178119SMel Gorman unsigned long addr = (unsigned long)vaddr; 1005a178119SMel Gorman 101498c2280SWill Deacon if (addr >= PKMAP_ADDR(0) && addr < PKMAP_ADDR(LAST_PKMAP)) { 1024de22c05SJoonsoo Kim int i = PKMAP_NR(addr); 1035a178119SMel Gorman return pte_page(pkmap_page_table[i]); 1045a178119SMel Gorman } 1055a178119SMel Gorman 1065a178119SMel Gorman return virt_to_page(addr); 1075a178119SMel Gorman } 1085a178119SMel Gorman 1091da177e4SLinus Torvalds static void flush_all_zero_pkmaps(void) 1101da177e4SLinus Torvalds { 1111da177e4SLinus Torvalds int i; 1125843d9a4SNick Piggin int need_flush = 0; 1131da177e4SLinus Torvalds 1141da177e4SLinus Torvalds flush_cache_kmaps(); 1151da177e4SLinus Torvalds 1161da177e4SLinus Torvalds for (i = 0; i < LAST_PKMAP; i++) { 1171da177e4SLinus Torvalds struct page *page; 1181da177e4SLinus Torvalds 1191da177e4SLinus Torvalds /* 1201da177e4SLinus Torvalds * zero means we don't have anything to do, 1211da177e4SLinus Torvalds * >1 means that it is still in use. Only 1221da177e4SLinus Torvalds * a count of 1 means that it is free but 1231da177e4SLinus Torvalds * needs to be unmapped 1241da177e4SLinus Torvalds */ 1251da177e4SLinus Torvalds if (pkmap_count[i] != 1) 1261da177e4SLinus Torvalds continue; 1271da177e4SLinus Torvalds pkmap_count[i] = 0; 1281da177e4SLinus Torvalds 1291da177e4SLinus Torvalds /* sanity check */ 13075babcacSEric Sesterhenn BUG_ON(pte_none(pkmap_page_table[i])); 1311da177e4SLinus Torvalds 1321da177e4SLinus Torvalds /* 1331da177e4SLinus Torvalds * Don't need an atomic fetch-and-clear op here; 1341da177e4SLinus Torvalds * no-one has the page mapped, and cannot get at 1351da177e4SLinus Torvalds * its virtual address (and hence PTE) without first 1361da177e4SLinus Torvalds * getting the kmap_lock (which is held here). 1371da177e4SLinus Torvalds * So no dangers, even with speculative execution. 1381da177e4SLinus Torvalds */ 1391da177e4SLinus Torvalds page = pte_page(pkmap_page_table[i]); 140*eb2db439SJoonsoo Kim pte_clear(&init_mm, PKMAP_ADDR(i), &pkmap_page_table[i]); 1411da177e4SLinus Torvalds 1421da177e4SLinus Torvalds set_page_address(page, NULL); 1435843d9a4SNick Piggin need_flush = 1; 1441da177e4SLinus Torvalds } 1455843d9a4SNick Piggin if (need_flush) 1461da177e4SLinus Torvalds flush_tlb_kernel_range(PKMAP_ADDR(0), PKMAP_ADDR(LAST_PKMAP)); 1471da177e4SLinus Torvalds } 1481da177e4SLinus Torvalds 14977f6078aSRandy Dunlap /** 15077f6078aSRandy Dunlap * kmap_flush_unused - flush all unused kmap mappings in order to remove stray mappings 15177f6078aSRandy Dunlap */ 152ce6234b5SJeremy Fitzhardinge void kmap_flush_unused(void) 153ce6234b5SJeremy Fitzhardinge { 1543297e760SNicolas Pitre lock_kmap(); 155ce6234b5SJeremy Fitzhardinge flush_all_zero_pkmaps(); 1563297e760SNicolas Pitre unlock_kmap(); 157ce6234b5SJeremy Fitzhardinge } 158ce6234b5SJeremy Fitzhardinge 1591da177e4SLinus Torvalds static inline unsigned long map_new_virtual(struct page *page) 1601da177e4SLinus Torvalds { 1611da177e4SLinus Torvalds unsigned long vaddr; 1621da177e4SLinus Torvalds int count; 1631da177e4SLinus Torvalds 1641da177e4SLinus Torvalds start: 1651da177e4SLinus Torvalds count = LAST_PKMAP; 1661da177e4SLinus Torvalds /* Find an empty entry */ 1671da177e4SLinus Torvalds for (;;) { 1681da177e4SLinus Torvalds last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK; 1691da177e4SLinus Torvalds if (!last_pkmap_nr) { 1701da177e4SLinus Torvalds flush_all_zero_pkmaps(); 1711da177e4SLinus Torvalds count = LAST_PKMAP; 1721da177e4SLinus Torvalds } 1731da177e4SLinus Torvalds if (!pkmap_count[last_pkmap_nr]) 1741da177e4SLinus Torvalds break; /* Found a usable entry */ 1751da177e4SLinus Torvalds if (--count) 1761da177e4SLinus Torvalds continue; 1771da177e4SLinus Torvalds 1781da177e4SLinus Torvalds /* 1791da177e4SLinus Torvalds * Sleep for somebody else to unmap their entries 1801da177e4SLinus Torvalds */ 1811da177e4SLinus Torvalds { 1821da177e4SLinus Torvalds DECLARE_WAITQUEUE(wait, current); 1831da177e4SLinus Torvalds 1841da177e4SLinus Torvalds __set_current_state(TASK_UNINTERRUPTIBLE); 1851da177e4SLinus Torvalds add_wait_queue(&pkmap_map_wait, &wait); 1863297e760SNicolas Pitre unlock_kmap(); 1871da177e4SLinus Torvalds schedule(); 1881da177e4SLinus Torvalds remove_wait_queue(&pkmap_map_wait, &wait); 1893297e760SNicolas Pitre lock_kmap(); 1901da177e4SLinus Torvalds 1911da177e4SLinus Torvalds /* Somebody else might have mapped it while we slept */ 1921da177e4SLinus Torvalds if (page_address(page)) 1931da177e4SLinus Torvalds return (unsigned long)page_address(page); 1941da177e4SLinus Torvalds 1951da177e4SLinus Torvalds /* Re-start */ 1961da177e4SLinus Torvalds goto start; 1971da177e4SLinus Torvalds } 1981da177e4SLinus Torvalds } 1991da177e4SLinus Torvalds vaddr = PKMAP_ADDR(last_pkmap_nr); 2001da177e4SLinus Torvalds set_pte_at(&init_mm, vaddr, 2011da177e4SLinus Torvalds &(pkmap_page_table[last_pkmap_nr]), mk_pte(page, kmap_prot)); 2021da177e4SLinus Torvalds 2031da177e4SLinus Torvalds pkmap_count[last_pkmap_nr] = 1; 2041da177e4SLinus Torvalds set_page_address(page, (void *)vaddr); 2051da177e4SLinus Torvalds 2061da177e4SLinus Torvalds return vaddr; 2071da177e4SLinus Torvalds } 2081da177e4SLinus Torvalds 20977f6078aSRandy Dunlap /** 21077f6078aSRandy Dunlap * kmap_high - map a highmem page into memory 21177f6078aSRandy Dunlap * @page: &struct page to map 21277f6078aSRandy Dunlap * 21377f6078aSRandy Dunlap * Returns the page's virtual memory address. 21477f6078aSRandy Dunlap * 21577f6078aSRandy Dunlap * We cannot call this from interrupts, as it may block. 21677f6078aSRandy Dunlap */ 217920c7a5dSHarvey Harrison void *kmap_high(struct page *page) 2181da177e4SLinus Torvalds { 2191da177e4SLinus Torvalds unsigned long vaddr; 2201da177e4SLinus Torvalds 2211da177e4SLinus Torvalds /* 2221da177e4SLinus Torvalds * For highmem pages, we can't trust "virtual" until 2231da177e4SLinus Torvalds * after we have the lock. 2241da177e4SLinus Torvalds */ 2253297e760SNicolas Pitre lock_kmap(); 2261da177e4SLinus Torvalds vaddr = (unsigned long)page_address(page); 2271da177e4SLinus Torvalds if (!vaddr) 2281da177e4SLinus Torvalds vaddr = map_new_virtual(page); 2291da177e4SLinus Torvalds pkmap_count[PKMAP_NR(vaddr)]++; 23075babcacSEric Sesterhenn BUG_ON(pkmap_count[PKMAP_NR(vaddr)] < 2); 2313297e760SNicolas Pitre unlock_kmap(); 2321da177e4SLinus Torvalds return (void*) vaddr; 2331da177e4SLinus Torvalds } 2341da177e4SLinus Torvalds 2351da177e4SLinus Torvalds EXPORT_SYMBOL(kmap_high); 2361da177e4SLinus Torvalds 2373297e760SNicolas Pitre #ifdef ARCH_NEEDS_KMAP_HIGH_GET 2383297e760SNicolas Pitre /** 2393297e760SNicolas Pitre * kmap_high_get - pin a highmem page into memory 2403297e760SNicolas Pitre * @page: &struct page to pin 2413297e760SNicolas Pitre * 2423297e760SNicolas Pitre * Returns the page's current virtual memory address, or NULL if no mapping 2435e39df56SUwe Kleine-König * exists. If and only if a non null address is returned then a 2443297e760SNicolas Pitre * matching call to kunmap_high() is necessary. 2453297e760SNicolas Pitre * 2463297e760SNicolas Pitre * This can be called from any context. 2473297e760SNicolas Pitre */ 2483297e760SNicolas Pitre void *kmap_high_get(struct page *page) 2493297e760SNicolas Pitre { 2503297e760SNicolas Pitre unsigned long vaddr, flags; 2513297e760SNicolas Pitre 2523297e760SNicolas Pitre lock_kmap_any(flags); 2533297e760SNicolas Pitre vaddr = (unsigned long)page_address(page); 2543297e760SNicolas Pitre if (vaddr) { 2553297e760SNicolas Pitre BUG_ON(pkmap_count[PKMAP_NR(vaddr)] < 1); 2563297e760SNicolas Pitre pkmap_count[PKMAP_NR(vaddr)]++; 2573297e760SNicolas Pitre } 2583297e760SNicolas Pitre unlock_kmap_any(flags); 2593297e760SNicolas Pitre return (void*) vaddr; 2603297e760SNicolas Pitre } 2613297e760SNicolas Pitre #endif 2623297e760SNicolas Pitre 26377f6078aSRandy Dunlap /** 2644e9dc5dfSLi Haifeng * kunmap_high - unmap a highmem page into memory 26577f6078aSRandy Dunlap * @page: &struct page to unmap 2663297e760SNicolas Pitre * 2673297e760SNicolas Pitre * If ARCH_NEEDS_KMAP_HIGH_GET is not defined then this may be called 2683297e760SNicolas Pitre * only from user context. 26977f6078aSRandy Dunlap */ 270920c7a5dSHarvey Harrison void kunmap_high(struct page *page) 2711da177e4SLinus Torvalds { 2721da177e4SLinus Torvalds unsigned long vaddr; 2731da177e4SLinus Torvalds unsigned long nr; 2743297e760SNicolas Pitre unsigned long flags; 2751da177e4SLinus Torvalds int need_wakeup; 2761da177e4SLinus Torvalds 2773297e760SNicolas Pitre lock_kmap_any(flags); 2781da177e4SLinus Torvalds vaddr = (unsigned long)page_address(page); 27975babcacSEric Sesterhenn BUG_ON(!vaddr); 2801da177e4SLinus Torvalds nr = PKMAP_NR(vaddr); 2811da177e4SLinus Torvalds 2821da177e4SLinus Torvalds /* 2831da177e4SLinus Torvalds * A count must never go down to zero 2841da177e4SLinus Torvalds * without a TLB flush! 2851da177e4SLinus Torvalds */ 2861da177e4SLinus Torvalds need_wakeup = 0; 2871da177e4SLinus Torvalds switch (--pkmap_count[nr]) { 2881da177e4SLinus Torvalds case 0: 2891da177e4SLinus Torvalds BUG(); 2901da177e4SLinus Torvalds case 1: 2911da177e4SLinus Torvalds /* 2921da177e4SLinus Torvalds * Avoid an unnecessary wake_up() function call. 2931da177e4SLinus Torvalds * The common case is pkmap_count[] == 1, but 2941da177e4SLinus Torvalds * no waiters. 2951da177e4SLinus Torvalds * The tasks queued in the wait-queue are guarded 2961da177e4SLinus Torvalds * by both the lock in the wait-queue-head and by 2971da177e4SLinus Torvalds * the kmap_lock. As the kmap_lock is held here, 2981da177e4SLinus Torvalds * no need for the wait-queue-head's lock. Simply 2991da177e4SLinus Torvalds * test if the queue is empty. 3001da177e4SLinus Torvalds */ 3011da177e4SLinus Torvalds need_wakeup = waitqueue_active(&pkmap_map_wait); 3021da177e4SLinus Torvalds } 3033297e760SNicolas Pitre unlock_kmap_any(flags); 3041da177e4SLinus Torvalds 3051da177e4SLinus Torvalds /* do wake-up, if needed, race-free outside of the spin lock */ 3061da177e4SLinus Torvalds if (need_wakeup) 3071da177e4SLinus Torvalds wake_up(&pkmap_map_wait); 3081da177e4SLinus Torvalds } 3091da177e4SLinus Torvalds 3101da177e4SLinus Torvalds EXPORT_SYMBOL(kunmap_high); 3111da177e4SLinus Torvalds #endif 3121da177e4SLinus Torvalds 3131da177e4SLinus Torvalds #if defined(HASHED_PAGE_VIRTUAL) 3141da177e4SLinus Torvalds 3151da177e4SLinus Torvalds #define PA_HASH_ORDER 7 3161da177e4SLinus Torvalds 3171da177e4SLinus Torvalds /* 3181da177e4SLinus Torvalds * Describes one page->virtual association 3191da177e4SLinus Torvalds */ 3201da177e4SLinus Torvalds struct page_address_map { 3211da177e4SLinus Torvalds struct page *page; 3221da177e4SLinus Torvalds void *virtual; 3231da177e4SLinus Torvalds struct list_head list; 3241da177e4SLinus Torvalds }; 3251da177e4SLinus Torvalds 326a354e2c8SJoonsoo Kim static struct page_address_map page_address_maps[LAST_PKMAP]; 3271da177e4SLinus Torvalds 3281da177e4SLinus Torvalds /* 3291da177e4SLinus Torvalds * Hash table bucket 3301da177e4SLinus Torvalds */ 3311da177e4SLinus Torvalds static struct page_address_slot { 3321da177e4SLinus Torvalds struct list_head lh; /* List of page_address_maps */ 3331da177e4SLinus Torvalds spinlock_t lock; /* Protect this bucket's list */ 3341da177e4SLinus Torvalds } ____cacheline_aligned_in_smp page_address_htable[1<<PA_HASH_ORDER]; 3351da177e4SLinus Torvalds 336f9918794SIan Campbell static struct page_address_slot *page_slot(const struct page *page) 3371da177e4SLinus Torvalds { 3381da177e4SLinus Torvalds return &page_address_htable[hash_ptr(page, PA_HASH_ORDER)]; 3391da177e4SLinus Torvalds } 3401da177e4SLinus Torvalds 34177f6078aSRandy Dunlap /** 34277f6078aSRandy Dunlap * page_address - get the mapped virtual address of a page 34377f6078aSRandy Dunlap * @page: &struct page to get the virtual address of 34477f6078aSRandy Dunlap * 34577f6078aSRandy Dunlap * Returns the page's virtual address. 34677f6078aSRandy Dunlap */ 347f9918794SIan Campbell void *page_address(const struct page *page) 3481da177e4SLinus Torvalds { 3491da177e4SLinus Torvalds unsigned long flags; 3501da177e4SLinus Torvalds void *ret; 3511da177e4SLinus Torvalds struct page_address_slot *pas; 3521da177e4SLinus Torvalds 3531da177e4SLinus Torvalds if (!PageHighMem(page)) 3541da177e4SLinus Torvalds return lowmem_page_address(page); 3551da177e4SLinus Torvalds 3561da177e4SLinus Torvalds pas = page_slot(page); 3571da177e4SLinus Torvalds ret = NULL; 3581da177e4SLinus Torvalds spin_lock_irqsave(&pas->lock, flags); 3591da177e4SLinus Torvalds if (!list_empty(&pas->lh)) { 3601da177e4SLinus Torvalds struct page_address_map *pam; 3611da177e4SLinus Torvalds 3621da177e4SLinus Torvalds list_for_each_entry(pam, &pas->lh, list) { 3631da177e4SLinus Torvalds if (pam->page == page) { 3641da177e4SLinus Torvalds ret = pam->virtual; 3651da177e4SLinus Torvalds goto done; 3661da177e4SLinus Torvalds } 3671da177e4SLinus Torvalds } 3681da177e4SLinus Torvalds } 3691da177e4SLinus Torvalds done: 3701da177e4SLinus Torvalds spin_unlock_irqrestore(&pas->lock, flags); 3711da177e4SLinus Torvalds return ret; 3721da177e4SLinus Torvalds } 3731da177e4SLinus Torvalds 3741da177e4SLinus Torvalds EXPORT_SYMBOL(page_address); 3751da177e4SLinus Torvalds 37677f6078aSRandy Dunlap /** 37777f6078aSRandy Dunlap * set_page_address - set a page's virtual address 37877f6078aSRandy Dunlap * @page: &struct page to set 37977f6078aSRandy Dunlap * @virtual: virtual address to use 38077f6078aSRandy Dunlap */ 3811da177e4SLinus Torvalds void set_page_address(struct page *page, void *virtual) 3821da177e4SLinus Torvalds { 3831da177e4SLinus Torvalds unsigned long flags; 3841da177e4SLinus Torvalds struct page_address_slot *pas; 3851da177e4SLinus Torvalds struct page_address_map *pam; 3861da177e4SLinus Torvalds 3871da177e4SLinus Torvalds BUG_ON(!PageHighMem(page)); 3881da177e4SLinus Torvalds 3891da177e4SLinus Torvalds pas = page_slot(page); 3901da177e4SLinus Torvalds if (virtual) { /* Add */ 391a354e2c8SJoonsoo Kim pam = &page_address_maps[PKMAP_NR((unsigned long)virtual)]; 3921da177e4SLinus Torvalds pam->page = page; 3931da177e4SLinus Torvalds pam->virtual = virtual; 3941da177e4SLinus Torvalds 3951da177e4SLinus Torvalds spin_lock_irqsave(&pas->lock, flags); 3961da177e4SLinus Torvalds list_add_tail(&pam->list, &pas->lh); 3971da177e4SLinus Torvalds spin_unlock_irqrestore(&pas->lock, flags); 3981da177e4SLinus Torvalds } else { /* Remove */ 3991da177e4SLinus Torvalds spin_lock_irqsave(&pas->lock, flags); 4001da177e4SLinus Torvalds list_for_each_entry(pam, &pas->lh, list) { 4011da177e4SLinus Torvalds if (pam->page == page) { 4021da177e4SLinus Torvalds list_del(&pam->list); 4031da177e4SLinus Torvalds spin_unlock_irqrestore(&pas->lock, flags); 4041da177e4SLinus Torvalds goto done; 4051da177e4SLinus Torvalds } 4061da177e4SLinus Torvalds } 4071da177e4SLinus Torvalds spin_unlock_irqrestore(&pas->lock, flags); 4081da177e4SLinus Torvalds } 4091da177e4SLinus Torvalds done: 4101da177e4SLinus Torvalds return; 4111da177e4SLinus Torvalds } 4121da177e4SLinus Torvalds 4131da177e4SLinus Torvalds void __init page_address_init(void) 4141da177e4SLinus Torvalds { 4151da177e4SLinus Torvalds int i; 4161da177e4SLinus Torvalds 4171da177e4SLinus Torvalds for (i = 0; i < ARRAY_SIZE(page_address_htable); i++) { 4181da177e4SLinus Torvalds INIT_LIST_HEAD(&page_address_htable[i].lh); 4191da177e4SLinus Torvalds spin_lock_init(&page_address_htable[i].lock); 4201da177e4SLinus Torvalds } 4211da177e4SLinus Torvalds } 4221da177e4SLinus Torvalds 4231da177e4SLinus Torvalds #endif /* defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL) */ 424