11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * High memory handling common code and variables. 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * (C) 1999 Andrea Arcangeli, SuSE GmbH, andrea@suse.de 51da177e4SLinus Torvalds * Gerhard Wichert, Siemens AG, Gerhard.Wichert@pdb.siemens.de 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * 81da177e4SLinus Torvalds * Redesigned the x86 32-bit VM architecture to deal with 91da177e4SLinus Torvalds * 64-bit physical space. With current x86 CPUs this 101da177e4SLinus Torvalds * means up to 64 Gigabytes physical RAM. 111da177e4SLinus Torvalds * 121da177e4SLinus Torvalds * Rewrote high memory support to move the page cache into 131da177e4SLinus Torvalds * high memory. Implemented permanent (schedulable) kmaps 141da177e4SLinus Torvalds * based on Linus' idea. 151da177e4SLinus Torvalds * 161da177e4SLinus Torvalds * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com> 171da177e4SLinus Torvalds */ 181da177e4SLinus Torvalds 191da177e4SLinus Torvalds #include <linux/mm.h> 20*b95f1b31SPaul Gortmaker #include <linux/export.h> 211da177e4SLinus Torvalds #include <linux/swap.h> 221da177e4SLinus Torvalds #include <linux/bio.h> 231da177e4SLinus Torvalds #include <linux/pagemap.h> 241da177e4SLinus Torvalds #include <linux/mempool.h> 251da177e4SLinus Torvalds #include <linux/blkdev.h> 261da177e4SLinus Torvalds #include <linux/init.h> 271da177e4SLinus Torvalds #include <linux/hash.h> 281da177e4SLinus Torvalds #include <linux/highmem.h> 29eac79005SJason Wessel #include <linux/kgdb.h> 301da177e4SLinus Torvalds #include <asm/tlbflush.h> 311da177e4SLinus Torvalds 32a8e23a29SPeter Zijlstra 33a8e23a29SPeter Zijlstra #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32) 34a8e23a29SPeter Zijlstra DEFINE_PER_CPU(int, __kmap_atomic_idx); 35a8e23a29SPeter Zijlstra #endif 36a8e23a29SPeter Zijlstra 371da177e4SLinus Torvalds /* 381da177e4SLinus Torvalds * Virtual_count is not a pure "count". 391da177e4SLinus Torvalds * 0 means that it is not mapped, and has not been mapped 401da177e4SLinus Torvalds * since a TLB flush - it is usable. 411da177e4SLinus Torvalds * 1 means that there are no users, but it has been mapped 421da177e4SLinus Torvalds * since the last TLB flush - so we can't use it. 431da177e4SLinus Torvalds * n means that there are (n-1) current users of it. 441da177e4SLinus Torvalds */ 451da177e4SLinus Torvalds #ifdef CONFIG_HIGHMEM 46260b2367SAl Viro 47c1f60a5aSChristoph Lameter unsigned long totalhigh_pages __read_mostly; 48db7a94d6SDavid S. Miller EXPORT_SYMBOL(totalhigh_pages); 49c1f60a5aSChristoph Lameter 503e4d3af5SPeter Zijlstra 513e4d3af5SPeter Zijlstra EXPORT_PER_CPU_SYMBOL(__kmap_atomic_idx); 523e4d3af5SPeter Zijlstra 53c1f60a5aSChristoph Lameter unsigned int nr_free_highpages (void) 54c1f60a5aSChristoph Lameter { 55c1f60a5aSChristoph Lameter pg_data_t *pgdat; 56c1f60a5aSChristoph Lameter unsigned int pages = 0; 57c1f60a5aSChristoph Lameter 582a1e274aSMel Gorman for_each_online_pgdat(pgdat) { 59d23ad423SChristoph Lameter pages += zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM], 60d23ad423SChristoph Lameter NR_FREE_PAGES); 612a1e274aSMel Gorman if (zone_movable_is_highmem()) 622a1e274aSMel Gorman pages += zone_page_state( 632a1e274aSMel Gorman &pgdat->node_zones[ZONE_MOVABLE], 642a1e274aSMel Gorman NR_FREE_PAGES); 652a1e274aSMel Gorman } 66c1f60a5aSChristoph Lameter 67c1f60a5aSChristoph Lameter return pages; 68c1f60a5aSChristoph Lameter } 69c1f60a5aSChristoph Lameter 701da177e4SLinus Torvalds static int pkmap_count[LAST_PKMAP]; 711da177e4SLinus Torvalds static unsigned int last_pkmap_nr; 721da177e4SLinus Torvalds static __cacheline_aligned_in_smp DEFINE_SPINLOCK(kmap_lock); 731da177e4SLinus Torvalds 741da177e4SLinus Torvalds pte_t * pkmap_page_table; 751da177e4SLinus Torvalds 761da177e4SLinus Torvalds static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait); 771da177e4SLinus Torvalds 783297e760SNicolas Pitre /* 793297e760SNicolas Pitre * Most architectures have no use for kmap_high_get(), so let's abstract 803297e760SNicolas Pitre * the disabling of IRQ out of the locking in that case to save on a 813297e760SNicolas Pitre * potential useless overhead. 823297e760SNicolas Pitre */ 833297e760SNicolas Pitre #ifdef ARCH_NEEDS_KMAP_HIGH_GET 843297e760SNicolas Pitre #define lock_kmap() spin_lock_irq(&kmap_lock) 853297e760SNicolas Pitre #define unlock_kmap() spin_unlock_irq(&kmap_lock) 863297e760SNicolas Pitre #define lock_kmap_any(flags) spin_lock_irqsave(&kmap_lock, flags) 873297e760SNicolas Pitre #define unlock_kmap_any(flags) spin_unlock_irqrestore(&kmap_lock, flags) 883297e760SNicolas Pitre #else 893297e760SNicolas Pitre #define lock_kmap() spin_lock(&kmap_lock) 903297e760SNicolas Pitre #define unlock_kmap() spin_unlock(&kmap_lock) 913297e760SNicolas Pitre #define lock_kmap_any(flags) \ 923297e760SNicolas Pitre do { spin_lock(&kmap_lock); (void)(flags); } while (0) 933297e760SNicolas Pitre #define unlock_kmap_any(flags) \ 943297e760SNicolas Pitre do { spin_unlock(&kmap_lock); (void)(flags); } while (0) 953297e760SNicolas Pitre #endif 963297e760SNicolas Pitre 971da177e4SLinus Torvalds static void flush_all_zero_pkmaps(void) 981da177e4SLinus Torvalds { 991da177e4SLinus Torvalds int i; 1005843d9a4SNick Piggin int need_flush = 0; 1011da177e4SLinus Torvalds 1021da177e4SLinus Torvalds flush_cache_kmaps(); 1031da177e4SLinus Torvalds 1041da177e4SLinus Torvalds for (i = 0; i < LAST_PKMAP; i++) { 1051da177e4SLinus Torvalds struct page *page; 1061da177e4SLinus Torvalds 1071da177e4SLinus Torvalds /* 1081da177e4SLinus Torvalds * zero means we don't have anything to do, 1091da177e4SLinus Torvalds * >1 means that it is still in use. Only 1101da177e4SLinus Torvalds * a count of 1 means that it is free but 1111da177e4SLinus Torvalds * needs to be unmapped 1121da177e4SLinus Torvalds */ 1131da177e4SLinus Torvalds if (pkmap_count[i] != 1) 1141da177e4SLinus Torvalds continue; 1151da177e4SLinus Torvalds pkmap_count[i] = 0; 1161da177e4SLinus Torvalds 1171da177e4SLinus Torvalds /* sanity check */ 11875babcacSEric Sesterhenn BUG_ON(pte_none(pkmap_page_table[i])); 1191da177e4SLinus Torvalds 1201da177e4SLinus Torvalds /* 1211da177e4SLinus Torvalds * Don't need an atomic fetch-and-clear op here; 1221da177e4SLinus Torvalds * no-one has the page mapped, and cannot get at 1231da177e4SLinus Torvalds * its virtual address (and hence PTE) without first 1241da177e4SLinus Torvalds * getting the kmap_lock (which is held here). 1251da177e4SLinus Torvalds * So no dangers, even with speculative execution. 1261da177e4SLinus Torvalds */ 1271da177e4SLinus Torvalds page = pte_page(pkmap_page_table[i]); 1281da177e4SLinus Torvalds pte_clear(&init_mm, (unsigned long)page_address(page), 1291da177e4SLinus Torvalds &pkmap_page_table[i]); 1301da177e4SLinus Torvalds 1311da177e4SLinus Torvalds set_page_address(page, NULL); 1325843d9a4SNick Piggin need_flush = 1; 1331da177e4SLinus Torvalds } 1345843d9a4SNick Piggin if (need_flush) 1351da177e4SLinus Torvalds flush_tlb_kernel_range(PKMAP_ADDR(0), PKMAP_ADDR(LAST_PKMAP)); 1361da177e4SLinus Torvalds } 1371da177e4SLinus Torvalds 13877f6078aSRandy Dunlap /** 13977f6078aSRandy Dunlap * kmap_flush_unused - flush all unused kmap mappings in order to remove stray mappings 14077f6078aSRandy Dunlap */ 141ce6234b5SJeremy Fitzhardinge void kmap_flush_unused(void) 142ce6234b5SJeremy Fitzhardinge { 1433297e760SNicolas Pitre lock_kmap(); 144ce6234b5SJeremy Fitzhardinge flush_all_zero_pkmaps(); 1453297e760SNicolas Pitre unlock_kmap(); 146ce6234b5SJeremy Fitzhardinge } 147ce6234b5SJeremy Fitzhardinge 1481da177e4SLinus Torvalds static inline unsigned long map_new_virtual(struct page *page) 1491da177e4SLinus Torvalds { 1501da177e4SLinus Torvalds unsigned long vaddr; 1511da177e4SLinus Torvalds int count; 1521da177e4SLinus Torvalds 1531da177e4SLinus Torvalds start: 1541da177e4SLinus Torvalds count = LAST_PKMAP; 1551da177e4SLinus Torvalds /* Find an empty entry */ 1561da177e4SLinus Torvalds for (;;) { 1571da177e4SLinus Torvalds last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK; 1581da177e4SLinus Torvalds if (!last_pkmap_nr) { 1591da177e4SLinus Torvalds flush_all_zero_pkmaps(); 1601da177e4SLinus Torvalds count = LAST_PKMAP; 1611da177e4SLinus Torvalds } 1621da177e4SLinus Torvalds if (!pkmap_count[last_pkmap_nr]) 1631da177e4SLinus Torvalds break; /* Found a usable entry */ 1641da177e4SLinus Torvalds if (--count) 1651da177e4SLinus Torvalds continue; 1661da177e4SLinus Torvalds 1671da177e4SLinus Torvalds /* 1681da177e4SLinus Torvalds * Sleep for somebody else to unmap their entries 1691da177e4SLinus Torvalds */ 1701da177e4SLinus Torvalds { 1711da177e4SLinus Torvalds DECLARE_WAITQUEUE(wait, current); 1721da177e4SLinus Torvalds 1731da177e4SLinus Torvalds __set_current_state(TASK_UNINTERRUPTIBLE); 1741da177e4SLinus Torvalds add_wait_queue(&pkmap_map_wait, &wait); 1753297e760SNicolas Pitre unlock_kmap(); 1761da177e4SLinus Torvalds schedule(); 1771da177e4SLinus Torvalds remove_wait_queue(&pkmap_map_wait, &wait); 1783297e760SNicolas Pitre lock_kmap(); 1791da177e4SLinus Torvalds 1801da177e4SLinus Torvalds /* Somebody else might have mapped it while we slept */ 1811da177e4SLinus Torvalds if (page_address(page)) 1821da177e4SLinus Torvalds return (unsigned long)page_address(page); 1831da177e4SLinus Torvalds 1841da177e4SLinus Torvalds /* Re-start */ 1851da177e4SLinus Torvalds goto start; 1861da177e4SLinus Torvalds } 1871da177e4SLinus Torvalds } 1881da177e4SLinus Torvalds vaddr = PKMAP_ADDR(last_pkmap_nr); 1891da177e4SLinus Torvalds set_pte_at(&init_mm, vaddr, 1901da177e4SLinus Torvalds &(pkmap_page_table[last_pkmap_nr]), mk_pte(page, kmap_prot)); 1911da177e4SLinus Torvalds 1921da177e4SLinus Torvalds pkmap_count[last_pkmap_nr] = 1; 1931da177e4SLinus Torvalds set_page_address(page, (void *)vaddr); 1941da177e4SLinus Torvalds 1951da177e4SLinus Torvalds return vaddr; 1961da177e4SLinus Torvalds } 1971da177e4SLinus Torvalds 19877f6078aSRandy Dunlap /** 19977f6078aSRandy Dunlap * kmap_high - map a highmem page into memory 20077f6078aSRandy Dunlap * @page: &struct page to map 20177f6078aSRandy Dunlap * 20277f6078aSRandy Dunlap * Returns the page's virtual memory address. 20377f6078aSRandy Dunlap * 20477f6078aSRandy Dunlap * We cannot call this from interrupts, as it may block. 20577f6078aSRandy Dunlap */ 206920c7a5dSHarvey Harrison void *kmap_high(struct page *page) 2071da177e4SLinus Torvalds { 2081da177e4SLinus Torvalds unsigned long vaddr; 2091da177e4SLinus Torvalds 2101da177e4SLinus Torvalds /* 2111da177e4SLinus Torvalds * For highmem pages, we can't trust "virtual" until 2121da177e4SLinus Torvalds * after we have the lock. 2131da177e4SLinus Torvalds */ 2143297e760SNicolas Pitre lock_kmap(); 2151da177e4SLinus Torvalds vaddr = (unsigned long)page_address(page); 2161da177e4SLinus Torvalds if (!vaddr) 2171da177e4SLinus Torvalds vaddr = map_new_virtual(page); 2181da177e4SLinus Torvalds pkmap_count[PKMAP_NR(vaddr)]++; 21975babcacSEric Sesterhenn BUG_ON(pkmap_count[PKMAP_NR(vaddr)] < 2); 2203297e760SNicolas Pitre unlock_kmap(); 2211da177e4SLinus Torvalds return (void*) vaddr; 2221da177e4SLinus Torvalds } 2231da177e4SLinus Torvalds 2241da177e4SLinus Torvalds EXPORT_SYMBOL(kmap_high); 2251da177e4SLinus Torvalds 2263297e760SNicolas Pitre #ifdef ARCH_NEEDS_KMAP_HIGH_GET 2273297e760SNicolas Pitre /** 2283297e760SNicolas Pitre * kmap_high_get - pin a highmem page into memory 2293297e760SNicolas Pitre * @page: &struct page to pin 2303297e760SNicolas Pitre * 2313297e760SNicolas Pitre * Returns the page's current virtual memory address, or NULL if no mapping 2325e39df56SUwe Kleine-König * exists. If and only if a non null address is returned then a 2333297e760SNicolas Pitre * matching call to kunmap_high() is necessary. 2343297e760SNicolas Pitre * 2353297e760SNicolas Pitre * This can be called from any context. 2363297e760SNicolas Pitre */ 2373297e760SNicolas Pitre void *kmap_high_get(struct page *page) 2383297e760SNicolas Pitre { 2393297e760SNicolas Pitre unsigned long vaddr, flags; 2403297e760SNicolas Pitre 2413297e760SNicolas Pitre lock_kmap_any(flags); 2423297e760SNicolas Pitre vaddr = (unsigned long)page_address(page); 2433297e760SNicolas Pitre if (vaddr) { 2443297e760SNicolas Pitre BUG_ON(pkmap_count[PKMAP_NR(vaddr)] < 1); 2453297e760SNicolas Pitre pkmap_count[PKMAP_NR(vaddr)]++; 2463297e760SNicolas Pitre } 2473297e760SNicolas Pitre unlock_kmap_any(flags); 2483297e760SNicolas Pitre return (void*) vaddr; 2493297e760SNicolas Pitre } 2503297e760SNicolas Pitre #endif 2513297e760SNicolas Pitre 25277f6078aSRandy Dunlap /** 25377f6078aSRandy Dunlap * kunmap_high - map a highmem page into memory 25477f6078aSRandy Dunlap * @page: &struct page to unmap 2553297e760SNicolas Pitre * 2563297e760SNicolas Pitre * If ARCH_NEEDS_KMAP_HIGH_GET is not defined then this may be called 2573297e760SNicolas Pitre * only from user context. 25877f6078aSRandy Dunlap */ 259920c7a5dSHarvey Harrison void kunmap_high(struct page *page) 2601da177e4SLinus Torvalds { 2611da177e4SLinus Torvalds unsigned long vaddr; 2621da177e4SLinus Torvalds unsigned long nr; 2633297e760SNicolas Pitre unsigned long flags; 2641da177e4SLinus Torvalds int need_wakeup; 2651da177e4SLinus Torvalds 2663297e760SNicolas Pitre lock_kmap_any(flags); 2671da177e4SLinus Torvalds vaddr = (unsigned long)page_address(page); 26875babcacSEric Sesterhenn BUG_ON(!vaddr); 2691da177e4SLinus Torvalds nr = PKMAP_NR(vaddr); 2701da177e4SLinus Torvalds 2711da177e4SLinus Torvalds /* 2721da177e4SLinus Torvalds * A count must never go down to zero 2731da177e4SLinus Torvalds * without a TLB flush! 2741da177e4SLinus Torvalds */ 2751da177e4SLinus Torvalds need_wakeup = 0; 2761da177e4SLinus Torvalds switch (--pkmap_count[nr]) { 2771da177e4SLinus Torvalds case 0: 2781da177e4SLinus Torvalds BUG(); 2791da177e4SLinus Torvalds case 1: 2801da177e4SLinus Torvalds /* 2811da177e4SLinus Torvalds * Avoid an unnecessary wake_up() function call. 2821da177e4SLinus Torvalds * The common case is pkmap_count[] == 1, but 2831da177e4SLinus Torvalds * no waiters. 2841da177e4SLinus Torvalds * The tasks queued in the wait-queue are guarded 2851da177e4SLinus Torvalds * by both the lock in the wait-queue-head and by 2861da177e4SLinus Torvalds * the kmap_lock. As the kmap_lock is held here, 2871da177e4SLinus Torvalds * no need for the wait-queue-head's lock. Simply 2881da177e4SLinus Torvalds * test if the queue is empty. 2891da177e4SLinus Torvalds */ 2901da177e4SLinus Torvalds need_wakeup = waitqueue_active(&pkmap_map_wait); 2911da177e4SLinus Torvalds } 2923297e760SNicolas Pitre unlock_kmap_any(flags); 2931da177e4SLinus Torvalds 2941da177e4SLinus Torvalds /* do wake-up, if needed, race-free outside of the spin lock */ 2951da177e4SLinus Torvalds if (need_wakeup) 2961da177e4SLinus Torvalds wake_up(&pkmap_map_wait); 2971da177e4SLinus Torvalds } 2981da177e4SLinus Torvalds 2991da177e4SLinus Torvalds EXPORT_SYMBOL(kunmap_high); 3001da177e4SLinus Torvalds #endif 3011da177e4SLinus Torvalds 3021da177e4SLinus Torvalds #if defined(HASHED_PAGE_VIRTUAL) 3031da177e4SLinus Torvalds 3041da177e4SLinus Torvalds #define PA_HASH_ORDER 7 3051da177e4SLinus Torvalds 3061da177e4SLinus Torvalds /* 3071da177e4SLinus Torvalds * Describes one page->virtual association 3081da177e4SLinus Torvalds */ 3091da177e4SLinus Torvalds struct page_address_map { 3101da177e4SLinus Torvalds struct page *page; 3111da177e4SLinus Torvalds void *virtual; 3121da177e4SLinus Torvalds struct list_head list; 3131da177e4SLinus Torvalds }; 3141da177e4SLinus Torvalds 3151da177e4SLinus Torvalds /* 3161da177e4SLinus Torvalds * page_address_map freelist, allocated from page_address_maps. 3171da177e4SLinus Torvalds */ 3181da177e4SLinus Torvalds static struct list_head page_address_pool; /* freelist */ 3191da177e4SLinus Torvalds static spinlock_t pool_lock; /* protects page_address_pool */ 3201da177e4SLinus Torvalds 3211da177e4SLinus Torvalds /* 3221da177e4SLinus Torvalds * Hash table bucket 3231da177e4SLinus Torvalds */ 3241da177e4SLinus Torvalds static struct page_address_slot { 3251da177e4SLinus Torvalds struct list_head lh; /* List of page_address_maps */ 3261da177e4SLinus Torvalds spinlock_t lock; /* Protect this bucket's list */ 3271da177e4SLinus Torvalds } ____cacheline_aligned_in_smp page_address_htable[1<<PA_HASH_ORDER]; 3281da177e4SLinus Torvalds 329f9918794SIan Campbell static struct page_address_slot *page_slot(const struct page *page) 3301da177e4SLinus Torvalds { 3311da177e4SLinus Torvalds return &page_address_htable[hash_ptr(page, PA_HASH_ORDER)]; 3321da177e4SLinus Torvalds } 3331da177e4SLinus Torvalds 33477f6078aSRandy Dunlap /** 33577f6078aSRandy Dunlap * page_address - get the mapped virtual address of a page 33677f6078aSRandy Dunlap * @page: &struct page to get the virtual address of 33777f6078aSRandy Dunlap * 33877f6078aSRandy Dunlap * Returns the page's virtual address. 33977f6078aSRandy Dunlap */ 340f9918794SIan Campbell void *page_address(const struct page *page) 3411da177e4SLinus Torvalds { 3421da177e4SLinus Torvalds unsigned long flags; 3431da177e4SLinus Torvalds void *ret; 3441da177e4SLinus Torvalds struct page_address_slot *pas; 3451da177e4SLinus Torvalds 3461da177e4SLinus Torvalds if (!PageHighMem(page)) 3471da177e4SLinus Torvalds return lowmem_page_address(page); 3481da177e4SLinus Torvalds 3491da177e4SLinus Torvalds pas = page_slot(page); 3501da177e4SLinus Torvalds ret = NULL; 3511da177e4SLinus Torvalds spin_lock_irqsave(&pas->lock, flags); 3521da177e4SLinus Torvalds if (!list_empty(&pas->lh)) { 3531da177e4SLinus Torvalds struct page_address_map *pam; 3541da177e4SLinus Torvalds 3551da177e4SLinus Torvalds list_for_each_entry(pam, &pas->lh, list) { 3561da177e4SLinus Torvalds if (pam->page == page) { 3571da177e4SLinus Torvalds ret = pam->virtual; 3581da177e4SLinus Torvalds goto done; 3591da177e4SLinus Torvalds } 3601da177e4SLinus Torvalds } 3611da177e4SLinus Torvalds } 3621da177e4SLinus Torvalds done: 3631da177e4SLinus Torvalds spin_unlock_irqrestore(&pas->lock, flags); 3641da177e4SLinus Torvalds return ret; 3651da177e4SLinus Torvalds } 3661da177e4SLinus Torvalds 3671da177e4SLinus Torvalds EXPORT_SYMBOL(page_address); 3681da177e4SLinus Torvalds 36977f6078aSRandy Dunlap /** 37077f6078aSRandy Dunlap * set_page_address - set a page's virtual address 37177f6078aSRandy Dunlap * @page: &struct page to set 37277f6078aSRandy Dunlap * @virtual: virtual address to use 37377f6078aSRandy Dunlap */ 3741da177e4SLinus Torvalds void set_page_address(struct page *page, void *virtual) 3751da177e4SLinus Torvalds { 3761da177e4SLinus Torvalds unsigned long flags; 3771da177e4SLinus Torvalds struct page_address_slot *pas; 3781da177e4SLinus Torvalds struct page_address_map *pam; 3791da177e4SLinus Torvalds 3801da177e4SLinus Torvalds BUG_ON(!PageHighMem(page)); 3811da177e4SLinus Torvalds 3821da177e4SLinus Torvalds pas = page_slot(page); 3831da177e4SLinus Torvalds if (virtual) { /* Add */ 3841da177e4SLinus Torvalds BUG_ON(list_empty(&page_address_pool)); 3851da177e4SLinus Torvalds 3861da177e4SLinus Torvalds spin_lock_irqsave(&pool_lock, flags); 3871da177e4SLinus Torvalds pam = list_entry(page_address_pool.next, 3881da177e4SLinus Torvalds struct page_address_map, list); 3891da177e4SLinus Torvalds list_del(&pam->list); 3901da177e4SLinus Torvalds spin_unlock_irqrestore(&pool_lock, flags); 3911da177e4SLinus Torvalds 3921da177e4SLinus Torvalds pam->page = page; 3931da177e4SLinus Torvalds pam->virtual = virtual; 3941da177e4SLinus Torvalds 3951da177e4SLinus Torvalds spin_lock_irqsave(&pas->lock, flags); 3961da177e4SLinus Torvalds list_add_tail(&pam->list, &pas->lh); 3971da177e4SLinus Torvalds spin_unlock_irqrestore(&pas->lock, flags); 3981da177e4SLinus Torvalds } else { /* Remove */ 3991da177e4SLinus Torvalds spin_lock_irqsave(&pas->lock, flags); 4001da177e4SLinus Torvalds list_for_each_entry(pam, &pas->lh, list) { 4011da177e4SLinus Torvalds if (pam->page == page) { 4021da177e4SLinus Torvalds list_del(&pam->list); 4031da177e4SLinus Torvalds spin_unlock_irqrestore(&pas->lock, flags); 4041da177e4SLinus Torvalds spin_lock_irqsave(&pool_lock, flags); 4051da177e4SLinus Torvalds list_add_tail(&pam->list, &page_address_pool); 4061da177e4SLinus Torvalds spin_unlock_irqrestore(&pool_lock, flags); 4071da177e4SLinus Torvalds goto done; 4081da177e4SLinus Torvalds } 4091da177e4SLinus Torvalds } 4101da177e4SLinus Torvalds spin_unlock_irqrestore(&pas->lock, flags); 4111da177e4SLinus Torvalds } 4121da177e4SLinus Torvalds done: 4131da177e4SLinus Torvalds return; 4141da177e4SLinus Torvalds } 4151da177e4SLinus Torvalds 4161da177e4SLinus Torvalds static struct page_address_map page_address_maps[LAST_PKMAP]; 4171da177e4SLinus Torvalds 4181da177e4SLinus Torvalds void __init page_address_init(void) 4191da177e4SLinus Torvalds { 4201da177e4SLinus Torvalds int i; 4211da177e4SLinus Torvalds 4221da177e4SLinus Torvalds INIT_LIST_HEAD(&page_address_pool); 4231da177e4SLinus Torvalds for (i = 0; i < ARRAY_SIZE(page_address_maps); i++) 4241da177e4SLinus Torvalds list_add(&page_address_maps[i].list, &page_address_pool); 4251da177e4SLinus Torvalds for (i = 0; i < ARRAY_SIZE(page_address_htable); i++) { 4261da177e4SLinus Torvalds INIT_LIST_HEAD(&page_address_htable[i].lh); 4271da177e4SLinus Torvalds spin_lock_init(&page_address_htable[i].lock); 4281da177e4SLinus Torvalds } 4291da177e4SLinus Torvalds spin_lock_init(&pool_lock); 4301da177e4SLinus Torvalds } 4311da177e4SLinus Torvalds 4321da177e4SLinus Torvalds #endif /* defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL) */ 433