11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * High memory handling common code and variables. 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * (C) 1999 Andrea Arcangeli, SuSE GmbH, andrea@suse.de 51da177e4SLinus Torvalds * Gerhard Wichert, Siemens AG, Gerhard.Wichert@pdb.siemens.de 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * 81da177e4SLinus Torvalds * Redesigned the x86 32-bit VM architecture to deal with 91da177e4SLinus Torvalds * 64-bit physical space. With current x86 CPUs this 101da177e4SLinus Torvalds * means up to 64 Gigabytes physical RAM. 111da177e4SLinus Torvalds * 121da177e4SLinus Torvalds * Rewrote high memory support to move the page cache into 131da177e4SLinus Torvalds * high memory. Implemented permanent (schedulable) kmaps 141da177e4SLinus Torvalds * based on Linus' idea. 151da177e4SLinus Torvalds * 161da177e4SLinus Torvalds * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com> 171da177e4SLinus Torvalds */ 181da177e4SLinus Torvalds 191da177e4SLinus Torvalds #include <linux/mm.h> 201da177e4SLinus Torvalds #include <linux/module.h> 211da177e4SLinus Torvalds #include <linux/swap.h> 221da177e4SLinus Torvalds #include <linux/bio.h> 231da177e4SLinus Torvalds #include <linux/pagemap.h> 241da177e4SLinus Torvalds #include <linux/mempool.h> 251da177e4SLinus Torvalds #include <linux/blkdev.h> 261da177e4SLinus Torvalds #include <linux/init.h> 271da177e4SLinus Torvalds #include <linux/hash.h> 281da177e4SLinus Torvalds #include <linux/highmem.h> 29*eac79005SJason Wessel #include <linux/kgdb.h> 301da177e4SLinus Torvalds #include <asm/tlbflush.h> 311da177e4SLinus Torvalds 321da177e4SLinus Torvalds /* 331da177e4SLinus Torvalds * Virtual_count is not a pure "count". 341da177e4SLinus Torvalds * 0 means that it is not mapped, and has not been mapped 351da177e4SLinus Torvalds * since a TLB flush - it is usable. 361da177e4SLinus Torvalds * 1 means that there are no users, but it has been mapped 371da177e4SLinus Torvalds * since the last TLB flush - so we can't use it. 381da177e4SLinus Torvalds * n means that there are (n-1) current users of it. 391da177e4SLinus Torvalds */ 401da177e4SLinus Torvalds #ifdef CONFIG_HIGHMEM 41260b2367SAl Viro 42c1f60a5aSChristoph Lameter unsigned long totalhigh_pages __read_mostly; 43db7a94d6SDavid S. Miller EXPORT_SYMBOL(totalhigh_pages); 44c1f60a5aSChristoph Lameter 45c1f60a5aSChristoph Lameter unsigned int nr_free_highpages (void) 46c1f60a5aSChristoph Lameter { 47c1f60a5aSChristoph Lameter pg_data_t *pgdat; 48c1f60a5aSChristoph Lameter unsigned int pages = 0; 49c1f60a5aSChristoph Lameter 502a1e274aSMel Gorman for_each_online_pgdat(pgdat) { 51d23ad423SChristoph Lameter pages += zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM], 52d23ad423SChristoph Lameter NR_FREE_PAGES); 532a1e274aSMel Gorman if (zone_movable_is_highmem()) 542a1e274aSMel Gorman pages += zone_page_state( 552a1e274aSMel Gorman &pgdat->node_zones[ZONE_MOVABLE], 562a1e274aSMel Gorman NR_FREE_PAGES); 572a1e274aSMel Gorman } 58c1f60a5aSChristoph Lameter 59c1f60a5aSChristoph Lameter return pages; 60c1f60a5aSChristoph Lameter } 61c1f60a5aSChristoph Lameter 621da177e4SLinus Torvalds static int pkmap_count[LAST_PKMAP]; 631da177e4SLinus Torvalds static unsigned int last_pkmap_nr; 641da177e4SLinus Torvalds static __cacheline_aligned_in_smp DEFINE_SPINLOCK(kmap_lock); 651da177e4SLinus Torvalds 661da177e4SLinus Torvalds pte_t * pkmap_page_table; 671da177e4SLinus Torvalds 681da177e4SLinus Torvalds static DECLARE_WAIT_QUEUE_HEAD(pkmap_map_wait); 691da177e4SLinus Torvalds 703297e760SNicolas Pitre /* 713297e760SNicolas Pitre * Most architectures have no use for kmap_high_get(), so let's abstract 723297e760SNicolas Pitre * the disabling of IRQ out of the locking in that case to save on a 733297e760SNicolas Pitre * potential useless overhead. 743297e760SNicolas Pitre */ 753297e760SNicolas Pitre #ifdef ARCH_NEEDS_KMAP_HIGH_GET 763297e760SNicolas Pitre #define lock_kmap() spin_lock_irq(&kmap_lock) 773297e760SNicolas Pitre #define unlock_kmap() spin_unlock_irq(&kmap_lock) 783297e760SNicolas Pitre #define lock_kmap_any(flags) spin_lock_irqsave(&kmap_lock, flags) 793297e760SNicolas Pitre #define unlock_kmap_any(flags) spin_unlock_irqrestore(&kmap_lock, flags) 803297e760SNicolas Pitre #else 813297e760SNicolas Pitre #define lock_kmap() spin_lock(&kmap_lock) 823297e760SNicolas Pitre #define unlock_kmap() spin_unlock(&kmap_lock) 833297e760SNicolas Pitre #define lock_kmap_any(flags) \ 843297e760SNicolas Pitre do { spin_lock(&kmap_lock); (void)(flags); } while (0) 853297e760SNicolas Pitre #define unlock_kmap_any(flags) \ 863297e760SNicolas Pitre do { spin_unlock(&kmap_lock); (void)(flags); } while (0) 873297e760SNicolas Pitre #endif 883297e760SNicolas Pitre 891da177e4SLinus Torvalds static void flush_all_zero_pkmaps(void) 901da177e4SLinus Torvalds { 911da177e4SLinus Torvalds int i; 925843d9a4SNick Piggin int need_flush = 0; 931da177e4SLinus Torvalds 941da177e4SLinus Torvalds flush_cache_kmaps(); 951da177e4SLinus Torvalds 961da177e4SLinus Torvalds for (i = 0; i < LAST_PKMAP; i++) { 971da177e4SLinus Torvalds struct page *page; 981da177e4SLinus Torvalds 991da177e4SLinus Torvalds /* 1001da177e4SLinus Torvalds * zero means we don't have anything to do, 1011da177e4SLinus Torvalds * >1 means that it is still in use. Only 1021da177e4SLinus Torvalds * a count of 1 means that it is free but 1031da177e4SLinus Torvalds * needs to be unmapped 1041da177e4SLinus Torvalds */ 1051da177e4SLinus Torvalds if (pkmap_count[i] != 1) 1061da177e4SLinus Torvalds continue; 1071da177e4SLinus Torvalds pkmap_count[i] = 0; 1081da177e4SLinus Torvalds 1091da177e4SLinus Torvalds /* sanity check */ 11075babcacSEric Sesterhenn BUG_ON(pte_none(pkmap_page_table[i])); 1111da177e4SLinus Torvalds 1121da177e4SLinus Torvalds /* 1131da177e4SLinus Torvalds * Don't need an atomic fetch-and-clear op here; 1141da177e4SLinus Torvalds * no-one has the page mapped, and cannot get at 1151da177e4SLinus Torvalds * its virtual address (and hence PTE) without first 1161da177e4SLinus Torvalds * getting the kmap_lock (which is held here). 1171da177e4SLinus Torvalds * So no dangers, even with speculative execution. 1181da177e4SLinus Torvalds */ 1191da177e4SLinus Torvalds page = pte_page(pkmap_page_table[i]); 1201da177e4SLinus Torvalds pte_clear(&init_mm, (unsigned long)page_address(page), 1211da177e4SLinus Torvalds &pkmap_page_table[i]); 1221da177e4SLinus Torvalds 1231da177e4SLinus Torvalds set_page_address(page, NULL); 1245843d9a4SNick Piggin need_flush = 1; 1251da177e4SLinus Torvalds } 1265843d9a4SNick Piggin if (need_flush) 1271da177e4SLinus Torvalds flush_tlb_kernel_range(PKMAP_ADDR(0), PKMAP_ADDR(LAST_PKMAP)); 1281da177e4SLinus Torvalds } 1291da177e4SLinus Torvalds 13077f6078aSRandy Dunlap /** 13177f6078aSRandy Dunlap * kmap_flush_unused - flush all unused kmap mappings in order to remove stray mappings 13277f6078aSRandy Dunlap */ 133ce6234b5SJeremy Fitzhardinge void kmap_flush_unused(void) 134ce6234b5SJeremy Fitzhardinge { 1353297e760SNicolas Pitre lock_kmap(); 136ce6234b5SJeremy Fitzhardinge flush_all_zero_pkmaps(); 1373297e760SNicolas Pitre unlock_kmap(); 138ce6234b5SJeremy Fitzhardinge } 139ce6234b5SJeremy Fitzhardinge 1401da177e4SLinus Torvalds static inline unsigned long map_new_virtual(struct page *page) 1411da177e4SLinus Torvalds { 1421da177e4SLinus Torvalds unsigned long vaddr; 1431da177e4SLinus Torvalds int count; 1441da177e4SLinus Torvalds 1451da177e4SLinus Torvalds start: 1461da177e4SLinus Torvalds count = LAST_PKMAP; 1471da177e4SLinus Torvalds /* Find an empty entry */ 1481da177e4SLinus Torvalds for (;;) { 1491da177e4SLinus Torvalds last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK; 1501da177e4SLinus Torvalds if (!last_pkmap_nr) { 1511da177e4SLinus Torvalds flush_all_zero_pkmaps(); 1521da177e4SLinus Torvalds count = LAST_PKMAP; 1531da177e4SLinus Torvalds } 1541da177e4SLinus Torvalds if (!pkmap_count[last_pkmap_nr]) 1551da177e4SLinus Torvalds break; /* Found a usable entry */ 1561da177e4SLinus Torvalds if (--count) 1571da177e4SLinus Torvalds continue; 1581da177e4SLinus Torvalds 1591da177e4SLinus Torvalds /* 1601da177e4SLinus Torvalds * Sleep for somebody else to unmap their entries 1611da177e4SLinus Torvalds */ 1621da177e4SLinus Torvalds { 1631da177e4SLinus Torvalds DECLARE_WAITQUEUE(wait, current); 1641da177e4SLinus Torvalds 1651da177e4SLinus Torvalds __set_current_state(TASK_UNINTERRUPTIBLE); 1661da177e4SLinus Torvalds add_wait_queue(&pkmap_map_wait, &wait); 1673297e760SNicolas Pitre unlock_kmap(); 1681da177e4SLinus Torvalds schedule(); 1691da177e4SLinus Torvalds remove_wait_queue(&pkmap_map_wait, &wait); 1703297e760SNicolas Pitre lock_kmap(); 1711da177e4SLinus Torvalds 1721da177e4SLinus Torvalds /* Somebody else might have mapped it while we slept */ 1731da177e4SLinus Torvalds if (page_address(page)) 1741da177e4SLinus Torvalds return (unsigned long)page_address(page); 1751da177e4SLinus Torvalds 1761da177e4SLinus Torvalds /* Re-start */ 1771da177e4SLinus Torvalds goto start; 1781da177e4SLinus Torvalds } 1791da177e4SLinus Torvalds } 1801da177e4SLinus Torvalds vaddr = PKMAP_ADDR(last_pkmap_nr); 1811da177e4SLinus Torvalds set_pte_at(&init_mm, vaddr, 1821da177e4SLinus Torvalds &(pkmap_page_table[last_pkmap_nr]), mk_pte(page, kmap_prot)); 1831da177e4SLinus Torvalds 1841da177e4SLinus Torvalds pkmap_count[last_pkmap_nr] = 1; 1851da177e4SLinus Torvalds set_page_address(page, (void *)vaddr); 1861da177e4SLinus Torvalds 1871da177e4SLinus Torvalds return vaddr; 1881da177e4SLinus Torvalds } 1891da177e4SLinus Torvalds 19077f6078aSRandy Dunlap /** 19177f6078aSRandy Dunlap * kmap_high - map a highmem page into memory 19277f6078aSRandy Dunlap * @page: &struct page to map 19377f6078aSRandy Dunlap * 19477f6078aSRandy Dunlap * Returns the page's virtual memory address. 19577f6078aSRandy Dunlap * 19677f6078aSRandy Dunlap * We cannot call this from interrupts, as it may block. 19777f6078aSRandy Dunlap */ 198920c7a5dSHarvey Harrison void *kmap_high(struct page *page) 1991da177e4SLinus Torvalds { 2001da177e4SLinus Torvalds unsigned long vaddr; 2011da177e4SLinus Torvalds 2021da177e4SLinus Torvalds /* 2031da177e4SLinus Torvalds * For highmem pages, we can't trust "virtual" until 2041da177e4SLinus Torvalds * after we have the lock. 2051da177e4SLinus Torvalds */ 2063297e760SNicolas Pitre lock_kmap(); 2071da177e4SLinus Torvalds vaddr = (unsigned long)page_address(page); 2081da177e4SLinus Torvalds if (!vaddr) 2091da177e4SLinus Torvalds vaddr = map_new_virtual(page); 2101da177e4SLinus Torvalds pkmap_count[PKMAP_NR(vaddr)]++; 21175babcacSEric Sesterhenn BUG_ON(pkmap_count[PKMAP_NR(vaddr)] < 2); 2123297e760SNicolas Pitre unlock_kmap(); 2131da177e4SLinus Torvalds return (void*) vaddr; 2141da177e4SLinus Torvalds } 2151da177e4SLinus Torvalds 2161da177e4SLinus Torvalds EXPORT_SYMBOL(kmap_high); 2171da177e4SLinus Torvalds 2183297e760SNicolas Pitre #ifdef ARCH_NEEDS_KMAP_HIGH_GET 2193297e760SNicolas Pitre /** 2203297e760SNicolas Pitre * kmap_high_get - pin a highmem page into memory 2213297e760SNicolas Pitre * @page: &struct page to pin 2223297e760SNicolas Pitre * 2233297e760SNicolas Pitre * Returns the page's current virtual memory address, or NULL if no mapping 2245e39df56SUwe Kleine-König * exists. If and only if a non null address is returned then a 2253297e760SNicolas Pitre * matching call to kunmap_high() is necessary. 2263297e760SNicolas Pitre * 2273297e760SNicolas Pitre * This can be called from any context. 2283297e760SNicolas Pitre */ 2293297e760SNicolas Pitre void *kmap_high_get(struct page *page) 2303297e760SNicolas Pitre { 2313297e760SNicolas Pitre unsigned long vaddr, flags; 2323297e760SNicolas Pitre 2333297e760SNicolas Pitre lock_kmap_any(flags); 2343297e760SNicolas Pitre vaddr = (unsigned long)page_address(page); 2353297e760SNicolas Pitre if (vaddr) { 2363297e760SNicolas Pitre BUG_ON(pkmap_count[PKMAP_NR(vaddr)] < 1); 2373297e760SNicolas Pitre pkmap_count[PKMAP_NR(vaddr)]++; 2383297e760SNicolas Pitre } 2393297e760SNicolas Pitre unlock_kmap_any(flags); 2403297e760SNicolas Pitre return (void*) vaddr; 2413297e760SNicolas Pitre } 2423297e760SNicolas Pitre #endif 2433297e760SNicolas Pitre 24477f6078aSRandy Dunlap /** 24577f6078aSRandy Dunlap * kunmap_high - map a highmem page into memory 24677f6078aSRandy Dunlap * @page: &struct page to unmap 2473297e760SNicolas Pitre * 2483297e760SNicolas Pitre * If ARCH_NEEDS_KMAP_HIGH_GET is not defined then this may be called 2493297e760SNicolas Pitre * only from user context. 25077f6078aSRandy Dunlap */ 251920c7a5dSHarvey Harrison void kunmap_high(struct page *page) 2521da177e4SLinus Torvalds { 2531da177e4SLinus Torvalds unsigned long vaddr; 2541da177e4SLinus Torvalds unsigned long nr; 2553297e760SNicolas Pitre unsigned long flags; 2561da177e4SLinus Torvalds int need_wakeup; 2571da177e4SLinus Torvalds 2583297e760SNicolas Pitre lock_kmap_any(flags); 2591da177e4SLinus Torvalds vaddr = (unsigned long)page_address(page); 26075babcacSEric Sesterhenn BUG_ON(!vaddr); 2611da177e4SLinus Torvalds nr = PKMAP_NR(vaddr); 2621da177e4SLinus Torvalds 2631da177e4SLinus Torvalds /* 2641da177e4SLinus Torvalds * A count must never go down to zero 2651da177e4SLinus Torvalds * without a TLB flush! 2661da177e4SLinus Torvalds */ 2671da177e4SLinus Torvalds need_wakeup = 0; 2681da177e4SLinus Torvalds switch (--pkmap_count[nr]) { 2691da177e4SLinus Torvalds case 0: 2701da177e4SLinus Torvalds BUG(); 2711da177e4SLinus Torvalds case 1: 2721da177e4SLinus Torvalds /* 2731da177e4SLinus Torvalds * Avoid an unnecessary wake_up() function call. 2741da177e4SLinus Torvalds * The common case is pkmap_count[] == 1, but 2751da177e4SLinus Torvalds * no waiters. 2761da177e4SLinus Torvalds * The tasks queued in the wait-queue are guarded 2771da177e4SLinus Torvalds * by both the lock in the wait-queue-head and by 2781da177e4SLinus Torvalds * the kmap_lock. As the kmap_lock is held here, 2791da177e4SLinus Torvalds * no need for the wait-queue-head's lock. Simply 2801da177e4SLinus Torvalds * test if the queue is empty. 2811da177e4SLinus Torvalds */ 2821da177e4SLinus Torvalds need_wakeup = waitqueue_active(&pkmap_map_wait); 2831da177e4SLinus Torvalds } 2843297e760SNicolas Pitre unlock_kmap_any(flags); 2851da177e4SLinus Torvalds 2861da177e4SLinus Torvalds /* do wake-up, if needed, race-free outside of the spin lock */ 2871da177e4SLinus Torvalds if (need_wakeup) 2881da177e4SLinus Torvalds wake_up(&pkmap_map_wait); 2891da177e4SLinus Torvalds } 2901da177e4SLinus Torvalds 2911da177e4SLinus Torvalds EXPORT_SYMBOL(kunmap_high); 2921da177e4SLinus Torvalds #endif 2931da177e4SLinus Torvalds 2941da177e4SLinus Torvalds #if defined(HASHED_PAGE_VIRTUAL) 2951da177e4SLinus Torvalds 2961da177e4SLinus Torvalds #define PA_HASH_ORDER 7 2971da177e4SLinus Torvalds 2981da177e4SLinus Torvalds /* 2991da177e4SLinus Torvalds * Describes one page->virtual association 3001da177e4SLinus Torvalds */ 3011da177e4SLinus Torvalds struct page_address_map { 3021da177e4SLinus Torvalds struct page *page; 3031da177e4SLinus Torvalds void *virtual; 3041da177e4SLinus Torvalds struct list_head list; 3051da177e4SLinus Torvalds }; 3061da177e4SLinus Torvalds 3071da177e4SLinus Torvalds /* 3081da177e4SLinus Torvalds * page_address_map freelist, allocated from page_address_maps. 3091da177e4SLinus Torvalds */ 3101da177e4SLinus Torvalds static struct list_head page_address_pool; /* freelist */ 3111da177e4SLinus Torvalds static spinlock_t pool_lock; /* protects page_address_pool */ 3121da177e4SLinus Torvalds 3131da177e4SLinus Torvalds /* 3141da177e4SLinus Torvalds * Hash table bucket 3151da177e4SLinus Torvalds */ 3161da177e4SLinus Torvalds static struct page_address_slot { 3171da177e4SLinus Torvalds struct list_head lh; /* List of page_address_maps */ 3181da177e4SLinus Torvalds spinlock_t lock; /* Protect this bucket's list */ 3191da177e4SLinus Torvalds } ____cacheline_aligned_in_smp page_address_htable[1<<PA_HASH_ORDER]; 3201da177e4SLinus Torvalds 3211da177e4SLinus Torvalds static struct page_address_slot *page_slot(struct page *page) 3221da177e4SLinus Torvalds { 3231da177e4SLinus Torvalds return &page_address_htable[hash_ptr(page, PA_HASH_ORDER)]; 3241da177e4SLinus Torvalds } 3251da177e4SLinus Torvalds 32677f6078aSRandy Dunlap /** 32777f6078aSRandy Dunlap * page_address - get the mapped virtual address of a page 32877f6078aSRandy Dunlap * @page: &struct page to get the virtual address of 32977f6078aSRandy Dunlap * 33077f6078aSRandy Dunlap * Returns the page's virtual address. 33177f6078aSRandy Dunlap */ 3321da177e4SLinus Torvalds void *page_address(struct page *page) 3331da177e4SLinus Torvalds { 3341da177e4SLinus Torvalds unsigned long flags; 3351da177e4SLinus Torvalds void *ret; 3361da177e4SLinus Torvalds struct page_address_slot *pas; 3371da177e4SLinus Torvalds 3381da177e4SLinus Torvalds if (!PageHighMem(page)) 3391da177e4SLinus Torvalds return lowmem_page_address(page); 3401da177e4SLinus Torvalds 3411da177e4SLinus Torvalds pas = page_slot(page); 3421da177e4SLinus Torvalds ret = NULL; 3431da177e4SLinus Torvalds spin_lock_irqsave(&pas->lock, flags); 3441da177e4SLinus Torvalds if (!list_empty(&pas->lh)) { 3451da177e4SLinus Torvalds struct page_address_map *pam; 3461da177e4SLinus Torvalds 3471da177e4SLinus Torvalds list_for_each_entry(pam, &pas->lh, list) { 3481da177e4SLinus Torvalds if (pam->page == page) { 3491da177e4SLinus Torvalds ret = pam->virtual; 3501da177e4SLinus Torvalds goto done; 3511da177e4SLinus Torvalds } 3521da177e4SLinus Torvalds } 3531da177e4SLinus Torvalds } 3541da177e4SLinus Torvalds done: 3551da177e4SLinus Torvalds spin_unlock_irqrestore(&pas->lock, flags); 3561da177e4SLinus Torvalds return ret; 3571da177e4SLinus Torvalds } 3581da177e4SLinus Torvalds 3591da177e4SLinus Torvalds EXPORT_SYMBOL(page_address); 3601da177e4SLinus Torvalds 36177f6078aSRandy Dunlap /** 36277f6078aSRandy Dunlap * set_page_address - set a page's virtual address 36377f6078aSRandy Dunlap * @page: &struct page to set 36477f6078aSRandy Dunlap * @virtual: virtual address to use 36577f6078aSRandy Dunlap */ 3661da177e4SLinus Torvalds void set_page_address(struct page *page, void *virtual) 3671da177e4SLinus Torvalds { 3681da177e4SLinus Torvalds unsigned long flags; 3691da177e4SLinus Torvalds struct page_address_slot *pas; 3701da177e4SLinus Torvalds struct page_address_map *pam; 3711da177e4SLinus Torvalds 3721da177e4SLinus Torvalds BUG_ON(!PageHighMem(page)); 3731da177e4SLinus Torvalds 3741da177e4SLinus Torvalds pas = page_slot(page); 3751da177e4SLinus Torvalds if (virtual) { /* Add */ 3761da177e4SLinus Torvalds BUG_ON(list_empty(&page_address_pool)); 3771da177e4SLinus Torvalds 3781da177e4SLinus Torvalds spin_lock_irqsave(&pool_lock, flags); 3791da177e4SLinus Torvalds pam = list_entry(page_address_pool.next, 3801da177e4SLinus Torvalds struct page_address_map, list); 3811da177e4SLinus Torvalds list_del(&pam->list); 3821da177e4SLinus Torvalds spin_unlock_irqrestore(&pool_lock, flags); 3831da177e4SLinus Torvalds 3841da177e4SLinus Torvalds pam->page = page; 3851da177e4SLinus Torvalds pam->virtual = virtual; 3861da177e4SLinus Torvalds 3871da177e4SLinus Torvalds spin_lock_irqsave(&pas->lock, flags); 3881da177e4SLinus Torvalds list_add_tail(&pam->list, &pas->lh); 3891da177e4SLinus Torvalds spin_unlock_irqrestore(&pas->lock, flags); 3901da177e4SLinus Torvalds } else { /* Remove */ 3911da177e4SLinus Torvalds spin_lock_irqsave(&pas->lock, flags); 3921da177e4SLinus Torvalds list_for_each_entry(pam, &pas->lh, list) { 3931da177e4SLinus Torvalds if (pam->page == page) { 3941da177e4SLinus Torvalds list_del(&pam->list); 3951da177e4SLinus Torvalds spin_unlock_irqrestore(&pas->lock, flags); 3961da177e4SLinus Torvalds spin_lock_irqsave(&pool_lock, flags); 3971da177e4SLinus Torvalds list_add_tail(&pam->list, &page_address_pool); 3981da177e4SLinus Torvalds spin_unlock_irqrestore(&pool_lock, flags); 3991da177e4SLinus Torvalds goto done; 4001da177e4SLinus Torvalds } 4011da177e4SLinus Torvalds } 4021da177e4SLinus Torvalds spin_unlock_irqrestore(&pas->lock, flags); 4031da177e4SLinus Torvalds } 4041da177e4SLinus Torvalds done: 4051da177e4SLinus Torvalds return; 4061da177e4SLinus Torvalds } 4071da177e4SLinus Torvalds 4081da177e4SLinus Torvalds static struct page_address_map page_address_maps[LAST_PKMAP]; 4091da177e4SLinus Torvalds 4101da177e4SLinus Torvalds void __init page_address_init(void) 4111da177e4SLinus Torvalds { 4121da177e4SLinus Torvalds int i; 4131da177e4SLinus Torvalds 4141da177e4SLinus Torvalds INIT_LIST_HEAD(&page_address_pool); 4151da177e4SLinus Torvalds for (i = 0; i < ARRAY_SIZE(page_address_maps); i++) 4161da177e4SLinus Torvalds list_add(&page_address_maps[i].list, &page_address_pool); 4171da177e4SLinus Torvalds for (i = 0; i < ARRAY_SIZE(page_address_htable); i++) { 4181da177e4SLinus Torvalds INIT_LIST_HEAD(&page_address_htable[i].lh); 4191da177e4SLinus Torvalds spin_lock_init(&page_address_htable[i].lock); 4201da177e4SLinus Torvalds } 4211da177e4SLinus Torvalds spin_lock_init(&pool_lock); 4221da177e4SLinus Torvalds } 4231da177e4SLinus Torvalds 4241da177e4SLinus Torvalds #endif /* defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL) */ 425f4112de6SAkinobu Mita 426ff3d58c2SAkinobu Mita #ifdef CONFIG_DEBUG_HIGHMEM 427f4112de6SAkinobu Mita 428f4112de6SAkinobu Mita void debug_kmap_atomic(enum km_type type) 429f4112de6SAkinobu Mita { 4305ebd4c22SSoeren Sandmann static int warn_count = 10; 431f4112de6SAkinobu Mita 4325ebd4c22SSoeren Sandmann if (unlikely(warn_count < 0)) 433f4112de6SAkinobu Mita return; 434f4112de6SAkinobu Mita 435f4112de6SAkinobu Mita if (unlikely(in_interrupt())) { 436d4515646SSoeren Sandmann if (in_nmi()) { 437d4515646SSoeren Sandmann if (type != KM_NMI && type != KM_NMI_PTE) { 438d4515646SSoeren Sandmann WARN_ON(1); 439d4515646SSoeren Sandmann warn_count--; 440d4515646SSoeren Sandmann } 441d4515646SSoeren Sandmann } else if (in_irq()) { 442f4112de6SAkinobu Mita if (type != KM_IRQ0 && type != KM_IRQ1 && 443f4112de6SAkinobu Mita type != KM_BIO_SRC_IRQ && type != KM_BIO_DST_IRQ && 444d4515646SSoeren Sandmann type != KM_BOUNCE_READ && type != KM_IRQ_PTE) { 445f4112de6SAkinobu Mita WARN_ON(1); 446f4112de6SAkinobu Mita warn_count--; 447f4112de6SAkinobu Mita } 448f4112de6SAkinobu Mita } else if (!irqs_disabled()) { /* softirq */ 449f4112de6SAkinobu Mita if (type != KM_IRQ0 && type != KM_IRQ1 && 450f4112de6SAkinobu Mita type != KM_SOFTIRQ0 && type != KM_SOFTIRQ1 && 451f4112de6SAkinobu Mita type != KM_SKB_SUNRPC_DATA && 452f4112de6SAkinobu Mita type != KM_SKB_DATA_SOFTIRQ && 453f4112de6SAkinobu Mita type != KM_BOUNCE_READ) { 454f4112de6SAkinobu Mita WARN_ON(1); 455f4112de6SAkinobu Mita warn_count--; 456f4112de6SAkinobu Mita } 457f4112de6SAkinobu Mita } 458f4112de6SAkinobu Mita } 459f4112de6SAkinobu Mita 460f4112de6SAkinobu Mita if (type == KM_IRQ0 || type == KM_IRQ1 || type == KM_BOUNCE_READ || 461d4515646SSoeren Sandmann type == KM_BIO_SRC_IRQ || type == KM_BIO_DST_IRQ || 462d4515646SSoeren Sandmann type == KM_IRQ_PTE || type == KM_NMI || 463d4515646SSoeren Sandmann type == KM_NMI_PTE ) { 464f4112de6SAkinobu Mita if (!irqs_disabled()) { 465f4112de6SAkinobu Mita WARN_ON(1); 466f4112de6SAkinobu Mita warn_count--; 467f4112de6SAkinobu Mita } 468f4112de6SAkinobu Mita } else if (type == KM_SOFTIRQ0 || type == KM_SOFTIRQ1) { 469f4112de6SAkinobu Mita if (irq_count() == 0 && !irqs_disabled()) { 470f4112de6SAkinobu Mita WARN_ON(1); 471f4112de6SAkinobu Mita warn_count--; 472f4112de6SAkinobu Mita } 473f4112de6SAkinobu Mita } 474*eac79005SJason Wessel #ifdef CONFIG_KGDB_KDB 475*eac79005SJason Wessel if (unlikely(type == KM_KDB && atomic_read(&kgdb_active) == -1)) { 476*eac79005SJason Wessel WARN_ON(1); 477*eac79005SJason Wessel warn_count--; 478*eac79005SJason Wessel } 479*eac79005SJason Wessel #endif /* CONFIG_KGDB_KDB */ 480f4112de6SAkinobu Mita } 481f4112de6SAkinobu Mita 482f4112de6SAkinobu Mita #endif 483