1 /* 2 * This file contains ioremap and related functions for 64-bit machines. 3 * 4 * Derived from arch/ppc64/mm/init.c 5 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 6 * 7 * Modifications by Paul Mackerras (PowerMac) (paulus@samba.org) 8 * and Cort Dougan (PReP) (cort@cs.nmt.edu) 9 * Copyright (C) 1996 Paul Mackerras 10 * 11 * Derived from "arch/i386/mm/init.c" 12 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 13 * 14 * Dave Engebretsen <engebret@us.ibm.com> 15 * Rework for PPC64 port. 16 * 17 * This program is free software; you can redistribute it and/or 18 * modify it under the terms of the GNU General Public License 19 * as published by the Free Software Foundation; either version 20 * 2 of the License, or (at your option) any later version. 21 * 22 */ 23 24 #include <linux/signal.h> 25 #include <linux/sched.h> 26 #include <linux/kernel.h> 27 #include <linux/errno.h> 28 #include <linux/string.h> 29 #include <linux/export.h> 30 #include <linux/types.h> 31 #include <linux/mman.h> 32 #include <linux/mm.h> 33 #include <linux/swap.h> 34 #include <linux/stddef.h> 35 #include <linux/vmalloc.h> 36 #include <linux/init.h> 37 #include <linux/bootmem.h> 38 #include <linux/memblock.h> 39 #include <linux/slab.h> 40 41 #include <asm/pgalloc.h> 42 #include <asm/page.h> 43 #include <asm/prom.h> 44 #include <asm/io.h> 45 #include <asm/mmu_context.h> 46 #include <asm/pgtable.h> 47 #include <asm/mmu.h> 48 #include <asm/smp.h> 49 #include <asm/machdep.h> 50 #include <asm/tlb.h> 51 #include <asm/processor.h> 52 #include <asm/cputable.h> 53 #include <asm/sections.h> 54 #include <asm/system.h> 55 #include <asm/abs_addr.h> 56 #include <asm/firmware.h> 57 58 #include "mmu_decl.h" 59 60 unsigned long ioremap_bot = IOREMAP_BASE; 61 62 63 #ifdef CONFIG_PPC_MMU_NOHASH 64 static void *early_alloc_pgtable(unsigned long size) 65 { 66 void *pt; 67 68 if (init_bootmem_done) 69 pt = __alloc_bootmem(size, size, __pa(MAX_DMA_ADDRESS)); 70 else 71 pt = __va(memblock_alloc_base(size, size, 72 __pa(MAX_DMA_ADDRESS))); 73 memset(pt, 0, size); 74 75 return pt; 76 } 77 #endif /* CONFIG_PPC_MMU_NOHASH */ 78 79 /* 80 * map_kernel_page currently only called by __ioremap 81 * map_kernel_page adds an entry to the ioremap page table 82 * and adds an entry to the HPT, possibly bolting it 83 */ 84 int map_kernel_page(unsigned long ea, unsigned long pa, int flags) 85 { 86 pgd_t *pgdp; 87 pud_t *pudp; 88 pmd_t *pmdp; 89 pte_t *ptep; 90 91 if (slab_is_available()) { 92 pgdp = pgd_offset_k(ea); 93 pudp = pud_alloc(&init_mm, pgdp, ea); 94 if (!pudp) 95 return -ENOMEM; 96 pmdp = pmd_alloc(&init_mm, pudp, ea); 97 if (!pmdp) 98 return -ENOMEM; 99 ptep = pte_alloc_kernel(pmdp, ea); 100 if (!ptep) 101 return -ENOMEM; 102 set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, 103 __pgprot(flags))); 104 } else { 105 #ifdef CONFIG_PPC_MMU_NOHASH 106 /* Warning ! This will blow up if bootmem is not initialized 107 * which our ppc64 code is keen to do that, we'll need to 108 * fix it and/or be more careful 109 */ 110 pgdp = pgd_offset_k(ea); 111 #ifdef PUD_TABLE_SIZE 112 if (pgd_none(*pgdp)) { 113 pudp = early_alloc_pgtable(PUD_TABLE_SIZE); 114 BUG_ON(pudp == NULL); 115 pgd_populate(&init_mm, pgdp, pudp); 116 } 117 #endif /* PUD_TABLE_SIZE */ 118 pudp = pud_offset(pgdp, ea); 119 if (pud_none(*pudp)) { 120 pmdp = early_alloc_pgtable(PMD_TABLE_SIZE); 121 BUG_ON(pmdp == NULL); 122 pud_populate(&init_mm, pudp, pmdp); 123 } 124 pmdp = pmd_offset(pudp, ea); 125 if (!pmd_present(*pmdp)) { 126 ptep = early_alloc_pgtable(PAGE_SIZE); 127 BUG_ON(ptep == NULL); 128 pmd_populate_kernel(&init_mm, pmdp, ptep); 129 } 130 ptep = pte_offset_kernel(pmdp, ea); 131 set_pte_at(&init_mm, ea, ptep, pfn_pte(pa >> PAGE_SHIFT, 132 __pgprot(flags))); 133 #else /* CONFIG_PPC_MMU_NOHASH */ 134 /* 135 * If the mm subsystem is not fully up, we cannot create a 136 * linux page table entry for this mapping. Simply bolt an 137 * entry in the hardware page table. 138 * 139 */ 140 if (htab_bolt_mapping(ea, ea + PAGE_SIZE, pa, flags, 141 mmu_io_psize, mmu_kernel_ssize)) { 142 printk(KERN_ERR "Failed to do bolted mapping IO " 143 "memory at %016lx !\n", pa); 144 return -ENOMEM; 145 } 146 #endif /* !CONFIG_PPC_MMU_NOHASH */ 147 } 148 return 0; 149 } 150 151 152 /** 153 * __ioremap_at - Low level function to establish the page tables 154 * for an IO mapping 155 */ 156 void __iomem * __ioremap_at(phys_addr_t pa, void *ea, unsigned long size, 157 unsigned long flags) 158 { 159 unsigned long i; 160 161 /* Make sure we have the base flags */ 162 if ((flags & _PAGE_PRESENT) == 0) 163 flags |= pgprot_val(PAGE_KERNEL); 164 165 /* Non-cacheable page cannot be coherent */ 166 if (flags & _PAGE_NO_CACHE) 167 flags &= ~_PAGE_COHERENT; 168 169 /* We don't support the 4K PFN hack with ioremap */ 170 if (flags & _PAGE_4K_PFN) 171 return NULL; 172 173 WARN_ON(pa & ~PAGE_MASK); 174 WARN_ON(((unsigned long)ea) & ~PAGE_MASK); 175 WARN_ON(size & ~PAGE_MASK); 176 177 for (i = 0; i < size; i += PAGE_SIZE) 178 if (map_kernel_page((unsigned long)ea+i, pa+i, flags)) 179 return NULL; 180 181 return (void __iomem *)ea; 182 } 183 184 /** 185 * __iounmap_from - Low level function to tear down the page tables 186 * for an IO mapping. This is used for mappings that 187 * are manipulated manually, like partial unmapping of 188 * PCI IOs or ISA space. 189 */ 190 void __iounmap_at(void *ea, unsigned long size) 191 { 192 WARN_ON(((unsigned long)ea) & ~PAGE_MASK); 193 WARN_ON(size & ~PAGE_MASK); 194 195 unmap_kernel_range((unsigned long)ea, size); 196 } 197 198 void __iomem * __ioremap_caller(phys_addr_t addr, unsigned long size, 199 unsigned long flags, void *caller) 200 { 201 phys_addr_t paligned; 202 void __iomem *ret; 203 204 /* 205 * Choose an address to map it to. 206 * Once the imalloc system is running, we use it. 207 * Before that, we map using addresses going 208 * up from ioremap_bot. imalloc will use 209 * the addresses from ioremap_bot through 210 * IMALLOC_END 211 * 212 */ 213 paligned = addr & PAGE_MASK; 214 size = PAGE_ALIGN(addr + size) - paligned; 215 216 if ((size == 0) || (paligned == 0)) 217 return NULL; 218 219 if (mem_init_done) { 220 struct vm_struct *area; 221 222 area = __get_vm_area_caller(size, VM_IOREMAP, 223 ioremap_bot, IOREMAP_END, 224 caller); 225 if (area == NULL) 226 return NULL; 227 228 area->phys_addr = paligned; 229 ret = __ioremap_at(paligned, area->addr, size, flags); 230 if (!ret) 231 vunmap(area->addr); 232 } else { 233 ret = __ioremap_at(paligned, (void *)ioremap_bot, size, flags); 234 if (ret) 235 ioremap_bot += size; 236 } 237 238 if (ret) 239 ret += addr & ~PAGE_MASK; 240 return ret; 241 } 242 243 void __iomem * __ioremap(phys_addr_t addr, unsigned long size, 244 unsigned long flags) 245 { 246 return __ioremap_caller(addr, size, flags, __builtin_return_address(0)); 247 } 248 249 void __iomem * ioremap(phys_addr_t addr, unsigned long size) 250 { 251 unsigned long flags = _PAGE_NO_CACHE | _PAGE_GUARDED; 252 void *caller = __builtin_return_address(0); 253 254 if (ppc_md.ioremap) 255 return ppc_md.ioremap(addr, size, flags, caller); 256 return __ioremap_caller(addr, size, flags, caller); 257 } 258 259 void __iomem * ioremap_wc(phys_addr_t addr, unsigned long size) 260 { 261 unsigned long flags = _PAGE_NO_CACHE; 262 void *caller = __builtin_return_address(0); 263 264 if (ppc_md.ioremap) 265 return ppc_md.ioremap(addr, size, flags, caller); 266 return __ioremap_caller(addr, size, flags, caller); 267 } 268 269 void __iomem * ioremap_prot(phys_addr_t addr, unsigned long size, 270 unsigned long flags) 271 { 272 void *caller = __builtin_return_address(0); 273 274 /* writeable implies dirty for kernel addresses */ 275 if (flags & _PAGE_RW) 276 flags |= _PAGE_DIRTY; 277 278 /* we don't want to let _PAGE_USER and _PAGE_EXEC leak out */ 279 flags &= ~(_PAGE_USER | _PAGE_EXEC); 280 281 #ifdef _PAGE_BAP_SR 282 /* _PAGE_USER contains _PAGE_BAP_SR on BookE using the new PTE format 283 * which means that we just cleared supervisor access... oops ;-) This 284 * restores it 285 */ 286 flags |= _PAGE_BAP_SR; 287 #endif 288 289 if (ppc_md.ioremap) 290 return ppc_md.ioremap(addr, size, flags, caller); 291 return __ioremap_caller(addr, size, flags, caller); 292 } 293 294 295 /* 296 * Unmap an IO region and remove it from imalloc'd list. 297 * Access to IO memory should be serialized by driver. 298 */ 299 void __iounmap(volatile void __iomem *token) 300 { 301 void *addr; 302 303 if (!mem_init_done) 304 return; 305 306 addr = (void *) ((unsigned long __force) 307 PCI_FIX_ADDR(token) & PAGE_MASK); 308 if ((unsigned long)addr < ioremap_bot) { 309 printk(KERN_WARNING "Attempt to iounmap early bolted mapping" 310 " at 0x%p\n", addr); 311 return; 312 } 313 vunmap(addr); 314 } 315 316 void iounmap(volatile void __iomem *token) 317 { 318 if (ppc_md.iounmap) 319 ppc_md.iounmap(token); 320 else 321 __iounmap(token); 322 } 323 324 EXPORT_SYMBOL(ioremap); 325 EXPORT_SYMBOL(ioremap_wc); 326 EXPORT_SYMBOL(ioremap_prot); 327 EXPORT_SYMBOL(__ioremap); 328 EXPORT_SYMBOL(__ioremap_at); 329 EXPORT_SYMBOL(iounmap); 330 EXPORT_SYMBOL(__iounmap); 331 EXPORT_SYMBOL(__iounmap_at); 332