1 /* 2 * srmmu.c: SRMMU specific routines for memory management. 3 * 4 * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) 5 * Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com) 6 * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) 7 * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) 8 * Copyright (C) 1999,2000 Anton Blanchard (anton@samba.org) 9 */ 10 11 #include <linux/kernel.h> 12 #include <linux/mm.h> 13 #include <linux/vmalloc.h> 14 #include <linux/pagemap.h> 15 #include <linux/init.h> 16 #include <linux/spinlock.h> 17 #include <linux/bootmem.h> 18 #include <linux/fs.h> 19 #include <linux/seq_file.h> 20 #include <linux/kdebug.h> 21 #include <linux/log2.h> 22 #include <linux/gfp.h> 23 24 #include <asm/bitext.h> 25 #include <asm/page.h> 26 #include <asm/pgalloc.h> 27 #include <asm/pgtable.h> 28 #include <asm/io.h> 29 #include <asm/vaddrs.h> 30 #include <asm/traps.h> 31 #include <asm/smp.h> 32 #include <asm/mbus.h> 33 #include <asm/cache.h> 34 #include <asm/oplib.h> 35 #include <asm/asi.h> 36 #include <asm/msi.h> 37 #include <asm/mmu_context.h> 38 #include <asm/io-unit.h> 39 #include <asm/cacheflush.h> 40 #include <asm/tlbflush.h> 41 42 /* Now the cpu specific definitions. */ 43 #include <asm/viking.h> 44 #include <asm/mxcc.h> 45 #include <asm/ross.h> 46 #include <asm/tsunami.h> 47 #include <asm/swift.h> 48 #include <asm/turbosparc.h> 49 #include <asm/leon.h> 50 51 #include "srmmu.h" 52 53 enum mbus_module srmmu_modtype; 54 static unsigned int hwbug_bitmask; 55 int vac_cache_size; 56 int vac_line_size; 57 58 struct ctx_list *ctx_list_pool; 59 struct ctx_list ctx_free; 60 struct ctx_list ctx_used; 61 62 extern struct resource sparc_iomap; 63 64 extern unsigned long last_valid_pfn; 65 66 static pgd_t *srmmu_swapper_pg_dir; 67 68 const struct sparc32_cachetlb_ops *sparc32_cachetlb_ops; 69 70 #ifdef CONFIG_SMP 71 const struct sparc32_cachetlb_ops *local_ops; 72 73 #define FLUSH_BEGIN(mm) 74 #define FLUSH_END 75 #else 76 #define FLUSH_BEGIN(mm) if ((mm)->context != NO_CONTEXT) { 77 #define FLUSH_END } 78 #endif 79 80 int flush_page_for_dma_global = 1; 81 82 char *srmmu_name; 83 84 ctxd_t *srmmu_ctx_table_phys; 85 static ctxd_t *srmmu_context_table; 86 87 int viking_mxcc_present; 88 static DEFINE_SPINLOCK(srmmu_context_spinlock); 89 90 static int is_hypersparc; 91 92 static int srmmu_cache_pagetables; 93 94 /* these will be initialized in srmmu_nocache_calcsize() */ 95 static unsigned long srmmu_nocache_size; 96 static unsigned long srmmu_nocache_end; 97 98 /* 1 bit <=> 256 bytes of nocache <=> 64 PTEs */ 99 #define SRMMU_NOCACHE_BITMAP_SHIFT (PAGE_SHIFT - 4) 100 101 /* The context table is a nocache user with the biggest alignment needs. */ 102 #define SRMMU_NOCACHE_ALIGN_MAX (sizeof(ctxd_t)*SRMMU_MAX_CONTEXTS) 103 104 void *srmmu_nocache_pool; 105 void *srmmu_nocache_bitmap; 106 static struct bit_map srmmu_nocache_map; 107 108 static inline int srmmu_pmd_none(pmd_t pmd) 109 { return !(pmd_val(pmd) & 0xFFFFFFF); } 110 111 /* XXX should we hyper_flush_whole_icache here - Anton */ 112 static inline void srmmu_ctxd_set(ctxd_t *ctxp, pgd_t *pgdp) 113 { set_pte((pte_t *)ctxp, (SRMMU_ET_PTD | (__nocache_pa((unsigned long) pgdp) >> 4))); } 114 115 void pmd_set(pmd_t *pmdp, pte_t *ptep) 116 { 117 unsigned long ptp; /* Physical address, shifted right by 4 */ 118 int i; 119 120 ptp = __nocache_pa((unsigned long) ptep) >> 4; 121 for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) { 122 set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp); 123 ptp += (SRMMU_REAL_PTRS_PER_PTE*sizeof(pte_t) >> 4); 124 } 125 } 126 127 void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep) 128 { 129 unsigned long ptp; /* Physical address, shifted right by 4 */ 130 int i; 131 132 ptp = page_to_pfn(ptep) << (PAGE_SHIFT-4); /* watch for overflow */ 133 for (i = 0; i < PTRS_PER_PTE/SRMMU_REAL_PTRS_PER_PTE; i++) { 134 set_pte((pte_t *)&pmdp->pmdv[i], SRMMU_ET_PTD | ptp); 135 ptp += (SRMMU_REAL_PTRS_PER_PTE*sizeof(pte_t) >> 4); 136 } 137 } 138 139 /* Find an entry in the third-level page table.. */ 140 pte_t *pte_offset_kernel(pmd_t * dir, unsigned long address) 141 { 142 void *pte; 143 144 pte = __nocache_va((dir->pmdv[0] & SRMMU_PTD_PMASK) << 4); 145 return (pte_t *) pte + 146 ((address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1)); 147 } 148 149 /* 150 * size: bytes to allocate in the nocache area. 151 * align: bytes, number to align at. 152 * Returns the virtual address of the allocated area. 153 */ 154 static unsigned long __srmmu_get_nocache(int size, int align) 155 { 156 int offset; 157 158 if (size < SRMMU_NOCACHE_BITMAP_SHIFT) { 159 printk("Size 0x%x too small for nocache request\n", size); 160 size = SRMMU_NOCACHE_BITMAP_SHIFT; 161 } 162 if (size & (SRMMU_NOCACHE_BITMAP_SHIFT-1)) { 163 printk("Size 0x%x unaligned int nocache request\n", size); 164 size += SRMMU_NOCACHE_BITMAP_SHIFT-1; 165 } 166 BUG_ON(align > SRMMU_NOCACHE_ALIGN_MAX); 167 168 offset = bit_map_string_get(&srmmu_nocache_map, 169 size >> SRMMU_NOCACHE_BITMAP_SHIFT, 170 align >> SRMMU_NOCACHE_BITMAP_SHIFT); 171 if (offset == -1) { 172 printk("srmmu: out of nocache %d: %d/%d\n", 173 size, (int) srmmu_nocache_size, 174 srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT); 175 return 0; 176 } 177 178 return (SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT)); 179 } 180 181 unsigned long srmmu_get_nocache(int size, int align) 182 { 183 unsigned long tmp; 184 185 tmp = __srmmu_get_nocache(size, align); 186 187 if (tmp) 188 memset((void *)tmp, 0, size); 189 190 return tmp; 191 } 192 193 void srmmu_free_nocache(unsigned long vaddr, int size) 194 { 195 int offset; 196 197 if (vaddr < SRMMU_NOCACHE_VADDR) { 198 printk("Vaddr %lx is smaller than nocache base 0x%lx\n", 199 vaddr, (unsigned long)SRMMU_NOCACHE_VADDR); 200 BUG(); 201 } 202 if (vaddr+size > srmmu_nocache_end) { 203 printk("Vaddr %lx is bigger than nocache end 0x%lx\n", 204 vaddr, srmmu_nocache_end); 205 BUG(); 206 } 207 if (!is_power_of_2(size)) { 208 printk("Size 0x%x is not a power of 2\n", size); 209 BUG(); 210 } 211 if (size < SRMMU_NOCACHE_BITMAP_SHIFT) { 212 printk("Size 0x%x is too small\n", size); 213 BUG(); 214 } 215 if (vaddr & (size-1)) { 216 printk("Vaddr %lx is not aligned to size 0x%x\n", vaddr, size); 217 BUG(); 218 } 219 220 offset = (vaddr - SRMMU_NOCACHE_VADDR) >> SRMMU_NOCACHE_BITMAP_SHIFT; 221 size = size >> SRMMU_NOCACHE_BITMAP_SHIFT; 222 223 bit_map_clear(&srmmu_nocache_map, offset, size); 224 } 225 226 static void srmmu_early_allocate_ptable_skeleton(unsigned long start, 227 unsigned long end); 228 229 extern unsigned long probe_memory(void); /* in fault.c */ 230 231 /* 232 * Reserve nocache dynamically proportionally to the amount of 233 * system RAM. -- Tomas Szepe <szepe@pinerecords.com>, June 2002 234 */ 235 static void srmmu_nocache_calcsize(void) 236 { 237 unsigned long sysmemavail = probe_memory() / 1024; 238 int srmmu_nocache_npages; 239 240 srmmu_nocache_npages = 241 sysmemavail / SRMMU_NOCACHE_ALCRATIO / 1024 * 256; 242 243 /* P3 XXX The 4x overuse: corroborated by /proc/meminfo. */ 244 // if (srmmu_nocache_npages < 256) srmmu_nocache_npages = 256; 245 if (srmmu_nocache_npages < SRMMU_MIN_NOCACHE_PAGES) 246 srmmu_nocache_npages = SRMMU_MIN_NOCACHE_PAGES; 247 248 /* anything above 1280 blows up */ 249 if (srmmu_nocache_npages > SRMMU_MAX_NOCACHE_PAGES) 250 srmmu_nocache_npages = SRMMU_MAX_NOCACHE_PAGES; 251 252 srmmu_nocache_size = srmmu_nocache_npages * PAGE_SIZE; 253 srmmu_nocache_end = SRMMU_NOCACHE_VADDR + srmmu_nocache_size; 254 } 255 256 static void __init srmmu_nocache_init(void) 257 { 258 unsigned int bitmap_bits; 259 pgd_t *pgd; 260 pmd_t *pmd; 261 pte_t *pte; 262 unsigned long paddr, vaddr; 263 unsigned long pteval; 264 265 bitmap_bits = srmmu_nocache_size >> SRMMU_NOCACHE_BITMAP_SHIFT; 266 267 srmmu_nocache_pool = __alloc_bootmem(srmmu_nocache_size, 268 SRMMU_NOCACHE_ALIGN_MAX, 0UL); 269 memset(srmmu_nocache_pool, 0, srmmu_nocache_size); 270 271 srmmu_nocache_bitmap = __alloc_bootmem(bitmap_bits >> 3, SMP_CACHE_BYTES, 0UL); 272 bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits); 273 274 srmmu_swapper_pg_dir = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); 275 memset(__nocache_fix(srmmu_swapper_pg_dir), 0, SRMMU_PGD_TABLE_SIZE); 276 init_mm.pgd = srmmu_swapper_pg_dir; 277 278 srmmu_early_allocate_ptable_skeleton(SRMMU_NOCACHE_VADDR, srmmu_nocache_end); 279 280 paddr = __pa((unsigned long)srmmu_nocache_pool); 281 vaddr = SRMMU_NOCACHE_VADDR; 282 283 while (vaddr < srmmu_nocache_end) { 284 pgd = pgd_offset_k(vaddr); 285 pmd = pmd_offset(__nocache_fix(pgd), vaddr); 286 pte = pte_offset_kernel(__nocache_fix(pmd), vaddr); 287 288 pteval = ((paddr >> 4) | SRMMU_ET_PTE | SRMMU_PRIV); 289 290 if (srmmu_cache_pagetables) 291 pteval |= SRMMU_CACHE; 292 293 set_pte(__nocache_fix(pte), __pte(pteval)); 294 295 vaddr += PAGE_SIZE; 296 paddr += PAGE_SIZE; 297 } 298 299 flush_cache_all(); 300 flush_tlb_all(); 301 } 302 303 pgd_t *get_pgd_fast(void) 304 { 305 pgd_t *pgd = NULL; 306 307 pgd = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); 308 if (pgd) { 309 pgd_t *init = pgd_offset_k(0); 310 memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); 311 memcpy(pgd + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD, 312 (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); 313 } 314 315 return pgd; 316 } 317 318 /* 319 * Hardware needs alignment to 256 only, but we align to whole page size 320 * to reduce fragmentation problems due to the buddy principle. 321 * XXX Provide actual fragmentation statistics in /proc. 322 * 323 * Alignments up to the page size are the same for physical and virtual 324 * addresses of the nocache area. 325 */ 326 pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) 327 { 328 unsigned long pte; 329 struct page *page; 330 331 if ((pte = (unsigned long)pte_alloc_one_kernel(mm, address)) == 0) 332 return NULL; 333 page = pfn_to_page( __nocache_pa(pte) >> PAGE_SHIFT ); 334 pgtable_page_ctor(page); 335 return page; 336 } 337 338 void pte_free(struct mm_struct *mm, pgtable_t pte) 339 { 340 unsigned long p; 341 342 pgtable_page_dtor(pte); 343 p = (unsigned long)page_address(pte); /* Cached address (for test) */ 344 if (p == 0) 345 BUG(); 346 p = page_to_pfn(pte) << PAGE_SHIFT; /* Physical address */ 347 p = (unsigned long) __nocache_va(p); /* Nocached virtual */ 348 srmmu_free_nocache(p, PTE_SIZE); 349 } 350 351 /* 352 */ 353 static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm) 354 { 355 struct ctx_list *ctxp; 356 357 ctxp = ctx_free.next; 358 if(ctxp != &ctx_free) { 359 remove_from_ctx_list(ctxp); 360 add_to_used_ctxlist(ctxp); 361 mm->context = ctxp->ctx_number; 362 ctxp->ctx_mm = mm; 363 return; 364 } 365 ctxp = ctx_used.next; 366 if(ctxp->ctx_mm == old_mm) 367 ctxp = ctxp->next; 368 if(ctxp == &ctx_used) 369 panic("out of mmu contexts"); 370 flush_cache_mm(ctxp->ctx_mm); 371 flush_tlb_mm(ctxp->ctx_mm); 372 remove_from_ctx_list(ctxp); 373 add_to_used_ctxlist(ctxp); 374 ctxp->ctx_mm->context = NO_CONTEXT; 375 ctxp->ctx_mm = mm; 376 mm->context = ctxp->ctx_number; 377 } 378 379 static inline void free_context(int context) 380 { 381 struct ctx_list *ctx_old; 382 383 ctx_old = ctx_list_pool + context; 384 remove_from_ctx_list(ctx_old); 385 add_to_free_ctxlist(ctx_old); 386 } 387 388 389 void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, 390 struct task_struct *tsk) 391 { 392 if(mm->context == NO_CONTEXT) { 393 spin_lock(&srmmu_context_spinlock); 394 alloc_context(old_mm, mm); 395 spin_unlock(&srmmu_context_spinlock); 396 srmmu_ctxd_set(&srmmu_context_table[mm->context], mm->pgd); 397 } 398 399 if (sparc_cpu_model == sparc_leon) 400 leon_switch_mm(); 401 402 if (is_hypersparc) 403 hyper_flush_whole_icache(); 404 405 srmmu_set_context(mm->context); 406 } 407 408 /* Low level IO area allocation on the SRMMU. */ 409 static inline void srmmu_mapioaddr(unsigned long physaddr, 410 unsigned long virt_addr, int bus_type) 411 { 412 pgd_t *pgdp; 413 pmd_t *pmdp; 414 pte_t *ptep; 415 unsigned long tmp; 416 417 physaddr &= PAGE_MASK; 418 pgdp = pgd_offset_k(virt_addr); 419 pmdp = pmd_offset(pgdp, virt_addr); 420 ptep = pte_offset_kernel(pmdp, virt_addr); 421 tmp = (physaddr >> 4) | SRMMU_ET_PTE; 422 423 /* 424 * I need to test whether this is consistent over all 425 * sun4m's. The bus_type represents the upper 4 bits of 426 * 36-bit physical address on the I/O space lines... 427 */ 428 tmp |= (bus_type << 28); 429 tmp |= SRMMU_PRIV; 430 __flush_page_to_ram(virt_addr); 431 set_pte(ptep, __pte(tmp)); 432 } 433 434 void srmmu_mapiorange(unsigned int bus, unsigned long xpa, 435 unsigned long xva, unsigned int len) 436 { 437 while (len != 0) { 438 len -= PAGE_SIZE; 439 srmmu_mapioaddr(xpa, xva, bus); 440 xva += PAGE_SIZE; 441 xpa += PAGE_SIZE; 442 } 443 flush_tlb_all(); 444 } 445 446 static inline void srmmu_unmapioaddr(unsigned long virt_addr) 447 { 448 pgd_t *pgdp; 449 pmd_t *pmdp; 450 pte_t *ptep; 451 452 pgdp = pgd_offset_k(virt_addr); 453 pmdp = pmd_offset(pgdp, virt_addr); 454 ptep = pte_offset_kernel(pmdp, virt_addr); 455 456 /* No need to flush uncacheable page. */ 457 __pte_clear(ptep); 458 } 459 460 void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len) 461 { 462 while (len != 0) { 463 len -= PAGE_SIZE; 464 srmmu_unmapioaddr(virt_addr); 465 virt_addr += PAGE_SIZE; 466 } 467 flush_tlb_all(); 468 } 469 470 /* tsunami.S */ 471 extern void tsunami_flush_cache_all(void); 472 extern void tsunami_flush_cache_mm(struct mm_struct *mm); 473 extern void tsunami_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); 474 extern void tsunami_flush_cache_page(struct vm_area_struct *vma, unsigned long page); 475 extern void tsunami_flush_page_to_ram(unsigned long page); 476 extern void tsunami_flush_page_for_dma(unsigned long page); 477 extern void tsunami_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr); 478 extern void tsunami_flush_tlb_all(void); 479 extern void tsunami_flush_tlb_mm(struct mm_struct *mm); 480 extern void tsunami_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); 481 extern void tsunami_flush_tlb_page(struct vm_area_struct *vma, unsigned long page); 482 extern void tsunami_setup_blockops(void); 483 484 /* swift.S */ 485 extern void swift_flush_cache_all(void); 486 extern void swift_flush_cache_mm(struct mm_struct *mm); 487 extern void swift_flush_cache_range(struct vm_area_struct *vma, 488 unsigned long start, unsigned long end); 489 extern void swift_flush_cache_page(struct vm_area_struct *vma, unsigned long page); 490 extern void swift_flush_page_to_ram(unsigned long page); 491 extern void swift_flush_page_for_dma(unsigned long page); 492 extern void swift_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr); 493 extern void swift_flush_tlb_all(void); 494 extern void swift_flush_tlb_mm(struct mm_struct *mm); 495 extern void swift_flush_tlb_range(struct vm_area_struct *vma, 496 unsigned long start, unsigned long end); 497 extern void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page); 498 499 #if 0 /* P3: deadwood to debug precise flushes on Swift. */ 500 void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 501 { 502 int cctx, ctx1; 503 504 page &= PAGE_MASK; 505 if ((ctx1 = vma->vm_mm->context) != -1) { 506 cctx = srmmu_get_context(); 507 /* Is context # ever different from current context? P3 */ 508 if (cctx != ctx1) { 509 printk("flush ctx %02x curr %02x\n", ctx1, cctx); 510 srmmu_set_context(ctx1); 511 swift_flush_page(page); 512 __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : : 513 "r" (page), "i" (ASI_M_FLUSH_PROBE)); 514 srmmu_set_context(cctx); 515 } else { 516 /* Rm. prot. bits from virt. c. */ 517 /* swift_flush_cache_all(); */ 518 /* swift_flush_cache_page(vma, page); */ 519 swift_flush_page(page); 520 521 __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : : 522 "r" (page), "i" (ASI_M_FLUSH_PROBE)); 523 /* same as above: srmmu_flush_tlb_page() */ 524 } 525 } 526 } 527 #endif 528 529 /* 530 * The following are all MBUS based SRMMU modules, and therefore could 531 * be found in a multiprocessor configuration. On the whole, these 532 * chips seems to be much more touchy about DVMA and page tables 533 * with respect to cache coherency. 534 */ 535 536 /* viking.S */ 537 extern void viking_flush_cache_all(void); 538 extern void viking_flush_cache_mm(struct mm_struct *mm); 539 extern void viking_flush_cache_range(struct vm_area_struct *vma, unsigned long start, 540 unsigned long end); 541 extern void viking_flush_cache_page(struct vm_area_struct *vma, unsigned long page); 542 extern void viking_flush_page_to_ram(unsigned long page); 543 extern void viking_flush_page_for_dma(unsigned long page); 544 extern void viking_flush_sig_insns(struct mm_struct *mm, unsigned long addr); 545 extern void viking_flush_page(unsigned long page); 546 extern void viking_mxcc_flush_page(unsigned long page); 547 extern void viking_flush_tlb_all(void); 548 extern void viking_flush_tlb_mm(struct mm_struct *mm); 549 extern void viking_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 550 unsigned long end); 551 extern void viking_flush_tlb_page(struct vm_area_struct *vma, 552 unsigned long page); 553 extern void sun4dsmp_flush_tlb_all(void); 554 extern void sun4dsmp_flush_tlb_mm(struct mm_struct *mm); 555 extern void sun4dsmp_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 556 unsigned long end); 557 extern void sun4dsmp_flush_tlb_page(struct vm_area_struct *vma, 558 unsigned long page); 559 560 /* hypersparc.S */ 561 extern void hypersparc_flush_cache_all(void); 562 extern void hypersparc_flush_cache_mm(struct mm_struct *mm); 563 extern void hypersparc_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); 564 extern void hypersparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page); 565 extern void hypersparc_flush_page_to_ram(unsigned long page); 566 extern void hypersparc_flush_page_for_dma(unsigned long page); 567 extern void hypersparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr); 568 extern void hypersparc_flush_tlb_all(void); 569 extern void hypersparc_flush_tlb_mm(struct mm_struct *mm); 570 extern void hypersparc_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); 571 extern void hypersparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page); 572 extern void hypersparc_setup_blockops(void); 573 574 /* 575 * NOTE: All of this startup code assumes the low 16mb (approx.) of 576 * kernel mappings are done with one single contiguous chunk of 577 * ram. On small ram machines (classics mainly) we only get 578 * around 8mb mapped for us. 579 */ 580 581 static void __init early_pgtable_allocfail(char *type) 582 { 583 prom_printf("inherit_prom_mappings: Cannot alloc kernel %s.\n", type); 584 prom_halt(); 585 } 586 587 static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start, 588 unsigned long end) 589 { 590 pgd_t *pgdp; 591 pmd_t *pmdp; 592 pte_t *ptep; 593 594 while(start < end) { 595 pgdp = pgd_offset_k(start); 596 if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) { 597 pmdp = (pmd_t *) __srmmu_get_nocache( 598 SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); 599 if (pmdp == NULL) 600 early_pgtable_allocfail("pmd"); 601 memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE); 602 pgd_set(__nocache_fix(pgdp), pmdp); 603 } 604 pmdp = pmd_offset(__nocache_fix(pgdp), start); 605 if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { 606 ptep = (pte_t *)__srmmu_get_nocache(PTE_SIZE, PTE_SIZE); 607 if (ptep == NULL) 608 early_pgtable_allocfail("pte"); 609 memset(__nocache_fix(ptep), 0, PTE_SIZE); 610 pmd_set(__nocache_fix(pmdp), ptep); 611 } 612 if (start > (0xffffffffUL - PMD_SIZE)) 613 break; 614 start = (start + PMD_SIZE) & PMD_MASK; 615 } 616 } 617 618 static void __init srmmu_allocate_ptable_skeleton(unsigned long start, 619 unsigned long end) 620 { 621 pgd_t *pgdp; 622 pmd_t *pmdp; 623 pte_t *ptep; 624 625 while(start < end) { 626 pgdp = pgd_offset_k(start); 627 if (pgd_none(*pgdp)) { 628 pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); 629 if (pmdp == NULL) 630 early_pgtable_allocfail("pmd"); 631 memset(pmdp, 0, SRMMU_PMD_TABLE_SIZE); 632 pgd_set(pgdp, pmdp); 633 } 634 pmdp = pmd_offset(pgdp, start); 635 if(srmmu_pmd_none(*pmdp)) { 636 ptep = (pte_t *) __srmmu_get_nocache(PTE_SIZE, 637 PTE_SIZE); 638 if (ptep == NULL) 639 early_pgtable_allocfail("pte"); 640 memset(ptep, 0, PTE_SIZE); 641 pmd_set(pmdp, ptep); 642 } 643 if (start > (0xffffffffUL - PMD_SIZE)) 644 break; 645 start = (start + PMD_SIZE) & PMD_MASK; 646 } 647 } 648 649 /* These flush types are not available on all chips... */ 650 static inline unsigned long srmmu_probe(unsigned long vaddr) 651 { 652 unsigned long retval; 653 654 if (sparc_cpu_model != sparc_leon) { 655 656 vaddr &= PAGE_MASK; 657 __asm__ __volatile__("lda [%1] %2, %0\n\t" : 658 "=r" (retval) : 659 "r" (vaddr | 0x400), "i" (ASI_M_FLUSH_PROBE)); 660 } else { 661 retval = leon_swprobe(vaddr, 0); 662 } 663 return retval; 664 } 665 666 /* 667 * This is much cleaner than poking around physical address space 668 * looking at the prom's page table directly which is what most 669 * other OS's do. Yuck... this is much better. 670 */ 671 static void __init srmmu_inherit_prom_mappings(unsigned long start, 672 unsigned long end) 673 { 674 pgd_t *pgdp; 675 pmd_t *pmdp; 676 pte_t *ptep; 677 int what = 0; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */ 678 unsigned long prompte; 679 680 while(start <= end) { 681 if (start == 0) 682 break; /* probably wrap around */ 683 if(start == 0xfef00000) 684 start = KADB_DEBUGGER_BEGVM; 685 if(!(prompte = srmmu_probe(start))) { 686 start += PAGE_SIZE; 687 continue; 688 } 689 690 /* A red snapper, see what it really is. */ 691 what = 0; 692 693 if(!(start & ~(SRMMU_REAL_PMD_MASK))) { 694 if(srmmu_probe((start-PAGE_SIZE) + SRMMU_REAL_PMD_SIZE) == prompte) 695 what = 1; 696 } 697 698 if(!(start & ~(SRMMU_PGDIR_MASK))) { 699 if(srmmu_probe((start-PAGE_SIZE) + SRMMU_PGDIR_SIZE) == 700 prompte) 701 what = 2; 702 } 703 704 pgdp = pgd_offset_k(start); 705 if(what == 2) { 706 *(pgd_t *)__nocache_fix(pgdp) = __pgd(prompte); 707 start += SRMMU_PGDIR_SIZE; 708 continue; 709 } 710 if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) { 711 pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); 712 if (pmdp == NULL) 713 early_pgtable_allocfail("pmd"); 714 memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE); 715 pgd_set(__nocache_fix(pgdp), pmdp); 716 } 717 pmdp = pmd_offset(__nocache_fix(pgdp), start); 718 if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { 719 ptep = (pte_t *) __srmmu_get_nocache(PTE_SIZE, 720 PTE_SIZE); 721 if (ptep == NULL) 722 early_pgtable_allocfail("pte"); 723 memset(__nocache_fix(ptep), 0, PTE_SIZE); 724 pmd_set(__nocache_fix(pmdp), ptep); 725 } 726 if(what == 1) { 727 /* 728 * We bend the rule where all 16 PTPs in a pmd_t point 729 * inside the same PTE page, and we leak a perfectly 730 * good hardware PTE piece. Alternatives seem worse. 731 */ 732 unsigned int x; /* Index of HW PMD in soft cluster */ 733 x = (start >> PMD_SHIFT) & 15; 734 *(unsigned long *)__nocache_fix(&pmdp->pmdv[x]) = prompte; 735 start += SRMMU_REAL_PMD_SIZE; 736 continue; 737 } 738 ptep = pte_offset_kernel(__nocache_fix(pmdp), start); 739 *(pte_t *)__nocache_fix(ptep) = __pte(prompte); 740 start += PAGE_SIZE; 741 } 742 } 743 744 #define KERNEL_PTE(page_shifted) ((page_shifted)|SRMMU_CACHE|SRMMU_PRIV|SRMMU_VALID) 745 746 /* Create a third-level SRMMU 16MB page mapping. */ 747 static void __init do_large_mapping(unsigned long vaddr, unsigned long phys_base) 748 { 749 pgd_t *pgdp = pgd_offset_k(vaddr); 750 unsigned long big_pte; 751 752 big_pte = KERNEL_PTE(phys_base >> 4); 753 *(pgd_t *)__nocache_fix(pgdp) = __pgd(big_pte); 754 } 755 756 /* Map sp_bank entry SP_ENTRY, starting at virtual address VBASE. */ 757 static unsigned long __init map_spbank(unsigned long vbase, int sp_entry) 758 { 759 unsigned long pstart = (sp_banks[sp_entry].base_addr & SRMMU_PGDIR_MASK); 760 unsigned long vstart = (vbase & SRMMU_PGDIR_MASK); 761 unsigned long vend = SRMMU_PGDIR_ALIGN(vbase + sp_banks[sp_entry].num_bytes); 762 /* Map "low" memory only */ 763 const unsigned long min_vaddr = PAGE_OFFSET; 764 const unsigned long max_vaddr = PAGE_OFFSET + SRMMU_MAXMEM; 765 766 if (vstart < min_vaddr || vstart >= max_vaddr) 767 return vstart; 768 769 if (vend > max_vaddr || vend < min_vaddr) 770 vend = max_vaddr; 771 772 while(vstart < vend) { 773 do_large_mapping(vstart, pstart); 774 vstart += SRMMU_PGDIR_SIZE; pstart += SRMMU_PGDIR_SIZE; 775 } 776 return vstart; 777 } 778 779 static inline void map_kernel(void) 780 { 781 int i; 782 783 if (phys_base > 0) { 784 do_large_mapping(PAGE_OFFSET, phys_base); 785 } 786 787 for (i = 0; sp_banks[i].num_bytes != 0; i++) { 788 map_spbank((unsigned long)__va(sp_banks[i].base_addr), i); 789 } 790 } 791 792 /* Paging initialization on the Sparc Reference MMU. */ 793 extern void sparc_context_init(int); 794 795 void (*poke_srmmu)(void) __cpuinitdata = NULL; 796 797 extern unsigned long bootmem_init(unsigned long *pages_avail); 798 799 void __init srmmu_paging_init(void) 800 { 801 int i; 802 phandle cpunode; 803 char node_str[128]; 804 pgd_t *pgd; 805 pmd_t *pmd; 806 pte_t *pte; 807 unsigned long pages_avail; 808 809 sparc_iomap.start = SUN4M_IOBASE_VADDR; /* 16MB of IOSPACE on all sun4m's. */ 810 811 if (sparc_cpu_model == sun4d) 812 num_contexts = 65536; /* We know it is Viking */ 813 else { 814 /* Find the number of contexts on the srmmu. */ 815 cpunode = prom_getchild(prom_root_node); 816 num_contexts = 0; 817 while(cpunode != 0) { 818 prom_getstring(cpunode, "device_type", node_str, sizeof(node_str)); 819 if(!strcmp(node_str, "cpu")) { 820 num_contexts = prom_getintdefault(cpunode, "mmu-nctx", 0x8); 821 break; 822 } 823 cpunode = prom_getsibling(cpunode); 824 } 825 } 826 827 if(!num_contexts) { 828 prom_printf("Something wrong, can't find cpu node in paging_init.\n"); 829 prom_halt(); 830 } 831 832 pages_avail = 0; 833 last_valid_pfn = bootmem_init(&pages_avail); 834 835 srmmu_nocache_calcsize(); 836 srmmu_nocache_init(); 837 srmmu_inherit_prom_mappings(0xfe400000,(LINUX_OPPROM_ENDVM-PAGE_SIZE)); 838 map_kernel(); 839 840 /* ctx table has to be physically aligned to its size */ 841 srmmu_context_table = (ctxd_t *)__srmmu_get_nocache(num_contexts*sizeof(ctxd_t), num_contexts*sizeof(ctxd_t)); 842 srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa((unsigned long)srmmu_context_table); 843 844 for(i = 0; i < num_contexts; i++) 845 srmmu_ctxd_set((ctxd_t *)__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir); 846 847 flush_cache_all(); 848 srmmu_set_ctable_ptr((unsigned long)srmmu_ctx_table_phys); 849 #ifdef CONFIG_SMP 850 /* Stop from hanging here... */ 851 local_ops->tlb_all(); 852 #else 853 flush_tlb_all(); 854 #endif 855 poke_srmmu(); 856 857 srmmu_allocate_ptable_skeleton(sparc_iomap.start, IOBASE_END); 858 srmmu_allocate_ptable_skeleton(DVMA_VADDR, DVMA_END); 859 860 srmmu_allocate_ptable_skeleton( 861 __fix_to_virt(__end_of_fixed_addresses - 1), FIXADDR_TOP); 862 srmmu_allocate_ptable_skeleton(PKMAP_BASE, PKMAP_END); 863 864 pgd = pgd_offset_k(PKMAP_BASE); 865 pmd = pmd_offset(pgd, PKMAP_BASE); 866 pte = pte_offset_kernel(pmd, PKMAP_BASE); 867 pkmap_page_table = pte; 868 869 flush_cache_all(); 870 flush_tlb_all(); 871 872 sparc_context_init(num_contexts); 873 874 kmap_init(); 875 876 { 877 unsigned long zones_size[MAX_NR_ZONES]; 878 unsigned long zholes_size[MAX_NR_ZONES]; 879 unsigned long npages; 880 int znum; 881 882 for (znum = 0; znum < MAX_NR_ZONES; znum++) 883 zones_size[znum] = zholes_size[znum] = 0; 884 885 npages = max_low_pfn - pfn_base; 886 887 zones_size[ZONE_DMA] = npages; 888 zholes_size[ZONE_DMA] = npages - pages_avail; 889 890 npages = highend_pfn - max_low_pfn; 891 zones_size[ZONE_HIGHMEM] = npages; 892 zholes_size[ZONE_HIGHMEM] = npages - calc_highpages(); 893 894 free_area_init_node(0, zones_size, pfn_base, zholes_size); 895 } 896 } 897 898 void mmu_info(struct seq_file *m) 899 { 900 seq_printf(m, 901 "MMU type\t: %s\n" 902 "contexts\t: %d\n" 903 "nocache total\t: %ld\n" 904 "nocache used\t: %d\n", 905 srmmu_name, 906 num_contexts, 907 srmmu_nocache_size, 908 srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT); 909 } 910 911 void destroy_context(struct mm_struct *mm) 912 { 913 914 if(mm->context != NO_CONTEXT) { 915 flush_cache_mm(mm); 916 srmmu_ctxd_set(&srmmu_context_table[mm->context], srmmu_swapper_pg_dir); 917 flush_tlb_mm(mm); 918 spin_lock(&srmmu_context_spinlock); 919 free_context(mm->context); 920 spin_unlock(&srmmu_context_spinlock); 921 mm->context = NO_CONTEXT; 922 } 923 } 924 925 /* Init various srmmu chip types. */ 926 static void __init srmmu_is_bad(void) 927 { 928 prom_printf("Could not determine SRMMU chip type.\n"); 929 prom_halt(); 930 } 931 932 static void __init init_vac_layout(void) 933 { 934 phandle nd; 935 int cache_lines; 936 char node_str[128]; 937 #ifdef CONFIG_SMP 938 int cpu = 0; 939 unsigned long max_size = 0; 940 unsigned long min_line_size = 0x10000000; 941 #endif 942 943 nd = prom_getchild(prom_root_node); 944 while((nd = prom_getsibling(nd)) != 0) { 945 prom_getstring(nd, "device_type", node_str, sizeof(node_str)); 946 if(!strcmp(node_str, "cpu")) { 947 vac_line_size = prom_getint(nd, "cache-line-size"); 948 if (vac_line_size == -1) { 949 prom_printf("can't determine cache-line-size, " 950 "halting.\n"); 951 prom_halt(); 952 } 953 cache_lines = prom_getint(nd, "cache-nlines"); 954 if (cache_lines == -1) { 955 prom_printf("can't determine cache-nlines, halting.\n"); 956 prom_halt(); 957 } 958 959 vac_cache_size = cache_lines * vac_line_size; 960 #ifdef CONFIG_SMP 961 if(vac_cache_size > max_size) 962 max_size = vac_cache_size; 963 if(vac_line_size < min_line_size) 964 min_line_size = vac_line_size; 965 //FIXME: cpus not contiguous!! 966 cpu++; 967 if (cpu >= nr_cpu_ids || !cpu_online(cpu)) 968 break; 969 #else 970 break; 971 #endif 972 } 973 } 974 if(nd == 0) { 975 prom_printf("No CPU nodes found, halting.\n"); 976 prom_halt(); 977 } 978 #ifdef CONFIG_SMP 979 vac_cache_size = max_size; 980 vac_line_size = min_line_size; 981 #endif 982 printk("SRMMU: Using VAC size of %d bytes, line size %d bytes.\n", 983 (int)vac_cache_size, (int)vac_line_size); 984 } 985 986 static void __cpuinit poke_hypersparc(void) 987 { 988 volatile unsigned long clear; 989 unsigned long mreg = srmmu_get_mmureg(); 990 991 hyper_flush_unconditional_combined(); 992 993 mreg &= ~(HYPERSPARC_CWENABLE); 994 mreg |= (HYPERSPARC_CENABLE | HYPERSPARC_WBENABLE); 995 mreg |= (HYPERSPARC_CMODE); 996 997 srmmu_set_mmureg(mreg); 998 999 #if 0 /* XXX I think this is bad news... -DaveM */ 1000 hyper_clear_all_tags(); 1001 #endif 1002 1003 put_ross_icr(HYPERSPARC_ICCR_FTD | HYPERSPARC_ICCR_ICE); 1004 hyper_flush_whole_icache(); 1005 clear = srmmu_get_faddr(); 1006 clear = srmmu_get_fstatus(); 1007 } 1008 1009 static const struct sparc32_cachetlb_ops hypersparc_ops = { 1010 .cache_all = hypersparc_flush_cache_all, 1011 .cache_mm = hypersparc_flush_cache_mm, 1012 .cache_page = hypersparc_flush_cache_page, 1013 .cache_range = hypersparc_flush_cache_range, 1014 .tlb_all = hypersparc_flush_tlb_all, 1015 .tlb_mm = hypersparc_flush_tlb_mm, 1016 .tlb_page = hypersparc_flush_tlb_page, 1017 .tlb_range = hypersparc_flush_tlb_range, 1018 .page_to_ram = hypersparc_flush_page_to_ram, 1019 .sig_insns = hypersparc_flush_sig_insns, 1020 .page_for_dma = hypersparc_flush_page_for_dma, 1021 }; 1022 1023 static void __init init_hypersparc(void) 1024 { 1025 srmmu_name = "ROSS HyperSparc"; 1026 srmmu_modtype = HyperSparc; 1027 1028 init_vac_layout(); 1029 1030 is_hypersparc = 1; 1031 sparc32_cachetlb_ops = &hypersparc_ops; 1032 1033 poke_srmmu = poke_hypersparc; 1034 1035 hypersparc_setup_blockops(); 1036 } 1037 1038 static void __cpuinit poke_swift(void) 1039 { 1040 unsigned long mreg; 1041 1042 /* Clear any crap from the cache or else... */ 1043 swift_flush_cache_all(); 1044 1045 /* Enable I & D caches */ 1046 mreg = srmmu_get_mmureg(); 1047 mreg |= (SWIFT_IE | SWIFT_DE); 1048 /* 1049 * The Swift branch folding logic is completely broken. At 1050 * trap time, if things are just right, if can mistakenly 1051 * think that a trap is coming from kernel mode when in fact 1052 * it is coming from user mode (it mis-executes the branch in 1053 * the trap code). So you see things like crashme completely 1054 * hosing your machine which is completely unacceptable. Turn 1055 * this shit off... nice job Fujitsu. 1056 */ 1057 mreg &= ~(SWIFT_BF); 1058 srmmu_set_mmureg(mreg); 1059 } 1060 1061 static const struct sparc32_cachetlb_ops swift_ops = { 1062 .cache_all = swift_flush_cache_all, 1063 .cache_mm = swift_flush_cache_mm, 1064 .cache_page = swift_flush_cache_page, 1065 .cache_range = swift_flush_cache_range, 1066 .tlb_all = swift_flush_tlb_all, 1067 .tlb_mm = swift_flush_tlb_mm, 1068 .tlb_page = swift_flush_tlb_page, 1069 .tlb_range = swift_flush_tlb_range, 1070 .page_to_ram = swift_flush_page_to_ram, 1071 .sig_insns = swift_flush_sig_insns, 1072 .page_for_dma = swift_flush_page_for_dma, 1073 }; 1074 1075 #define SWIFT_MASKID_ADDR 0x10003018 1076 static void __init init_swift(void) 1077 { 1078 unsigned long swift_rev; 1079 1080 __asm__ __volatile__("lda [%1] %2, %0\n\t" 1081 "srl %0, 0x18, %0\n\t" : 1082 "=r" (swift_rev) : 1083 "r" (SWIFT_MASKID_ADDR), "i" (ASI_M_BYPASS)); 1084 srmmu_name = "Fujitsu Swift"; 1085 switch(swift_rev) { 1086 case 0x11: 1087 case 0x20: 1088 case 0x23: 1089 case 0x30: 1090 srmmu_modtype = Swift_lots_o_bugs; 1091 hwbug_bitmask |= (HWBUG_KERN_ACCBROKEN | HWBUG_KERN_CBITBROKEN); 1092 /* 1093 * Gee george, I wonder why Sun is so hush hush about 1094 * this hardware bug... really braindamage stuff going 1095 * on here. However I think we can find a way to avoid 1096 * all of the workaround overhead under Linux. Basically, 1097 * any page fault can cause kernel pages to become user 1098 * accessible (the mmu gets confused and clears some of 1099 * the ACC bits in kernel ptes). Aha, sounds pretty 1100 * horrible eh? But wait, after extensive testing it appears 1101 * that if you use pgd_t level large kernel pte's (like the 1102 * 4MB pages on the Pentium) the bug does not get tripped 1103 * at all. This avoids almost all of the major overhead. 1104 * Welcome to a world where your vendor tells you to, 1105 * "apply this kernel patch" instead of "sorry for the 1106 * broken hardware, send it back and we'll give you 1107 * properly functioning parts" 1108 */ 1109 break; 1110 case 0x25: 1111 case 0x31: 1112 srmmu_modtype = Swift_bad_c; 1113 hwbug_bitmask |= HWBUG_KERN_CBITBROKEN; 1114 /* 1115 * You see Sun allude to this hardware bug but never 1116 * admit things directly, they'll say things like, 1117 * "the Swift chip cache problems" or similar. 1118 */ 1119 break; 1120 default: 1121 srmmu_modtype = Swift_ok; 1122 break; 1123 } 1124 1125 sparc32_cachetlb_ops = &swift_ops; 1126 flush_page_for_dma_global = 0; 1127 1128 /* 1129 * Are you now convinced that the Swift is one of the 1130 * biggest VLSI abortions of all time? Bravo Fujitsu! 1131 * Fujitsu, the !#?!%$'d up processor people. I bet if 1132 * you examined the microcode of the Swift you'd find 1133 * XXX's all over the place. 1134 */ 1135 poke_srmmu = poke_swift; 1136 } 1137 1138 static void turbosparc_flush_cache_all(void) 1139 { 1140 flush_user_windows(); 1141 turbosparc_idflash_clear(); 1142 } 1143 1144 static void turbosparc_flush_cache_mm(struct mm_struct *mm) 1145 { 1146 FLUSH_BEGIN(mm) 1147 flush_user_windows(); 1148 turbosparc_idflash_clear(); 1149 FLUSH_END 1150 } 1151 1152 static void turbosparc_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) 1153 { 1154 FLUSH_BEGIN(vma->vm_mm) 1155 flush_user_windows(); 1156 turbosparc_idflash_clear(); 1157 FLUSH_END 1158 } 1159 1160 static void turbosparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page) 1161 { 1162 FLUSH_BEGIN(vma->vm_mm) 1163 flush_user_windows(); 1164 if (vma->vm_flags & VM_EXEC) 1165 turbosparc_flush_icache(); 1166 turbosparc_flush_dcache(); 1167 FLUSH_END 1168 } 1169 1170 /* TurboSparc is copy-back, if we turn it on, but this does not work. */ 1171 static void turbosparc_flush_page_to_ram(unsigned long page) 1172 { 1173 #ifdef TURBOSPARC_WRITEBACK 1174 volatile unsigned long clear; 1175 1176 if (srmmu_probe(page)) 1177 turbosparc_flush_page_cache(page); 1178 clear = srmmu_get_fstatus(); 1179 #endif 1180 } 1181 1182 static void turbosparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr) 1183 { 1184 } 1185 1186 static void turbosparc_flush_page_for_dma(unsigned long page) 1187 { 1188 turbosparc_flush_dcache(); 1189 } 1190 1191 static void turbosparc_flush_tlb_all(void) 1192 { 1193 srmmu_flush_whole_tlb(); 1194 } 1195 1196 static void turbosparc_flush_tlb_mm(struct mm_struct *mm) 1197 { 1198 FLUSH_BEGIN(mm) 1199 srmmu_flush_whole_tlb(); 1200 FLUSH_END 1201 } 1202 1203 static void turbosparc_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) 1204 { 1205 FLUSH_BEGIN(vma->vm_mm) 1206 srmmu_flush_whole_tlb(); 1207 FLUSH_END 1208 } 1209 1210 static void turbosparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 1211 { 1212 FLUSH_BEGIN(vma->vm_mm) 1213 srmmu_flush_whole_tlb(); 1214 FLUSH_END 1215 } 1216 1217 1218 static void __cpuinit poke_turbosparc(void) 1219 { 1220 unsigned long mreg = srmmu_get_mmureg(); 1221 unsigned long ccreg; 1222 1223 /* Clear any crap from the cache or else... */ 1224 turbosparc_flush_cache_all(); 1225 mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* Temporarily disable I & D caches */ 1226 mreg &= ~(TURBOSPARC_PCENABLE); /* Don't check parity */ 1227 srmmu_set_mmureg(mreg); 1228 1229 ccreg = turbosparc_get_ccreg(); 1230 1231 #ifdef TURBOSPARC_WRITEBACK 1232 ccreg |= (TURBOSPARC_SNENABLE); /* Do DVMA snooping in Dcache */ 1233 ccreg &= ~(TURBOSPARC_uS2 | TURBOSPARC_WTENABLE); 1234 /* Write-back D-cache, emulate VLSI 1235 * abortion number three, not number one */ 1236 #else 1237 /* For now let's play safe, optimize later */ 1238 ccreg |= (TURBOSPARC_SNENABLE | TURBOSPARC_WTENABLE); 1239 /* Do DVMA snooping in Dcache, Write-thru D-cache */ 1240 ccreg &= ~(TURBOSPARC_uS2); 1241 /* Emulate VLSI abortion number three, not number one */ 1242 #endif 1243 1244 switch (ccreg & 7) { 1245 case 0: /* No SE cache */ 1246 case 7: /* Test mode */ 1247 break; 1248 default: 1249 ccreg |= (TURBOSPARC_SCENABLE); 1250 } 1251 turbosparc_set_ccreg (ccreg); 1252 1253 mreg |= (TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* I & D caches on */ 1254 mreg |= (TURBOSPARC_ICSNOOP); /* Icache snooping on */ 1255 srmmu_set_mmureg(mreg); 1256 } 1257 1258 static const struct sparc32_cachetlb_ops turbosparc_ops = { 1259 .cache_all = turbosparc_flush_cache_all, 1260 .cache_mm = turbosparc_flush_cache_mm, 1261 .cache_page = turbosparc_flush_cache_page, 1262 .cache_range = turbosparc_flush_cache_range, 1263 .tlb_all = turbosparc_flush_tlb_all, 1264 .tlb_mm = turbosparc_flush_tlb_mm, 1265 .tlb_page = turbosparc_flush_tlb_page, 1266 .tlb_range = turbosparc_flush_tlb_range, 1267 .page_to_ram = turbosparc_flush_page_to_ram, 1268 .sig_insns = turbosparc_flush_sig_insns, 1269 .page_for_dma = turbosparc_flush_page_for_dma, 1270 }; 1271 1272 static void __init init_turbosparc(void) 1273 { 1274 srmmu_name = "Fujitsu TurboSparc"; 1275 srmmu_modtype = TurboSparc; 1276 sparc32_cachetlb_ops = &turbosparc_ops; 1277 poke_srmmu = poke_turbosparc; 1278 } 1279 1280 static void __cpuinit poke_tsunami(void) 1281 { 1282 unsigned long mreg = srmmu_get_mmureg(); 1283 1284 tsunami_flush_icache(); 1285 tsunami_flush_dcache(); 1286 mreg &= ~TSUNAMI_ITD; 1287 mreg |= (TSUNAMI_IENAB | TSUNAMI_DENAB); 1288 srmmu_set_mmureg(mreg); 1289 } 1290 1291 static const struct sparc32_cachetlb_ops tsunami_ops = { 1292 .cache_all = tsunami_flush_cache_all, 1293 .cache_mm = tsunami_flush_cache_mm, 1294 .cache_page = tsunami_flush_cache_page, 1295 .cache_range = tsunami_flush_cache_range, 1296 .tlb_all = tsunami_flush_tlb_all, 1297 .tlb_mm = tsunami_flush_tlb_mm, 1298 .tlb_page = tsunami_flush_tlb_page, 1299 .tlb_range = tsunami_flush_tlb_range, 1300 .page_to_ram = tsunami_flush_page_to_ram, 1301 .sig_insns = tsunami_flush_sig_insns, 1302 .page_for_dma = tsunami_flush_page_for_dma, 1303 }; 1304 1305 static void __init init_tsunami(void) 1306 { 1307 /* 1308 * Tsunami's pretty sane, Sun and TI actually got it 1309 * somewhat right this time. Fujitsu should have 1310 * taken some lessons from them. 1311 */ 1312 1313 srmmu_name = "TI Tsunami"; 1314 srmmu_modtype = Tsunami; 1315 sparc32_cachetlb_ops = &tsunami_ops; 1316 poke_srmmu = poke_tsunami; 1317 1318 tsunami_setup_blockops(); 1319 } 1320 1321 static void __cpuinit poke_viking(void) 1322 { 1323 unsigned long mreg = srmmu_get_mmureg(); 1324 static int smp_catch; 1325 1326 if (viking_mxcc_present) { 1327 unsigned long mxcc_control = mxcc_get_creg(); 1328 1329 mxcc_control |= (MXCC_CTL_ECE | MXCC_CTL_PRE | MXCC_CTL_MCE); 1330 mxcc_control &= ~(MXCC_CTL_RRC); 1331 mxcc_set_creg(mxcc_control); 1332 1333 /* 1334 * We don't need memory parity checks. 1335 * XXX This is a mess, have to dig out later. ecd. 1336 viking_mxcc_turn_off_parity(&mreg, &mxcc_control); 1337 */ 1338 1339 /* We do cache ptables on MXCC. */ 1340 mreg |= VIKING_TCENABLE; 1341 } else { 1342 unsigned long bpreg; 1343 1344 mreg &= ~(VIKING_TCENABLE); 1345 if(smp_catch++) { 1346 /* Must disable mixed-cmd mode here for other cpu's. */ 1347 bpreg = viking_get_bpreg(); 1348 bpreg &= ~(VIKING_ACTION_MIX); 1349 viking_set_bpreg(bpreg); 1350 1351 /* Just in case PROM does something funny. */ 1352 msi_set_sync(); 1353 } 1354 } 1355 1356 mreg |= VIKING_SPENABLE; 1357 mreg |= (VIKING_ICENABLE | VIKING_DCENABLE); 1358 mreg |= VIKING_SBENABLE; 1359 mreg &= ~(VIKING_ACENABLE); 1360 srmmu_set_mmureg(mreg); 1361 } 1362 1363 static struct sparc32_cachetlb_ops viking_ops = { 1364 .cache_all = viking_flush_cache_all, 1365 .cache_mm = viking_flush_cache_mm, 1366 .cache_page = viking_flush_cache_page, 1367 .cache_range = viking_flush_cache_range, 1368 .tlb_all = viking_flush_tlb_all, 1369 .tlb_mm = viking_flush_tlb_mm, 1370 .tlb_page = viking_flush_tlb_page, 1371 .tlb_range = viking_flush_tlb_range, 1372 .page_to_ram = viking_flush_page_to_ram, 1373 .sig_insns = viking_flush_sig_insns, 1374 .page_for_dma = viking_flush_page_for_dma, 1375 }; 1376 1377 #ifdef CONFIG_SMP 1378 /* On sun4d the cpu broadcasts local TLB flushes, so we can just 1379 * perform the local TLB flush and all the other cpus will see it. 1380 * But, unfortunately, there is a bug in the sun4d XBUS backplane 1381 * that requires that we add some synchronization to these flushes. 1382 * 1383 * The bug is that the fifo which keeps track of all the pending TLB 1384 * broadcasts in the system is an entry or two too small, so if we 1385 * have too many going at once we'll overflow that fifo and lose a TLB 1386 * flush resulting in corruption. 1387 * 1388 * Our workaround is to take a global spinlock around the TLB flushes, 1389 * which guarentees we won't ever have too many pending. It's a big 1390 * hammer, but a semaphore like system to make sure we only have N TLB 1391 * flushes going at once will require SMP locking anyways so there's 1392 * no real value in trying any harder than this. 1393 */ 1394 static struct sparc32_cachetlb_ops viking_sun4d_smp_ops = { 1395 .cache_all = viking_flush_cache_all, 1396 .cache_mm = viking_flush_cache_mm, 1397 .cache_page = viking_flush_cache_page, 1398 .cache_range = viking_flush_cache_range, 1399 .tlb_all = sun4dsmp_flush_tlb_all, 1400 .tlb_mm = sun4dsmp_flush_tlb_mm, 1401 .tlb_page = sun4dsmp_flush_tlb_page, 1402 .tlb_range = sun4dsmp_flush_tlb_range, 1403 .page_to_ram = viking_flush_page_to_ram, 1404 .sig_insns = viking_flush_sig_insns, 1405 .page_for_dma = viking_flush_page_for_dma, 1406 }; 1407 #endif 1408 1409 static void __init init_viking(void) 1410 { 1411 unsigned long mreg = srmmu_get_mmureg(); 1412 1413 /* Ahhh, the viking. SRMMU VLSI abortion number two... */ 1414 if(mreg & VIKING_MMODE) { 1415 srmmu_name = "TI Viking"; 1416 viking_mxcc_present = 0; 1417 msi_set_sync(); 1418 1419 /* 1420 * We need this to make sure old viking takes no hits 1421 * on it's cache for dma snoops to workaround the 1422 * "load from non-cacheable memory" interrupt bug. 1423 * This is only necessary because of the new way in 1424 * which we use the IOMMU. 1425 */ 1426 viking_ops.page_for_dma = viking_flush_page; 1427 #ifdef CONFIG_SMP 1428 viking_sun4d_smp_ops.page_for_dma = viking_flush_page; 1429 #endif 1430 flush_page_for_dma_global = 0; 1431 } else { 1432 srmmu_name = "TI Viking/MXCC"; 1433 viking_mxcc_present = 1; 1434 srmmu_cache_pagetables = 1; 1435 } 1436 1437 sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *) 1438 &viking_ops; 1439 #ifdef CONFIG_SMP 1440 if (sparc_cpu_model == sun4d) 1441 sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *) 1442 &viking_sun4d_smp_ops; 1443 #endif 1444 1445 poke_srmmu = poke_viking; 1446 } 1447 1448 /* Probe for the srmmu chip version. */ 1449 static void __init get_srmmu_type(void) 1450 { 1451 unsigned long mreg, psr; 1452 unsigned long mod_typ, mod_rev, psr_typ, psr_vers; 1453 1454 srmmu_modtype = SRMMU_INVAL_MOD; 1455 hwbug_bitmask = 0; 1456 1457 mreg = srmmu_get_mmureg(); psr = get_psr(); 1458 mod_typ = (mreg & 0xf0000000) >> 28; 1459 mod_rev = (mreg & 0x0f000000) >> 24; 1460 psr_typ = (psr >> 28) & 0xf; 1461 psr_vers = (psr >> 24) & 0xf; 1462 1463 /* First, check for sparc-leon. */ 1464 if (sparc_cpu_model == sparc_leon) { 1465 init_leon(); 1466 return; 1467 } 1468 1469 /* Second, check for HyperSparc or Cypress. */ 1470 if(mod_typ == 1) { 1471 switch(mod_rev) { 1472 case 7: 1473 /* UP or MP Hypersparc */ 1474 init_hypersparc(); 1475 break; 1476 case 0: 1477 case 2: 1478 case 10: 1479 case 11: 1480 case 12: 1481 case 13: 1482 case 14: 1483 case 15: 1484 default: 1485 prom_printf("Sparc-Linux Cypress support does not longer exit.\n"); 1486 prom_halt(); 1487 break; 1488 } 1489 return; 1490 } 1491 1492 /* 1493 * Now Fujitsu TurboSparc. It might happen that it is 1494 * in Swift emulation mode, so we will check later... 1495 */ 1496 if (psr_typ == 0 && psr_vers == 5) { 1497 init_turbosparc(); 1498 return; 1499 } 1500 1501 /* Next check for Fujitsu Swift. */ 1502 if(psr_typ == 0 && psr_vers == 4) { 1503 phandle cpunode; 1504 char node_str[128]; 1505 1506 /* Look if it is not a TurboSparc emulating Swift... */ 1507 cpunode = prom_getchild(prom_root_node); 1508 while((cpunode = prom_getsibling(cpunode)) != 0) { 1509 prom_getstring(cpunode, "device_type", node_str, sizeof(node_str)); 1510 if(!strcmp(node_str, "cpu")) { 1511 if (!prom_getintdefault(cpunode, "psr-implementation", 1) && 1512 prom_getintdefault(cpunode, "psr-version", 1) == 5) { 1513 init_turbosparc(); 1514 return; 1515 } 1516 break; 1517 } 1518 } 1519 1520 init_swift(); 1521 return; 1522 } 1523 1524 /* Now the Viking family of srmmu. */ 1525 if(psr_typ == 4 && 1526 ((psr_vers == 0) || 1527 ((psr_vers == 1) && (mod_typ == 0) && (mod_rev == 0)))) { 1528 init_viking(); 1529 return; 1530 } 1531 1532 /* Finally the Tsunami. */ 1533 if(psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) { 1534 init_tsunami(); 1535 return; 1536 } 1537 1538 /* Oh well */ 1539 srmmu_is_bad(); 1540 } 1541 1542 #ifdef CONFIG_SMP 1543 /* Local cross-calls. */ 1544 static void smp_flush_page_for_dma(unsigned long page) 1545 { 1546 xc1((smpfunc_t) local_ops->page_for_dma, page); 1547 local_ops->page_for_dma(page); 1548 } 1549 1550 static void smp_flush_cache_all(void) 1551 { 1552 xc0((smpfunc_t) local_ops->cache_all); 1553 local_ops->cache_all(); 1554 } 1555 1556 static void smp_flush_tlb_all(void) 1557 { 1558 xc0((smpfunc_t) local_ops->tlb_all); 1559 local_ops->tlb_all(); 1560 } 1561 1562 static void smp_flush_cache_mm(struct mm_struct *mm) 1563 { 1564 if (mm->context != NO_CONTEXT) { 1565 cpumask_t cpu_mask; 1566 cpumask_copy(&cpu_mask, mm_cpumask(mm)); 1567 cpumask_clear_cpu(smp_processor_id(), &cpu_mask); 1568 if (!cpumask_empty(&cpu_mask)) 1569 xc1((smpfunc_t) local_ops->cache_mm, (unsigned long) mm); 1570 local_ops->cache_mm(mm); 1571 } 1572 } 1573 1574 static void smp_flush_tlb_mm(struct mm_struct *mm) 1575 { 1576 if (mm->context != NO_CONTEXT) { 1577 cpumask_t cpu_mask; 1578 cpumask_copy(&cpu_mask, mm_cpumask(mm)); 1579 cpumask_clear_cpu(smp_processor_id(), &cpu_mask); 1580 if (!cpumask_empty(&cpu_mask)) { 1581 xc1((smpfunc_t) local_ops->tlb_mm, (unsigned long) mm); 1582 if (atomic_read(&mm->mm_users) == 1 && current->active_mm == mm) 1583 cpumask_copy(mm_cpumask(mm), 1584 cpumask_of(smp_processor_id())); 1585 } 1586 local_ops->tlb_mm(mm); 1587 } 1588 } 1589 1590 static void smp_flush_cache_range(struct vm_area_struct *vma, 1591 unsigned long start, 1592 unsigned long end) 1593 { 1594 struct mm_struct *mm = vma->vm_mm; 1595 1596 if (mm->context != NO_CONTEXT) { 1597 cpumask_t cpu_mask; 1598 cpumask_copy(&cpu_mask, mm_cpumask(mm)); 1599 cpumask_clear_cpu(smp_processor_id(), &cpu_mask); 1600 if (!cpumask_empty(&cpu_mask)) 1601 xc3((smpfunc_t) local_ops->cache_range, 1602 (unsigned long) vma, start, end); 1603 local_ops->cache_range(vma, start, end); 1604 } 1605 } 1606 1607 static void smp_flush_tlb_range(struct vm_area_struct *vma, 1608 unsigned long start, 1609 unsigned long end) 1610 { 1611 struct mm_struct *mm = vma->vm_mm; 1612 1613 if (mm->context != NO_CONTEXT) { 1614 cpumask_t cpu_mask; 1615 cpumask_copy(&cpu_mask, mm_cpumask(mm)); 1616 cpumask_clear_cpu(smp_processor_id(), &cpu_mask); 1617 if (!cpumask_empty(&cpu_mask)) 1618 xc3((smpfunc_t) local_ops->tlb_range, 1619 (unsigned long) vma, start, end); 1620 local_ops->tlb_range(vma, start, end); 1621 } 1622 } 1623 1624 static void smp_flush_cache_page(struct vm_area_struct *vma, unsigned long page) 1625 { 1626 struct mm_struct *mm = vma->vm_mm; 1627 1628 if (mm->context != NO_CONTEXT) { 1629 cpumask_t cpu_mask; 1630 cpumask_copy(&cpu_mask, mm_cpumask(mm)); 1631 cpumask_clear_cpu(smp_processor_id(), &cpu_mask); 1632 if (!cpumask_empty(&cpu_mask)) 1633 xc2((smpfunc_t) local_ops->cache_page, 1634 (unsigned long) vma, page); 1635 local_ops->cache_page(vma, page); 1636 } 1637 } 1638 1639 static void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) 1640 { 1641 struct mm_struct *mm = vma->vm_mm; 1642 1643 if (mm->context != NO_CONTEXT) { 1644 cpumask_t cpu_mask; 1645 cpumask_copy(&cpu_mask, mm_cpumask(mm)); 1646 cpumask_clear_cpu(smp_processor_id(), &cpu_mask); 1647 if (!cpumask_empty(&cpu_mask)) 1648 xc2((smpfunc_t) local_ops->tlb_page, 1649 (unsigned long) vma, page); 1650 local_ops->tlb_page(vma, page); 1651 } 1652 } 1653 1654 static void smp_flush_page_to_ram(unsigned long page) 1655 { 1656 /* Current theory is that those who call this are the one's 1657 * who have just dirtied their cache with the pages contents 1658 * in kernel space, therefore we only run this on local cpu. 1659 * 1660 * XXX This experiment failed, research further... -DaveM 1661 */ 1662 #if 1 1663 xc1((smpfunc_t) local_ops->page_to_ram, page); 1664 #endif 1665 local_ops->page_to_ram(page); 1666 } 1667 1668 static void smp_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr) 1669 { 1670 cpumask_t cpu_mask; 1671 cpumask_copy(&cpu_mask, mm_cpumask(mm)); 1672 cpumask_clear_cpu(smp_processor_id(), &cpu_mask); 1673 if (!cpumask_empty(&cpu_mask)) 1674 xc2((smpfunc_t) local_ops->sig_insns, 1675 (unsigned long) mm, insn_addr); 1676 local_ops->sig_insns(mm, insn_addr); 1677 } 1678 1679 static struct sparc32_cachetlb_ops smp_cachetlb_ops = { 1680 .cache_all = smp_flush_cache_all, 1681 .cache_mm = smp_flush_cache_mm, 1682 .cache_page = smp_flush_cache_page, 1683 .cache_range = smp_flush_cache_range, 1684 .tlb_all = smp_flush_tlb_all, 1685 .tlb_mm = smp_flush_tlb_mm, 1686 .tlb_page = smp_flush_tlb_page, 1687 .tlb_range = smp_flush_tlb_range, 1688 .page_to_ram = smp_flush_page_to_ram, 1689 .sig_insns = smp_flush_sig_insns, 1690 .page_for_dma = smp_flush_page_for_dma, 1691 }; 1692 #endif 1693 1694 /* Load up routines and constants for sun4m and sun4d mmu */ 1695 void __init load_mmu(void) 1696 { 1697 extern void ld_mmu_iommu(void); 1698 extern void ld_mmu_iounit(void); 1699 1700 /* Functions */ 1701 get_srmmu_type(); 1702 1703 #ifdef CONFIG_SMP 1704 /* El switcheroo... */ 1705 local_ops = sparc32_cachetlb_ops; 1706 1707 if (sparc_cpu_model == sun4d || sparc_cpu_model == sparc_leon) { 1708 smp_cachetlb_ops.tlb_all = local_ops->tlb_all; 1709 smp_cachetlb_ops.tlb_mm = local_ops->tlb_mm; 1710 smp_cachetlb_ops.tlb_range = local_ops->tlb_range; 1711 smp_cachetlb_ops.tlb_page = local_ops->tlb_page; 1712 } 1713 1714 if (poke_srmmu == poke_viking) { 1715 /* Avoid unnecessary cross calls. */ 1716 smp_cachetlb_ops.cache_all = local_ops->cache_all; 1717 smp_cachetlb_ops.cache_mm = local_ops->cache_mm; 1718 smp_cachetlb_ops.cache_range = local_ops->cache_range; 1719 smp_cachetlb_ops.cache_page = local_ops->cache_page; 1720 1721 smp_cachetlb_ops.page_to_ram = local_ops->page_to_ram; 1722 smp_cachetlb_ops.sig_insns = local_ops->sig_insns; 1723 smp_cachetlb_ops.page_for_dma = local_ops->page_for_dma; 1724 } 1725 1726 /* It really is const after this point. */ 1727 sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *) 1728 &smp_cachetlb_ops; 1729 #endif 1730 1731 if (sparc_cpu_model == sun4d) 1732 ld_mmu_iounit(); 1733 else 1734 ld_mmu_iommu(); 1735 #ifdef CONFIG_SMP 1736 if (sparc_cpu_model == sun4d) 1737 sun4d_init_smp(); 1738 else if (sparc_cpu_model == sparc_leon) 1739 leon_init_smp(); 1740 else 1741 sun4m_init_smp(); 1742 #endif 1743 } 1744