1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) 7 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org) 8 * Copyright (C) 1999, 2000 Silicon Graphics, Inc. 9 */ 10 #include <linux/init.h> 11 #include <linux/highmem.h> 12 #include <linux/kernel.h> 13 #include <linux/linkage.h> 14 #include <linux/sched.h> 15 #include <linux/mm.h> 16 #include <linux/bitops.h> 17 18 #include <asm/bcache.h> 19 #include <asm/bootinfo.h> 20 #include <asm/cache.h> 21 #include <asm/cacheops.h> 22 #include <asm/cpu.h> 23 #include <asm/cpu-features.h> 24 #include <asm/io.h> 25 #include <asm/page.h> 26 #include <asm/pgtable.h> 27 #include <asm/r4kcache.h> 28 #include <asm/sections.h> 29 #include <asm/system.h> 30 #include <asm/mmu_context.h> 31 #include <asm/war.h> 32 #include <asm/cacheflush.h> /* for run_uncached() */ 33 34 35 /* 36 * Special Variant of smp_call_function for use by cache functions: 37 * 38 * o No return value 39 * o collapses to normal function call on UP kernels 40 * o collapses to normal function call on systems with a single shared 41 * primary cache. 42 */ 43 static inline void r4k_on_each_cpu(void (*func) (void *info), void *info, 44 int retry, int wait) 45 { 46 preempt_disable(); 47 48 #if !defined(CONFIG_MIPS_MT_SMP) && !defined(CONFIG_MIPS_MT_SMTC) 49 smp_call_function(func, info, retry, wait); 50 #endif 51 func(info); 52 preempt_enable(); 53 } 54 55 /* 56 * Must die. 57 */ 58 static unsigned long icache_size __read_mostly; 59 static unsigned long dcache_size __read_mostly; 60 static unsigned long scache_size __read_mostly; 61 62 /* 63 * Dummy cache handling routines for machines without boardcaches 64 */ 65 static void cache_noop(void) {} 66 67 static struct bcache_ops no_sc_ops = { 68 .bc_enable = (void *)cache_noop, 69 .bc_disable = (void *)cache_noop, 70 .bc_wback_inv = (void *)cache_noop, 71 .bc_inv = (void *)cache_noop 72 }; 73 74 struct bcache_ops *bcops = &no_sc_ops; 75 76 #define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010) 77 #define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020) 78 79 #define R4600_HIT_CACHEOP_WAR_IMPL \ 80 do { \ 81 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) \ 82 *(volatile unsigned long *)CKSEG1; \ 83 if (R4600_V1_HIT_CACHEOP_WAR) \ 84 __asm__ __volatile__("nop;nop;nop;nop"); \ 85 } while (0) 86 87 static void (*r4k_blast_dcache_page)(unsigned long addr); 88 89 static inline void r4k_blast_dcache_page_dc32(unsigned long addr) 90 { 91 R4600_HIT_CACHEOP_WAR_IMPL; 92 blast_dcache32_page(addr); 93 } 94 95 static void __init r4k_blast_dcache_page_setup(void) 96 { 97 unsigned long dc_lsize = cpu_dcache_line_size(); 98 99 if (dc_lsize == 0) 100 r4k_blast_dcache_page = (void *)cache_noop; 101 else if (dc_lsize == 16) 102 r4k_blast_dcache_page = blast_dcache16_page; 103 else if (dc_lsize == 32) 104 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32; 105 } 106 107 static void (* r4k_blast_dcache_page_indexed)(unsigned long addr); 108 109 static void __init r4k_blast_dcache_page_indexed_setup(void) 110 { 111 unsigned long dc_lsize = cpu_dcache_line_size(); 112 113 if (dc_lsize == 0) 114 r4k_blast_dcache_page_indexed = (void *)cache_noop; 115 else if (dc_lsize == 16) 116 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed; 117 else if (dc_lsize == 32) 118 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed; 119 } 120 121 static void (* r4k_blast_dcache)(void); 122 123 static void __init r4k_blast_dcache_setup(void) 124 { 125 unsigned long dc_lsize = cpu_dcache_line_size(); 126 127 if (dc_lsize == 0) 128 r4k_blast_dcache = (void *)cache_noop; 129 else if (dc_lsize == 16) 130 r4k_blast_dcache = blast_dcache16; 131 else if (dc_lsize == 32) 132 r4k_blast_dcache = blast_dcache32; 133 } 134 135 /* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */ 136 #define JUMP_TO_ALIGN(order) \ 137 __asm__ __volatile__( \ 138 "b\t1f\n\t" \ 139 ".align\t" #order "\n\t" \ 140 "1:\n\t" \ 141 ) 142 #define CACHE32_UNROLL32_ALIGN JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */ 143 #define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11) 144 145 static inline void blast_r4600_v1_icache32(void) 146 { 147 unsigned long flags; 148 149 local_irq_save(flags); 150 blast_icache32(); 151 local_irq_restore(flags); 152 } 153 154 static inline void tx49_blast_icache32(void) 155 { 156 unsigned long start = INDEX_BASE; 157 unsigned long end = start + current_cpu_data.icache.waysize; 158 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 159 unsigned long ws_end = current_cpu_data.icache.ways << 160 current_cpu_data.icache.waybit; 161 unsigned long ws, addr; 162 163 CACHE32_UNROLL32_ALIGN2; 164 /* I'm in even chunk. blast odd chunks */ 165 for (ws = 0; ws < ws_end; ws += ws_inc) 166 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 167 cache32_unroll32(addr|ws, Index_Invalidate_I); 168 CACHE32_UNROLL32_ALIGN; 169 /* I'm in odd chunk. blast even chunks */ 170 for (ws = 0; ws < ws_end; ws += ws_inc) 171 for (addr = start; addr < end; addr += 0x400 * 2) 172 cache32_unroll32(addr|ws, Index_Invalidate_I); 173 } 174 175 static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page) 176 { 177 unsigned long flags; 178 179 local_irq_save(flags); 180 blast_icache32_page_indexed(page); 181 local_irq_restore(flags); 182 } 183 184 static inline void tx49_blast_icache32_page_indexed(unsigned long page) 185 { 186 unsigned long indexmask = current_cpu_data.icache.waysize - 1; 187 unsigned long start = INDEX_BASE + (page & indexmask); 188 unsigned long end = start + PAGE_SIZE; 189 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 190 unsigned long ws_end = current_cpu_data.icache.ways << 191 current_cpu_data.icache.waybit; 192 unsigned long ws, addr; 193 194 CACHE32_UNROLL32_ALIGN2; 195 /* I'm in even chunk. blast odd chunks */ 196 for (ws = 0; ws < ws_end; ws += ws_inc) 197 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 198 cache32_unroll32(addr|ws, Index_Invalidate_I); 199 CACHE32_UNROLL32_ALIGN; 200 /* I'm in odd chunk. blast even chunks */ 201 for (ws = 0; ws < ws_end; ws += ws_inc) 202 for (addr = start; addr < end; addr += 0x400 * 2) 203 cache32_unroll32(addr|ws, Index_Invalidate_I); 204 } 205 206 static void (* r4k_blast_icache_page)(unsigned long addr); 207 208 static void __init r4k_blast_icache_page_setup(void) 209 { 210 unsigned long ic_lsize = cpu_icache_line_size(); 211 212 if (ic_lsize == 0) 213 r4k_blast_icache_page = (void *)cache_noop; 214 else if (ic_lsize == 16) 215 r4k_blast_icache_page = blast_icache16_page; 216 else if (ic_lsize == 32) 217 r4k_blast_icache_page = blast_icache32_page; 218 else if (ic_lsize == 64) 219 r4k_blast_icache_page = blast_icache64_page; 220 } 221 222 223 static void (* r4k_blast_icache_page_indexed)(unsigned long addr); 224 225 static void __init r4k_blast_icache_page_indexed_setup(void) 226 { 227 unsigned long ic_lsize = cpu_icache_line_size(); 228 229 if (ic_lsize == 0) 230 r4k_blast_icache_page_indexed = (void *)cache_noop; 231 else if (ic_lsize == 16) 232 r4k_blast_icache_page_indexed = blast_icache16_page_indexed; 233 else if (ic_lsize == 32) { 234 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 235 r4k_blast_icache_page_indexed = 236 blast_icache32_r4600_v1_page_indexed; 237 else if (TX49XX_ICACHE_INDEX_INV_WAR) 238 r4k_blast_icache_page_indexed = 239 tx49_blast_icache32_page_indexed; 240 else 241 r4k_blast_icache_page_indexed = 242 blast_icache32_page_indexed; 243 } else if (ic_lsize == 64) 244 r4k_blast_icache_page_indexed = blast_icache64_page_indexed; 245 } 246 247 static void (* r4k_blast_icache)(void); 248 249 static void __init r4k_blast_icache_setup(void) 250 { 251 unsigned long ic_lsize = cpu_icache_line_size(); 252 253 if (ic_lsize == 0) 254 r4k_blast_icache = (void *)cache_noop; 255 else if (ic_lsize == 16) 256 r4k_blast_icache = blast_icache16; 257 else if (ic_lsize == 32) { 258 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 259 r4k_blast_icache = blast_r4600_v1_icache32; 260 else if (TX49XX_ICACHE_INDEX_INV_WAR) 261 r4k_blast_icache = tx49_blast_icache32; 262 else 263 r4k_blast_icache = blast_icache32; 264 } else if (ic_lsize == 64) 265 r4k_blast_icache = blast_icache64; 266 } 267 268 static void (* r4k_blast_scache_page)(unsigned long addr); 269 270 static void __init r4k_blast_scache_page_setup(void) 271 { 272 unsigned long sc_lsize = cpu_scache_line_size(); 273 274 if (scache_size == 0) 275 r4k_blast_scache_page = (void *)cache_noop; 276 else if (sc_lsize == 16) 277 r4k_blast_scache_page = blast_scache16_page; 278 else if (sc_lsize == 32) 279 r4k_blast_scache_page = blast_scache32_page; 280 else if (sc_lsize == 64) 281 r4k_blast_scache_page = blast_scache64_page; 282 else if (sc_lsize == 128) 283 r4k_blast_scache_page = blast_scache128_page; 284 } 285 286 static void (* r4k_blast_scache_page_indexed)(unsigned long addr); 287 288 static void __init r4k_blast_scache_page_indexed_setup(void) 289 { 290 unsigned long sc_lsize = cpu_scache_line_size(); 291 292 if (scache_size == 0) 293 r4k_blast_scache_page_indexed = (void *)cache_noop; 294 else if (sc_lsize == 16) 295 r4k_blast_scache_page_indexed = blast_scache16_page_indexed; 296 else if (sc_lsize == 32) 297 r4k_blast_scache_page_indexed = blast_scache32_page_indexed; 298 else if (sc_lsize == 64) 299 r4k_blast_scache_page_indexed = blast_scache64_page_indexed; 300 else if (sc_lsize == 128) 301 r4k_blast_scache_page_indexed = blast_scache128_page_indexed; 302 } 303 304 static void (* r4k_blast_scache)(void); 305 306 static void __init r4k_blast_scache_setup(void) 307 { 308 unsigned long sc_lsize = cpu_scache_line_size(); 309 310 if (scache_size == 0) 311 r4k_blast_scache = (void *)cache_noop; 312 else if (sc_lsize == 16) 313 r4k_blast_scache = blast_scache16; 314 else if (sc_lsize == 32) 315 r4k_blast_scache = blast_scache32; 316 else if (sc_lsize == 64) 317 r4k_blast_scache = blast_scache64; 318 else if (sc_lsize == 128) 319 r4k_blast_scache = blast_scache128; 320 } 321 322 static inline void local_r4k___flush_cache_all(void * args) 323 { 324 #if defined(CONFIG_CPU_LOONGSON2) 325 r4k_blast_scache(); 326 return; 327 #endif 328 r4k_blast_dcache(); 329 r4k_blast_icache(); 330 331 switch (current_cpu_type()) { 332 case CPU_R4000SC: 333 case CPU_R4000MC: 334 case CPU_R4400SC: 335 case CPU_R4400MC: 336 case CPU_R10000: 337 case CPU_R12000: 338 case CPU_R14000: 339 r4k_blast_scache(); 340 } 341 } 342 343 static void r4k___flush_cache_all(void) 344 { 345 r4k_on_each_cpu(local_r4k___flush_cache_all, NULL, 1, 1); 346 } 347 348 static inline void local_r4k_flush_cache_range(void * args) 349 { 350 struct vm_area_struct *vma = args; 351 352 if (!(cpu_context(smp_processor_id(), vma->vm_mm))) 353 return; 354 355 r4k_blast_dcache(); 356 } 357 358 static void r4k_flush_cache_range(struct vm_area_struct *vma, 359 unsigned long start, unsigned long end) 360 { 361 if (!cpu_has_dc_aliases) 362 return; 363 364 r4k_on_each_cpu(local_r4k_flush_cache_range, vma, 1, 1); 365 } 366 367 static inline void local_r4k_flush_cache_mm(void * args) 368 { 369 struct mm_struct *mm = args; 370 371 if (!cpu_context(smp_processor_id(), mm)) 372 return; 373 374 /* 375 * Kludge alert. For obscure reasons R4000SC and R4400SC go nuts if we 376 * only flush the primary caches but R10000 and R12000 behave sane ... 377 * R4000SC and R4400SC indexed S-cache ops also invalidate primary 378 * caches, so we can bail out early. 379 */ 380 if (current_cpu_type() == CPU_R4000SC || 381 current_cpu_type() == CPU_R4000MC || 382 current_cpu_type() == CPU_R4400SC || 383 current_cpu_type() == CPU_R4400MC) { 384 r4k_blast_scache(); 385 return; 386 } 387 388 r4k_blast_dcache(); 389 } 390 391 static void r4k_flush_cache_mm(struct mm_struct *mm) 392 { 393 if (!cpu_has_dc_aliases) 394 return; 395 396 r4k_on_each_cpu(local_r4k_flush_cache_mm, mm, 1, 1); 397 } 398 399 struct flush_cache_page_args { 400 struct vm_area_struct *vma; 401 unsigned long addr; 402 unsigned long pfn; 403 }; 404 405 static inline void local_r4k_flush_cache_page(void *args) 406 { 407 struct flush_cache_page_args *fcp_args = args; 408 struct vm_area_struct *vma = fcp_args->vma; 409 unsigned long addr = fcp_args->addr; 410 struct page *page = pfn_to_page(fcp_args->pfn); 411 int exec = vma->vm_flags & VM_EXEC; 412 struct mm_struct *mm = vma->vm_mm; 413 pgd_t *pgdp; 414 pud_t *pudp; 415 pmd_t *pmdp; 416 pte_t *ptep; 417 void *vaddr; 418 419 /* 420 * If ownes no valid ASID yet, cannot possibly have gotten 421 * this page into the cache. 422 */ 423 if (cpu_context(smp_processor_id(), mm) == 0) 424 return; 425 426 addr &= PAGE_MASK; 427 pgdp = pgd_offset(mm, addr); 428 pudp = pud_offset(pgdp, addr); 429 pmdp = pmd_offset(pudp, addr); 430 ptep = pte_offset(pmdp, addr); 431 432 /* 433 * If the page isn't marked valid, the page cannot possibly be 434 * in the cache. 435 */ 436 if (!(pte_val(*ptep) & _PAGE_PRESENT)) 437 return; 438 439 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) 440 vaddr = NULL; 441 else { 442 /* 443 * Use kmap_coherent or kmap_atomic to do flushes for 444 * another ASID than the current one. 445 */ 446 if (cpu_has_dc_aliases) 447 vaddr = kmap_coherent(page, addr); 448 else 449 vaddr = kmap_atomic(page, KM_USER0); 450 addr = (unsigned long)vaddr; 451 } 452 453 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { 454 r4k_blast_dcache_page(addr); 455 if (exec && !cpu_icache_snoops_remote_store) 456 r4k_blast_scache_page(addr); 457 } 458 if (exec) { 459 if (vaddr && cpu_has_vtag_icache && mm == current->active_mm) { 460 int cpu = smp_processor_id(); 461 462 if (cpu_context(cpu, mm) != 0) 463 drop_mmu_context(mm, cpu); 464 } else 465 r4k_blast_icache_page(addr); 466 } 467 468 if (vaddr) { 469 if (cpu_has_dc_aliases) 470 kunmap_coherent(); 471 else 472 kunmap_atomic(vaddr, KM_USER0); 473 } 474 } 475 476 static void r4k_flush_cache_page(struct vm_area_struct *vma, 477 unsigned long addr, unsigned long pfn) 478 { 479 struct flush_cache_page_args args; 480 481 args.vma = vma; 482 args.addr = addr; 483 args.pfn = pfn; 484 485 r4k_on_each_cpu(local_r4k_flush_cache_page, &args, 1, 1); 486 } 487 488 static inline void local_r4k_flush_data_cache_page(void * addr) 489 { 490 r4k_blast_dcache_page((unsigned long) addr); 491 } 492 493 static void r4k_flush_data_cache_page(unsigned long addr) 494 { 495 r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr, 1, 1); 496 } 497 498 struct flush_icache_range_args { 499 unsigned long start; 500 unsigned long end; 501 }; 502 503 static inline void local_r4k_flush_icache_range(void *args) 504 { 505 struct flush_icache_range_args *fir_args = args; 506 unsigned long start = fir_args->start; 507 unsigned long end = fir_args->end; 508 509 if (!cpu_has_ic_fills_f_dc) { 510 if (end - start >= dcache_size) { 511 r4k_blast_dcache(); 512 } else { 513 R4600_HIT_CACHEOP_WAR_IMPL; 514 protected_blast_dcache_range(start, end); 515 } 516 517 if (!cpu_icache_snoops_remote_store && scache_size) { 518 if (end - start > scache_size) 519 r4k_blast_scache(); 520 else 521 protected_blast_scache_range(start, end); 522 } 523 } 524 525 if (end - start > icache_size) 526 r4k_blast_icache(); 527 else 528 protected_blast_icache_range(start, end); 529 } 530 531 static void r4k_flush_icache_range(unsigned long start, unsigned long end) 532 { 533 struct flush_icache_range_args args; 534 535 args.start = start; 536 args.end = end; 537 538 r4k_on_each_cpu(local_r4k_flush_icache_range, &args, 1, 1); 539 instruction_hazard(); 540 } 541 542 #ifdef CONFIG_DMA_NONCOHERENT 543 544 static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size) 545 { 546 /* Catch bad driver code */ 547 BUG_ON(size == 0); 548 549 if (cpu_has_inclusive_pcaches) { 550 if (size >= scache_size) 551 r4k_blast_scache(); 552 else 553 blast_scache_range(addr, addr + size); 554 return; 555 } 556 557 /* 558 * Either no secondary cache or the available caches don't have the 559 * subset property so we have to flush the primary caches 560 * explicitly 561 */ 562 if (size >= dcache_size) { 563 r4k_blast_dcache(); 564 } else { 565 R4600_HIT_CACHEOP_WAR_IMPL; 566 blast_dcache_range(addr, addr + size); 567 } 568 569 bc_wback_inv(addr, size); 570 } 571 572 static void r4k_dma_cache_inv(unsigned long addr, unsigned long size) 573 { 574 /* Catch bad driver code */ 575 BUG_ON(size == 0); 576 577 if (cpu_has_inclusive_pcaches) { 578 if (size >= scache_size) 579 r4k_blast_scache(); 580 else 581 blast_scache_range(addr, addr + size); 582 return; 583 } 584 585 if (size >= dcache_size) { 586 r4k_blast_dcache(); 587 } else { 588 R4600_HIT_CACHEOP_WAR_IMPL; 589 blast_dcache_range(addr, addr + size); 590 } 591 592 bc_inv(addr, size); 593 } 594 #endif /* CONFIG_DMA_NONCOHERENT */ 595 596 /* 597 * While we're protected against bad userland addresses we don't care 598 * very much about what happens in that case. Usually a segmentation 599 * fault will dump the process later on anyway ... 600 */ 601 static void local_r4k_flush_cache_sigtramp(void * arg) 602 { 603 unsigned long ic_lsize = cpu_icache_line_size(); 604 unsigned long dc_lsize = cpu_dcache_line_size(); 605 unsigned long sc_lsize = cpu_scache_line_size(); 606 unsigned long addr = (unsigned long) arg; 607 608 R4600_HIT_CACHEOP_WAR_IMPL; 609 if (dc_lsize) 610 protected_writeback_dcache_line(addr & ~(dc_lsize - 1)); 611 if (!cpu_icache_snoops_remote_store && scache_size) 612 protected_writeback_scache_line(addr & ~(sc_lsize - 1)); 613 if (ic_lsize) 614 protected_flush_icache_line(addr & ~(ic_lsize - 1)); 615 if (MIPS4K_ICACHE_REFILL_WAR) { 616 __asm__ __volatile__ ( 617 ".set push\n\t" 618 ".set noat\n\t" 619 ".set mips3\n\t" 620 #ifdef CONFIG_32BIT 621 "la $at,1f\n\t" 622 #endif 623 #ifdef CONFIG_64BIT 624 "dla $at,1f\n\t" 625 #endif 626 "cache %0,($at)\n\t" 627 "nop; nop; nop\n" 628 "1:\n\t" 629 ".set pop" 630 : 631 : "i" (Hit_Invalidate_I)); 632 } 633 if (MIPS_CACHE_SYNC_WAR) 634 __asm__ __volatile__ ("sync"); 635 } 636 637 static void r4k_flush_cache_sigtramp(unsigned long addr) 638 { 639 r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr, 1, 1); 640 } 641 642 static void r4k_flush_icache_all(void) 643 { 644 if (cpu_has_vtag_icache) 645 r4k_blast_icache(); 646 } 647 648 static inline void rm7k_erratum31(void) 649 { 650 const unsigned long ic_lsize = 32; 651 unsigned long addr; 652 653 /* RM7000 erratum #31. The icache is screwed at startup. */ 654 write_c0_taglo(0); 655 write_c0_taghi(0); 656 657 for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) { 658 __asm__ __volatile__ ( 659 ".set push\n\t" 660 ".set noreorder\n\t" 661 ".set mips3\n\t" 662 "cache\t%1, 0(%0)\n\t" 663 "cache\t%1, 0x1000(%0)\n\t" 664 "cache\t%1, 0x2000(%0)\n\t" 665 "cache\t%1, 0x3000(%0)\n\t" 666 "cache\t%2, 0(%0)\n\t" 667 "cache\t%2, 0x1000(%0)\n\t" 668 "cache\t%2, 0x2000(%0)\n\t" 669 "cache\t%2, 0x3000(%0)\n\t" 670 "cache\t%1, 0(%0)\n\t" 671 "cache\t%1, 0x1000(%0)\n\t" 672 "cache\t%1, 0x2000(%0)\n\t" 673 "cache\t%1, 0x3000(%0)\n\t" 674 ".set pop\n" 675 : 676 : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill)); 677 } 678 } 679 680 static char *way_string[] __initdata = { NULL, "direct mapped", "2-way", 681 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way" 682 }; 683 684 static void __init probe_pcache(void) 685 { 686 struct cpuinfo_mips *c = ¤t_cpu_data; 687 unsigned int config = read_c0_config(); 688 unsigned int prid = read_c0_prid(); 689 unsigned long config1; 690 unsigned int lsize; 691 692 switch (c->cputype) { 693 case CPU_R4600: /* QED style two way caches? */ 694 case CPU_R4700: 695 case CPU_R5000: 696 case CPU_NEVADA: 697 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 698 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 699 c->icache.ways = 2; 700 c->icache.waybit = __ffs(icache_size/2); 701 702 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 703 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 704 c->dcache.ways = 2; 705 c->dcache.waybit= __ffs(dcache_size/2); 706 707 c->options |= MIPS_CPU_CACHE_CDEX_P; 708 break; 709 710 case CPU_R5432: 711 case CPU_R5500: 712 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 713 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 714 c->icache.ways = 2; 715 c->icache.waybit= 0; 716 717 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 718 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 719 c->dcache.ways = 2; 720 c->dcache.waybit = 0; 721 722 c->options |= MIPS_CPU_CACHE_CDEX_P; 723 break; 724 725 case CPU_TX49XX: 726 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 727 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 728 c->icache.ways = 4; 729 c->icache.waybit= 0; 730 731 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 732 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 733 c->dcache.ways = 4; 734 c->dcache.waybit = 0; 735 736 c->options |= MIPS_CPU_CACHE_CDEX_P; 737 c->options |= MIPS_CPU_PREFETCH; 738 break; 739 740 case CPU_R4000PC: 741 case CPU_R4000SC: 742 case CPU_R4000MC: 743 case CPU_R4400PC: 744 case CPU_R4400SC: 745 case CPU_R4400MC: 746 case CPU_R4300: 747 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 748 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 749 c->icache.ways = 1; 750 c->icache.waybit = 0; /* doesn't matter */ 751 752 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 753 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 754 c->dcache.ways = 1; 755 c->dcache.waybit = 0; /* does not matter */ 756 757 c->options |= MIPS_CPU_CACHE_CDEX_P; 758 break; 759 760 case CPU_R10000: 761 case CPU_R12000: 762 case CPU_R14000: 763 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29)); 764 c->icache.linesz = 64; 765 c->icache.ways = 2; 766 c->icache.waybit = 0; 767 768 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26)); 769 c->dcache.linesz = 32; 770 c->dcache.ways = 2; 771 c->dcache.waybit = 0; 772 773 c->options |= MIPS_CPU_PREFETCH; 774 break; 775 776 case CPU_VR4133: 777 write_c0_config(config & ~VR41_CONF_P4K); 778 case CPU_VR4131: 779 /* Workaround for cache instruction bug of VR4131 */ 780 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U || 781 c->processor_id == 0x0c82U) { 782 config |= 0x00400000U; 783 if (c->processor_id == 0x0c80U) 784 config |= VR41_CONF_BP; 785 write_c0_config(config); 786 } else 787 c->options |= MIPS_CPU_CACHE_CDEX_P; 788 789 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 790 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 791 c->icache.ways = 2; 792 c->icache.waybit = __ffs(icache_size/2); 793 794 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 795 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 796 c->dcache.ways = 2; 797 c->dcache.waybit = __ffs(dcache_size/2); 798 break; 799 800 case CPU_VR41XX: 801 case CPU_VR4111: 802 case CPU_VR4121: 803 case CPU_VR4122: 804 case CPU_VR4181: 805 case CPU_VR4181A: 806 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 807 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 808 c->icache.ways = 1; 809 c->icache.waybit = 0; /* doesn't matter */ 810 811 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 812 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 813 c->dcache.ways = 1; 814 c->dcache.waybit = 0; /* does not matter */ 815 816 c->options |= MIPS_CPU_CACHE_CDEX_P; 817 break; 818 819 case CPU_RM7000: 820 rm7k_erratum31(); 821 822 case CPU_RM9000: 823 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 824 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 825 c->icache.ways = 4; 826 c->icache.waybit = __ffs(icache_size / c->icache.ways); 827 828 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 829 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 830 c->dcache.ways = 4; 831 c->dcache.waybit = __ffs(dcache_size / c->dcache.ways); 832 833 #if !defined(CONFIG_SMP) || !defined(RM9000_CDEX_SMP_WAR) 834 c->options |= MIPS_CPU_CACHE_CDEX_P; 835 #endif 836 c->options |= MIPS_CPU_PREFETCH; 837 break; 838 839 case CPU_LOONGSON2: 840 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 841 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 842 if (prid & 0x3) 843 c->icache.ways = 4; 844 else 845 c->icache.ways = 2; 846 c->icache.waybit = 0; 847 848 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 849 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 850 if (prid & 0x3) 851 c->dcache.ways = 4; 852 else 853 c->dcache.ways = 2; 854 c->dcache.waybit = 0; 855 break; 856 857 default: 858 if (!(config & MIPS_CONF_M)) 859 panic("Don't know how to probe P-caches on this cpu."); 860 861 /* 862 * So we seem to be a MIPS32 or MIPS64 CPU 863 * So let's probe the I-cache ... 864 */ 865 config1 = read_c0_config1(); 866 867 if ((lsize = ((config1 >> 19) & 7))) 868 c->icache.linesz = 2 << lsize; 869 else 870 c->icache.linesz = lsize; 871 c->icache.sets = 64 << ((config1 >> 22) & 7); 872 c->icache.ways = 1 + ((config1 >> 16) & 7); 873 874 icache_size = c->icache.sets * 875 c->icache.ways * 876 c->icache.linesz; 877 c->icache.waybit = __ffs(icache_size/c->icache.ways); 878 879 if (config & 0x8) /* VI bit */ 880 c->icache.flags |= MIPS_CACHE_VTAG; 881 882 /* 883 * Now probe the MIPS32 / MIPS64 data cache. 884 */ 885 c->dcache.flags = 0; 886 887 if ((lsize = ((config1 >> 10) & 7))) 888 c->dcache.linesz = 2 << lsize; 889 else 890 c->dcache.linesz= lsize; 891 c->dcache.sets = 64 << ((config1 >> 13) & 7); 892 c->dcache.ways = 1 + ((config1 >> 7) & 7); 893 894 dcache_size = c->dcache.sets * 895 c->dcache.ways * 896 c->dcache.linesz; 897 c->dcache.waybit = __ffs(dcache_size/c->dcache.ways); 898 899 c->options |= MIPS_CPU_PREFETCH; 900 break; 901 } 902 903 /* 904 * Processor configuration sanity check for the R4000SC erratum 905 * #5. With page sizes larger than 32kB there is no possibility 906 * to get a VCE exception anymore so we don't care about this 907 * misconfiguration. The case is rather theoretical anyway; 908 * presumably no vendor is shipping his hardware in the "bad" 909 * configuration. 910 */ 911 if ((prid & 0xff00) == PRID_IMP_R4000 && (prid & 0xff) < 0x40 && 912 !(config & CONF_SC) && c->icache.linesz != 16 && 913 PAGE_SIZE <= 0x8000) 914 panic("Improper R4000SC processor configuration detected"); 915 916 /* compute a couple of other cache variables */ 917 c->icache.waysize = icache_size / c->icache.ways; 918 c->dcache.waysize = dcache_size / c->dcache.ways; 919 920 c->icache.sets = c->icache.linesz ? 921 icache_size / (c->icache.linesz * c->icache.ways) : 0; 922 c->dcache.sets = c->dcache.linesz ? 923 dcache_size / (c->dcache.linesz * c->dcache.ways) : 0; 924 925 /* 926 * R10000 and R12000 P-caches are odd in a positive way. They're 32kB 927 * 2-way virtually indexed so normally would suffer from aliases. So 928 * normally they'd suffer from aliases but magic in the hardware deals 929 * with that for us so we don't need to take care ourselves. 930 */ 931 switch (c->cputype) { 932 case CPU_20KC: 933 case CPU_25KF: 934 case CPU_SB1: 935 case CPU_SB1A: 936 c->dcache.flags |= MIPS_CACHE_PINDEX; 937 break; 938 939 case CPU_R10000: 940 case CPU_R12000: 941 case CPU_R14000: 942 break; 943 944 case CPU_24K: 945 case CPU_34K: 946 case CPU_74K: 947 if ((read_c0_config7() & (1 << 16))) { 948 /* effectively physically indexed dcache, 949 thus no virtual aliases. */ 950 c->dcache.flags |= MIPS_CACHE_PINDEX; 951 break; 952 } 953 default: 954 if (c->dcache.waysize > PAGE_SIZE) 955 c->dcache.flags |= MIPS_CACHE_ALIASES; 956 } 957 958 switch (c->cputype) { 959 case CPU_20KC: 960 /* 961 * Some older 20Kc chips doesn't have the 'VI' bit in 962 * the config register. 963 */ 964 c->icache.flags |= MIPS_CACHE_VTAG; 965 break; 966 967 case CPU_AU1000: 968 case CPU_AU1500: 969 case CPU_AU1100: 970 case CPU_AU1550: 971 case CPU_AU1200: 972 c->icache.flags |= MIPS_CACHE_IC_F_DC; 973 break; 974 } 975 976 #ifdef CONFIG_CPU_LOONGSON2 977 /* 978 * LOONGSON2 has 4 way icache, but when using indexed cache op, 979 * one op will act on all 4 ways 980 */ 981 c->icache.ways = 1; 982 #endif 983 984 printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n", 985 icache_size >> 10, 986 cpu_has_vtag_icache ? "virtually tagged" : "physically tagged", 987 way_string[c->icache.ways], c->icache.linesz); 988 989 printk("Primary data cache %ldkB, %s, linesize %d bytes.\n", 990 dcache_size >> 10, way_string[c->dcache.ways], c->dcache.linesz); 991 } 992 993 /* 994 * If you even _breathe_ on this function, look at the gcc output and make sure 995 * it does not pop things on and off the stack for the cache sizing loop that 996 * executes in KSEG1 space or else you will crash and burn badly. You have 997 * been warned. 998 */ 999 static int __init probe_scache(void) 1000 { 1001 unsigned long flags, addr, begin, end, pow2; 1002 unsigned int config = read_c0_config(); 1003 struct cpuinfo_mips *c = ¤t_cpu_data; 1004 int tmp; 1005 1006 if (config & CONF_SC) 1007 return 0; 1008 1009 begin = (unsigned long) &_stext; 1010 begin &= ~((4 * 1024 * 1024) - 1); 1011 end = begin + (4 * 1024 * 1024); 1012 1013 /* 1014 * This is such a bitch, you'd think they would make it easy to do 1015 * this. Away you daemons of stupidity! 1016 */ 1017 local_irq_save(flags); 1018 1019 /* Fill each size-multiple cache line with a valid tag. */ 1020 pow2 = (64 * 1024); 1021 for (addr = begin; addr < end; addr = (begin + pow2)) { 1022 unsigned long *p = (unsigned long *) addr; 1023 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */ 1024 pow2 <<= 1; 1025 } 1026 1027 /* Load first line with zero (therefore invalid) tag. */ 1028 write_c0_taglo(0); 1029 write_c0_taghi(0); 1030 __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */ 1031 cache_op(Index_Store_Tag_I, begin); 1032 cache_op(Index_Store_Tag_D, begin); 1033 cache_op(Index_Store_Tag_SD, begin); 1034 1035 /* Now search for the wrap around point. */ 1036 pow2 = (128 * 1024); 1037 tmp = 0; 1038 for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) { 1039 cache_op(Index_Load_Tag_SD, addr); 1040 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */ 1041 if (!read_c0_taglo()) 1042 break; 1043 pow2 <<= 1; 1044 } 1045 local_irq_restore(flags); 1046 addr -= begin; 1047 1048 scache_size = addr; 1049 c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22); 1050 c->scache.ways = 1; 1051 c->dcache.waybit = 0; /* does not matter */ 1052 1053 return 1; 1054 } 1055 1056 #if defined(CONFIG_CPU_LOONGSON2) 1057 static void __init loongson2_sc_init(void) 1058 { 1059 struct cpuinfo_mips *c = ¤t_cpu_data; 1060 1061 scache_size = 512*1024; 1062 c->scache.linesz = 32; 1063 c->scache.ways = 4; 1064 c->scache.waybit = 0; 1065 c->scache.waysize = scache_size / (c->scache.ways); 1066 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways); 1067 pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n", 1068 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); 1069 1070 c->options |= MIPS_CPU_INCLUSIVE_CACHES; 1071 } 1072 #endif 1073 1074 extern int r5k_sc_init(void); 1075 extern int rm7k_sc_init(void); 1076 extern int mips_sc_init(void); 1077 1078 static void __init setup_scache(void) 1079 { 1080 struct cpuinfo_mips *c = ¤t_cpu_data; 1081 unsigned int config = read_c0_config(); 1082 int sc_present = 0; 1083 1084 /* 1085 * Do the probing thing on R4000SC and R4400SC processors. Other 1086 * processors don't have a S-cache that would be relevant to the 1087 * Linux memory managment. 1088 */ 1089 switch (c->cputype) { 1090 case CPU_R4000SC: 1091 case CPU_R4000MC: 1092 case CPU_R4400SC: 1093 case CPU_R4400MC: 1094 sc_present = run_uncached(probe_scache); 1095 if (sc_present) 1096 c->options |= MIPS_CPU_CACHE_CDEX_S; 1097 break; 1098 1099 case CPU_R10000: 1100 case CPU_R12000: 1101 case CPU_R14000: 1102 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16); 1103 c->scache.linesz = 64 << ((config >> 13) & 1); 1104 c->scache.ways = 2; 1105 c->scache.waybit= 0; 1106 sc_present = 1; 1107 break; 1108 1109 case CPU_R5000: 1110 case CPU_NEVADA: 1111 #ifdef CONFIG_R5000_CPU_SCACHE 1112 r5k_sc_init(); 1113 #endif 1114 return; 1115 1116 case CPU_RM7000: 1117 case CPU_RM9000: 1118 #ifdef CONFIG_RM7000_CPU_SCACHE 1119 rm7k_sc_init(); 1120 #endif 1121 return; 1122 1123 #if defined(CONFIG_CPU_LOONGSON2) 1124 case CPU_LOONGSON2: 1125 loongson2_sc_init(); 1126 return; 1127 #endif 1128 1129 default: 1130 if (c->isa_level == MIPS_CPU_ISA_M32R1 || 1131 c->isa_level == MIPS_CPU_ISA_M32R2 || 1132 c->isa_level == MIPS_CPU_ISA_M64R1 || 1133 c->isa_level == MIPS_CPU_ISA_M64R2) { 1134 #ifdef CONFIG_MIPS_CPU_SCACHE 1135 if (mips_sc_init ()) { 1136 scache_size = c->scache.ways * c->scache.sets * c->scache.linesz; 1137 printk("MIPS secondary cache %ldkB, %s, linesize %d bytes.\n", 1138 scache_size >> 10, 1139 way_string[c->scache.ways], c->scache.linesz); 1140 } 1141 #else 1142 if (!(c->scache.flags & MIPS_CACHE_NOT_PRESENT)) 1143 panic("Dunno how to handle MIPS32 / MIPS64 second level cache"); 1144 #endif 1145 return; 1146 } 1147 sc_present = 0; 1148 } 1149 1150 if (!sc_present) 1151 return; 1152 1153 /* compute a couple of other cache variables */ 1154 c->scache.waysize = scache_size / c->scache.ways; 1155 1156 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways); 1157 1158 printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n", 1159 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); 1160 1161 c->options |= MIPS_CPU_INCLUSIVE_CACHES; 1162 } 1163 1164 void au1x00_fixup_config_od(void) 1165 { 1166 /* 1167 * c0_config.od (bit 19) was write only (and read as 0) 1168 * on the early revisions of Alchemy SOCs. It disables the bus 1169 * transaction overlapping and needs to be set to fix various errata. 1170 */ 1171 switch (read_c0_prid()) { 1172 case 0x00030100: /* Au1000 DA */ 1173 case 0x00030201: /* Au1000 HA */ 1174 case 0x00030202: /* Au1000 HB */ 1175 case 0x01030200: /* Au1500 AB */ 1176 /* 1177 * Au1100 errata actually keeps silence about this bit, so we set it 1178 * just in case for those revisions that require it to be set according 1179 * to arch/mips/au1000/common/cputable.c 1180 */ 1181 case 0x02030200: /* Au1100 AB */ 1182 case 0x02030201: /* Au1100 BA */ 1183 case 0x02030202: /* Au1100 BC */ 1184 set_c0_config(1 << 19); 1185 break; 1186 } 1187 } 1188 1189 static void __init coherency_setup(void) 1190 { 1191 change_c0_config(CONF_CM_CMASK, CONF_CM_DEFAULT); 1192 1193 /* 1194 * c0_status.cu=0 specifies that updates by the sc instruction use 1195 * the coherency mode specified by the TLB; 1 means cachable 1196 * coherent update on write will be used. Not all processors have 1197 * this bit and; some wire it to zero, others like Toshiba had the 1198 * silly idea of putting something else there ... 1199 */ 1200 switch (current_cpu_type()) { 1201 case CPU_R4000PC: 1202 case CPU_R4000SC: 1203 case CPU_R4000MC: 1204 case CPU_R4400PC: 1205 case CPU_R4400SC: 1206 case CPU_R4400MC: 1207 clear_c0_config(CONF_CU); 1208 break; 1209 /* 1210 * We need to catch the early Alchemy SOCs with 1211 * the write-only co_config.od bit and set it back to one... 1212 */ 1213 case CPU_AU1000: /* rev. DA, HA, HB */ 1214 case CPU_AU1100: /* rev. AB, BA, BC ?? */ 1215 case CPU_AU1500: /* rev. AB */ 1216 au1x00_fixup_config_od(); 1217 break; 1218 } 1219 } 1220 1221 void __init r4k_cache_init(void) 1222 { 1223 extern void build_clear_page(void); 1224 extern void build_copy_page(void); 1225 extern char __weak except_vec2_generic; 1226 extern char __weak except_vec2_sb1; 1227 struct cpuinfo_mips *c = ¤t_cpu_data; 1228 1229 switch (c->cputype) { 1230 case CPU_SB1: 1231 case CPU_SB1A: 1232 set_uncached_handler(0x100, &except_vec2_sb1, 0x80); 1233 break; 1234 1235 default: 1236 set_uncached_handler(0x100, &except_vec2_generic, 0x80); 1237 break; 1238 } 1239 1240 probe_pcache(); 1241 setup_scache(); 1242 1243 r4k_blast_dcache_page_setup(); 1244 r4k_blast_dcache_page_indexed_setup(); 1245 r4k_blast_dcache_setup(); 1246 r4k_blast_icache_page_setup(); 1247 r4k_blast_icache_page_indexed_setup(); 1248 r4k_blast_icache_setup(); 1249 r4k_blast_scache_page_setup(); 1250 r4k_blast_scache_page_indexed_setup(); 1251 r4k_blast_scache_setup(); 1252 1253 /* 1254 * Some MIPS32 and MIPS64 processors have physically indexed caches. 1255 * This code supports virtually indexed processors and will be 1256 * unnecessarily inefficient on physically indexed processors. 1257 */ 1258 if (c->dcache.linesz) 1259 shm_align_mask = max_t( unsigned long, 1260 c->dcache.sets * c->dcache.linesz - 1, 1261 PAGE_SIZE - 1); 1262 else 1263 shm_align_mask = PAGE_SIZE-1; 1264 flush_cache_all = cache_noop; 1265 __flush_cache_all = r4k___flush_cache_all; 1266 flush_cache_mm = r4k_flush_cache_mm; 1267 flush_cache_page = r4k_flush_cache_page; 1268 flush_cache_range = r4k_flush_cache_range; 1269 1270 flush_cache_sigtramp = r4k_flush_cache_sigtramp; 1271 flush_icache_all = r4k_flush_icache_all; 1272 local_flush_data_cache_page = local_r4k_flush_data_cache_page; 1273 flush_data_cache_page = r4k_flush_data_cache_page; 1274 flush_icache_range = r4k_flush_icache_range; 1275 1276 #ifdef CONFIG_DMA_NONCOHERENT 1277 _dma_cache_wback_inv = r4k_dma_cache_wback_inv; 1278 _dma_cache_wback = r4k_dma_cache_wback_inv; 1279 _dma_cache_inv = r4k_dma_cache_inv; 1280 #endif 1281 1282 build_clear_page(); 1283 build_copy_page(); 1284 local_r4k___flush_cache_all(NULL); 1285 coherency_setup(); 1286 } 1287