1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996 David S. Miller (davem@davemloft.net) 7 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org) 8 * Copyright (C) 1999, 2000 Silicon Graphics, Inc. 9 */ 10 #include <linux/cpu_pm.h> 11 #include <linux/hardirq.h> 12 #include <linux/init.h> 13 #include <linux/highmem.h> 14 #include <linux/kernel.h> 15 #include <linux/linkage.h> 16 #include <linux/preempt.h> 17 #include <linux/sched.h> 18 #include <linux/smp.h> 19 #include <linux/mm.h> 20 #include <linux/module.h> 21 #include <linux/bitops.h> 22 23 #include <asm/bcache.h> 24 #include <asm/bootinfo.h> 25 #include <asm/cache.h> 26 #include <asm/cacheops.h> 27 #include <asm/cpu.h> 28 #include <asm/cpu-features.h> 29 #include <asm/cpu-type.h> 30 #include <asm/io.h> 31 #include <asm/page.h> 32 #include <asm/pgtable.h> 33 #include <asm/r4kcache.h> 34 #include <asm/sections.h> 35 #include <asm/mmu_context.h> 36 #include <asm/war.h> 37 #include <asm/cacheflush.h> /* for run_uncached() */ 38 #include <asm/traps.h> 39 #include <asm/dma-coherence.h> 40 #include <asm/mips-cm.h> 41 42 /* 43 * Special Variant of smp_call_function for use by cache functions: 44 * 45 * o No return value 46 * o collapses to normal function call on UP kernels 47 * o collapses to normal function call on systems with a single shared 48 * primary cache. 49 * o doesn't disable interrupts on the local CPU 50 */ 51 static inline void r4k_on_each_cpu(void (*func) (void *info), void *info) 52 { 53 preempt_disable(); 54 55 /* 56 * The Coherent Manager propagates address-based cache ops to other 57 * cores but not index-based ops. However, r4k_on_each_cpu is used 58 * in both cases so there is no easy way to tell what kind of op is 59 * executed to the other cores. The best we can probably do is 60 * to restrict that call when a CM is not present because both 61 * CM-based SMP protocols (CMP & CPS) restrict index-based cache ops. 62 */ 63 if (!mips_cm_present()) 64 smp_call_function_many(&cpu_foreign_map, func, info, 1); 65 func(info); 66 preempt_enable(); 67 } 68 69 #if defined(CONFIG_MIPS_CMP) || defined(CONFIG_MIPS_CPS) 70 #define cpu_has_safe_index_cacheops 0 71 #else 72 #define cpu_has_safe_index_cacheops 1 73 #endif 74 75 /* 76 * Must die. 77 */ 78 static unsigned long icache_size __read_mostly; 79 static unsigned long dcache_size __read_mostly; 80 static unsigned long vcache_size __read_mostly; 81 static unsigned long scache_size __read_mostly; 82 83 /* 84 * Dummy cache handling routines for machines without boardcaches 85 */ 86 static void cache_noop(void) {} 87 88 static struct bcache_ops no_sc_ops = { 89 .bc_enable = (void *)cache_noop, 90 .bc_disable = (void *)cache_noop, 91 .bc_wback_inv = (void *)cache_noop, 92 .bc_inv = (void *)cache_noop 93 }; 94 95 struct bcache_ops *bcops = &no_sc_ops; 96 97 #define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010) 98 #define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020) 99 100 #define R4600_HIT_CACHEOP_WAR_IMPL \ 101 do { \ 102 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) \ 103 *(volatile unsigned long *)CKSEG1; \ 104 if (R4600_V1_HIT_CACHEOP_WAR) \ 105 __asm__ __volatile__("nop;nop;nop;nop"); \ 106 } while (0) 107 108 static void (*r4k_blast_dcache_page)(unsigned long addr); 109 110 static inline void r4k_blast_dcache_page_dc32(unsigned long addr) 111 { 112 R4600_HIT_CACHEOP_WAR_IMPL; 113 blast_dcache32_page(addr); 114 } 115 116 static inline void r4k_blast_dcache_page_dc64(unsigned long addr) 117 { 118 blast_dcache64_page(addr); 119 } 120 121 static inline void r4k_blast_dcache_page_dc128(unsigned long addr) 122 { 123 blast_dcache128_page(addr); 124 } 125 126 static void r4k_blast_dcache_page_setup(void) 127 { 128 unsigned long dc_lsize = cpu_dcache_line_size(); 129 130 switch (dc_lsize) { 131 case 0: 132 r4k_blast_dcache_page = (void *)cache_noop; 133 break; 134 case 16: 135 r4k_blast_dcache_page = blast_dcache16_page; 136 break; 137 case 32: 138 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32; 139 break; 140 case 64: 141 r4k_blast_dcache_page = r4k_blast_dcache_page_dc64; 142 break; 143 case 128: 144 r4k_blast_dcache_page = r4k_blast_dcache_page_dc128; 145 break; 146 default: 147 break; 148 } 149 } 150 151 #ifndef CONFIG_EVA 152 #define r4k_blast_dcache_user_page r4k_blast_dcache_page 153 #else 154 155 static void (*r4k_blast_dcache_user_page)(unsigned long addr); 156 157 static void r4k_blast_dcache_user_page_setup(void) 158 { 159 unsigned long dc_lsize = cpu_dcache_line_size(); 160 161 if (dc_lsize == 0) 162 r4k_blast_dcache_user_page = (void *)cache_noop; 163 else if (dc_lsize == 16) 164 r4k_blast_dcache_user_page = blast_dcache16_user_page; 165 else if (dc_lsize == 32) 166 r4k_blast_dcache_user_page = blast_dcache32_user_page; 167 else if (dc_lsize == 64) 168 r4k_blast_dcache_user_page = blast_dcache64_user_page; 169 } 170 171 #endif 172 173 static void (* r4k_blast_dcache_page_indexed)(unsigned long addr); 174 175 static void r4k_blast_dcache_page_indexed_setup(void) 176 { 177 unsigned long dc_lsize = cpu_dcache_line_size(); 178 179 if (dc_lsize == 0) 180 r4k_blast_dcache_page_indexed = (void *)cache_noop; 181 else if (dc_lsize == 16) 182 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed; 183 else if (dc_lsize == 32) 184 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed; 185 else if (dc_lsize == 64) 186 r4k_blast_dcache_page_indexed = blast_dcache64_page_indexed; 187 else if (dc_lsize == 128) 188 r4k_blast_dcache_page_indexed = blast_dcache128_page_indexed; 189 } 190 191 void (* r4k_blast_dcache)(void); 192 EXPORT_SYMBOL(r4k_blast_dcache); 193 194 static void r4k_blast_dcache_setup(void) 195 { 196 unsigned long dc_lsize = cpu_dcache_line_size(); 197 198 if (dc_lsize == 0) 199 r4k_blast_dcache = (void *)cache_noop; 200 else if (dc_lsize == 16) 201 r4k_blast_dcache = blast_dcache16; 202 else if (dc_lsize == 32) 203 r4k_blast_dcache = blast_dcache32; 204 else if (dc_lsize == 64) 205 r4k_blast_dcache = blast_dcache64; 206 else if (dc_lsize == 128) 207 r4k_blast_dcache = blast_dcache128; 208 } 209 210 /* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */ 211 #define JUMP_TO_ALIGN(order) \ 212 __asm__ __volatile__( \ 213 "b\t1f\n\t" \ 214 ".align\t" #order "\n\t" \ 215 "1:\n\t" \ 216 ) 217 #define CACHE32_UNROLL32_ALIGN JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */ 218 #define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11) 219 220 static inline void blast_r4600_v1_icache32(void) 221 { 222 unsigned long flags; 223 224 local_irq_save(flags); 225 blast_icache32(); 226 local_irq_restore(flags); 227 } 228 229 static inline void tx49_blast_icache32(void) 230 { 231 unsigned long start = INDEX_BASE; 232 unsigned long end = start + current_cpu_data.icache.waysize; 233 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 234 unsigned long ws_end = current_cpu_data.icache.ways << 235 current_cpu_data.icache.waybit; 236 unsigned long ws, addr; 237 238 CACHE32_UNROLL32_ALIGN2; 239 /* I'm in even chunk. blast odd chunks */ 240 for (ws = 0; ws < ws_end; ws += ws_inc) 241 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 242 cache32_unroll32(addr|ws, Index_Invalidate_I); 243 CACHE32_UNROLL32_ALIGN; 244 /* I'm in odd chunk. blast even chunks */ 245 for (ws = 0; ws < ws_end; ws += ws_inc) 246 for (addr = start; addr < end; addr += 0x400 * 2) 247 cache32_unroll32(addr|ws, Index_Invalidate_I); 248 } 249 250 static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page) 251 { 252 unsigned long flags; 253 254 local_irq_save(flags); 255 blast_icache32_page_indexed(page); 256 local_irq_restore(flags); 257 } 258 259 static inline void tx49_blast_icache32_page_indexed(unsigned long page) 260 { 261 unsigned long indexmask = current_cpu_data.icache.waysize - 1; 262 unsigned long start = INDEX_BASE + (page & indexmask); 263 unsigned long end = start + PAGE_SIZE; 264 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 265 unsigned long ws_end = current_cpu_data.icache.ways << 266 current_cpu_data.icache.waybit; 267 unsigned long ws, addr; 268 269 CACHE32_UNROLL32_ALIGN2; 270 /* I'm in even chunk. blast odd chunks */ 271 for (ws = 0; ws < ws_end; ws += ws_inc) 272 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 273 cache32_unroll32(addr|ws, Index_Invalidate_I); 274 CACHE32_UNROLL32_ALIGN; 275 /* I'm in odd chunk. blast even chunks */ 276 for (ws = 0; ws < ws_end; ws += ws_inc) 277 for (addr = start; addr < end; addr += 0x400 * 2) 278 cache32_unroll32(addr|ws, Index_Invalidate_I); 279 } 280 281 static void (* r4k_blast_icache_page)(unsigned long addr); 282 283 static void r4k_blast_icache_page_setup(void) 284 { 285 unsigned long ic_lsize = cpu_icache_line_size(); 286 287 if (ic_lsize == 0) 288 r4k_blast_icache_page = (void *)cache_noop; 289 else if (ic_lsize == 16) 290 r4k_blast_icache_page = blast_icache16_page; 291 else if (ic_lsize == 32 && current_cpu_type() == CPU_LOONGSON2) 292 r4k_blast_icache_page = loongson2_blast_icache32_page; 293 else if (ic_lsize == 32) 294 r4k_blast_icache_page = blast_icache32_page; 295 else if (ic_lsize == 64) 296 r4k_blast_icache_page = blast_icache64_page; 297 else if (ic_lsize == 128) 298 r4k_blast_icache_page = blast_icache128_page; 299 } 300 301 #ifndef CONFIG_EVA 302 #define r4k_blast_icache_user_page r4k_blast_icache_page 303 #else 304 305 static void (*r4k_blast_icache_user_page)(unsigned long addr); 306 307 static void r4k_blast_icache_user_page_setup(void) 308 { 309 unsigned long ic_lsize = cpu_icache_line_size(); 310 311 if (ic_lsize == 0) 312 r4k_blast_icache_user_page = (void *)cache_noop; 313 else if (ic_lsize == 16) 314 r4k_blast_icache_user_page = blast_icache16_user_page; 315 else if (ic_lsize == 32) 316 r4k_blast_icache_user_page = blast_icache32_user_page; 317 else if (ic_lsize == 64) 318 r4k_blast_icache_user_page = blast_icache64_user_page; 319 } 320 321 #endif 322 323 static void (* r4k_blast_icache_page_indexed)(unsigned long addr); 324 325 static void r4k_blast_icache_page_indexed_setup(void) 326 { 327 unsigned long ic_lsize = cpu_icache_line_size(); 328 329 if (ic_lsize == 0) 330 r4k_blast_icache_page_indexed = (void *)cache_noop; 331 else if (ic_lsize == 16) 332 r4k_blast_icache_page_indexed = blast_icache16_page_indexed; 333 else if (ic_lsize == 32) { 334 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 335 r4k_blast_icache_page_indexed = 336 blast_icache32_r4600_v1_page_indexed; 337 else if (TX49XX_ICACHE_INDEX_INV_WAR) 338 r4k_blast_icache_page_indexed = 339 tx49_blast_icache32_page_indexed; 340 else if (current_cpu_type() == CPU_LOONGSON2) 341 r4k_blast_icache_page_indexed = 342 loongson2_blast_icache32_page_indexed; 343 else 344 r4k_blast_icache_page_indexed = 345 blast_icache32_page_indexed; 346 } else if (ic_lsize == 64) 347 r4k_blast_icache_page_indexed = blast_icache64_page_indexed; 348 } 349 350 void (* r4k_blast_icache)(void); 351 EXPORT_SYMBOL(r4k_blast_icache); 352 353 static void r4k_blast_icache_setup(void) 354 { 355 unsigned long ic_lsize = cpu_icache_line_size(); 356 357 if (ic_lsize == 0) 358 r4k_blast_icache = (void *)cache_noop; 359 else if (ic_lsize == 16) 360 r4k_blast_icache = blast_icache16; 361 else if (ic_lsize == 32) { 362 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 363 r4k_blast_icache = blast_r4600_v1_icache32; 364 else if (TX49XX_ICACHE_INDEX_INV_WAR) 365 r4k_blast_icache = tx49_blast_icache32; 366 else if (current_cpu_type() == CPU_LOONGSON2) 367 r4k_blast_icache = loongson2_blast_icache32; 368 else 369 r4k_blast_icache = blast_icache32; 370 } else if (ic_lsize == 64) 371 r4k_blast_icache = blast_icache64; 372 else if (ic_lsize == 128) 373 r4k_blast_icache = blast_icache128; 374 } 375 376 static void (* r4k_blast_scache_page)(unsigned long addr); 377 378 static void r4k_blast_scache_page_setup(void) 379 { 380 unsigned long sc_lsize = cpu_scache_line_size(); 381 382 if (scache_size == 0) 383 r4k_blast_scache_page = (void *)cache_noop; 384 else if (sc_lsize == 16) 385 r4k_blast_scache_page = blast_scache16_page; 386 else if (sc_lsize == 32) 387 r4k_blast_scache_page = blast_scache32_page; 388 else if (sc_lsize == 64) 389 r4k_blast_scache_page = blast_scache64_page; 390 else if (sc_lsize == 128) 391 r4k_blast_scache_page = blast_scache128_page; 392 } 393 394 static void (* r4k_blast_scache_page_indexed)(unsigned long addr); 395 396 static void r4k_blast_scache_page_indexed_setup(void) 397 { 398 unsigned long sc_lsize = cpu_scache_line_size(); 399 400 if (scache_size == 0) 401 r4k_blast_scache_page_indexed = (void *)cache_noop; 402 else if (sc_lsize == 16) 403 r4k_blast_scache_page_indexed = blast_scache16_page_indexed; 404 else if (sc_lsize == 32) 405 r4k_blast_scache_page_indexed = blast_scache32_page_indexed; 406 else if (sc_lsize == 64) 407 r4k_blast_scache_page_indexed = blast_scache64_page_indexed; 408 else if (sc_lsize == 128) 409 r4k_blast_scache_page_indexed = blast_scache128_page_indexed; 410 } 411 412 static void (* r4k_blast_scache)(void); 413 414 static void r4k_blast_scache_setup(void) 415 { 416 unsigned long sc_lsize = cpu_scache_line_size(); 417 418 if (scache_size == 0) 419 r4k_blast_scache = (void *)cache_noop; 420 else if (sc_lsize == 16) 421 r4k_blast_scache = blast_scache16; 422 else if (sc_lsize == 32) 423 r4k_blast_scache = blast_scache32; 424 else if (sc_lsize == 64) 425 r4k_blast_scache = blast_scache64; 426 else if (sc_lsize == 128) 427 r4k_blast_scache = blast_scache128; 428 } 429 430 static inline void local_r4k___flush_cache_all(void * args) 431 { 432 switch (current_cpu_type()) { 433 case CPU_LOONGSON2: 434 case CPU_LOONGSON3: 435 case CPU_R4000SC: 436 case CPU_R4000MC: 437 case CPU_R4400SC: 438 case CPU_R4400MC: 439 case CPU_R10000: 440 case CPU_R12000: 441 case CPU_R14000: 442 case CPU_R16000: 443 /* 444 * These caches are inclusive caches, that is, if something 445 * is not cached in the S-cache, we know it also won't be 446 * in one of the primary caches. 447 */ 448 r4k_blast_scache(); 449 break; 450 451 case CPU_BMIPS5000: 452 r4k_blast_scache(); 453 __sync(); 454 break; 455 456 default: 457 r4k_blast_dcache(); 458 r4k_blast_icache(); 459 break; 460 } 461 } 462 463 static void r4k___flush_cache_all(void) 464 { 465 r4k_on_each_cpu(local_r4k___flush_cache_all, NULL); 466 } 467 468 static inline int has_valid_asid(const struct mm_struct *mm) 469 { 470 #ifdef CONFIG_MIPS_MT_SMP 471 int i; 472 473 for_each_online_cpu(i) 474 if (cpu_context(i, mm)) 475 return 1; 476 477 return 0; 478 #else 479 return cpu_context(smp_processor_id(), mm); 480 #endif 481 } 482 483 static void r4k__flush_cache_vmap(void) 484 { 485 r4k_blast_dcache(); 486 } 487 488 static void r4k__flush_cache_vunmap(void) 489 { 490 r4k_blast_dcache(); 491 } 492 493 static inline void local_r4k_flush_cache_range(void * args) 494 { 495 struct vm_area_struct *vma = args; 496 int exec = vma->vm_flags & VM_EXEC; 497 498 if (!(has_valid_asid(vma->vm_mm))) 499 return; 500 501 /* 502 * If dcache can alias, we must blast it since mapping is changing. 503 * If executable, we must ensure any dirty lines are written back far 504 * enough to be visible to icache. 505 */ 506 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) 507 r4k_blast_dcache(); 508 /* If executable, blast stale lines from icache */ 509 if (exec) 510 r4k_blast_icache(); 511 } 512 513 static void r4k_flush_cache_range(struct vm_area_struct *vma, 514 unsigned long start, unsigned long end) 515 { 516 int exec = vma->vm_flags & VM_EXEC; 517 518 if (cpu_has_dc_aliases || exec) 519 r4k_on_each_cpu(local_r4k_flush_cache_range, vma); 520 } 521 522 static inline void local_r4k_flush_cache_mm(void * args) 523 { 524 struct mm_struct *mm = args; 525 526 if (!has_valid_asid(mm)) 527 return; 528 529 /* 530 * Kludge alert. For obscure reasons R4000SC and R4400SC go nuts if we 531 * only flush the primary caches but R1x000 behave sane ... 532 * R4000SC and R4400SC indexed S-cache ops also invalidate primary 533 * caches, so we can bail out early. 534 */ 535 if (current_cpu_type() == CPU_R4000SC || 536 current_cpu_type() == CPU_R4000MC || 537 current_cpu_type() == CPU_R4400SC || 538 current_cpu_type() == CPU_R4400MC) { 539 r4k_blast_scache(); 540 return; 541 } 542 543 r4k_blast_dcache(); 544 } 545 546 static void r4k_flush_cache_mm(struct mm_struct *mm) 547 { 548 if (!cpu_has_dc_aliases) 549 return; 550 551 r4k_on_each_cpu(local_r4k_flush_cache_mm, mm); 552 } 553 554 struct flush_cache_page_args { 555 struct vm_area_struct *vma; 556 unsigned long addr; 557 unsigned long pfn; 558 }; 559 560 static inline void local_r4k_flush_cache_page(void *args) 561 { 562 struct flush_cache_page_args *fcp_args = args; 563 struct vm_area_struct *vma = fcp_args->vma; 564 unsigned long addr = fcp_args->addr; 565 struct page *page = pfn_to_page(fcp_args->pfn); 566 int exec = vma->vm_flags & VM_EXEC; 567 struct mm_struct *mm = vma->vm_mm; 568 int map_coherent = 0; 569 pgd_t *pgdp; 570 pud_t *pudp; 571 pmd_t *pmdp; 572 pte_t *ptep; 573 void *vaddr; 574 575 /* 576 * If ownes no valid ASID yet, cannot possibly have gotten 577 * this page into the cache. 578 */ 579 if (!has_valid_asid(mm)) 580 return; 581 582 addr &= PAGE_MASK; 583 pgdp = pgd_offset(mm, addr); 584 pudp = pud_offset(pgdp, addr); 585 pmdp = pmd_offset(pudp, addr); 586 ptep = pte_offset(pmdp, addr); 587 588 /* 589 * If the page isn't marked valid, the page cannot possibly be 590 * in the cache. 591 */ 592 if (!(pte_present(*ptep))) 593 return; 594 595 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) 596 vaddr = NULL; 597 else { 598 /* 599 * Use kmap_coherent or kmap_atomic to do flushes for 600 * another ASID than the current one. 601 */ 602 map_coherent = (cpu_has_dc_aliases && 603 page_mapcount(page) && 604 !Page_dcache_dirty(page)); 605 if (map_coherent) 606 vaddr = kmap_coherent(page, addr); 607 else 608 vaddr = kmap_atomic(page); 609 addr = (unsigned long)vaddr; 610 } 611 612 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { 613 vaddr ? r4k_blast_dcache_page(addr) : 614 r4k_blast_dcache_user_page(addr); 615 if (exec && !cpu_icache_snoops_remote_store) 616 r4k_blast_scache_page(addr); 617 } 618 if (exec) { 619 if (vaddr && cpu_has_vtag_icache && mm == current->active_mm) { 620 int cpu = smp_processor_id(); 621 622 if (cpu_context(cpu, mm) != 0) 623 drop_mmu_context(mm, cpu); 624 } else 625 vaddr ? r4k_blast_icache_page(addr) : 626 r4k_blast_icache_user_page(addr); 627 } 628 629 if (vaddr) { 630 if (map_coherent) 631 kunmap_coherent(); 632 else 633 kunmap_atomic(vaddr); 634 } 635 } 636 637 static void r4k_flush_cache_page(struct vm_area_struct *vma, 638 unsigned long addr, unsigned long pfn) 639 { 640 struct flush_cache_page_args args; 641 642 args.vma = vma; 643 args.addr = addr; 644 args.pfn = pfn; 645 646 r4k_on_each_cpu(local_r4k_flush_cache_page, &args); 647 } 648 649 static inline void local_r4k_flush_data_cache_page(void * addr) 650 { 651 r4k_blast_dcache_page((unsigned long) addr); 652 } 653 654 static void r4k_flush_data_cache_page(unsigned long addr) 655 { 656 if (in_atomic()) 657 local_r4k_flush_data_cache_page((void *)addr); 658 else 659 r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr); 660 } 661 662 struct flush_icache_range_args { 663 unsigned long start; 664 unsigned long end; 665 }; 666 667 static inline void local_r4k_flush_icache_range(unsigned long start, unsigned long end) 668 { 669 if (!cpu_has_ic_fills_f_dc) { 670 if (end - start >= dcache_size) { 671 r4k_blast_dcache(); 672 } else { 673 R4600_HIT_CACHEOP_WAR_IMPL; 674 protected_blast_dcache_range(start, end); 675 } 676 } 677 678 if (end - start > icache_size) 679 r4k_blast_icache(); 680 else { 681 switch (boot_cpu_type()) { 682 case CPU_LOONGSON2: 683 protected_loongson2_blast_icache_range(start, end); 684 break; 685 686 default: 687 protected_blast_icache_range(start, end); 688 break; 689 } 690 } 691 #ifdef CONFIG_EVA 692 /* 693 * Due to all possible segment mappings, there might cache aliases 694 * caused by the bootloader being in non-EVA mode, and the CPU switching 695 * to EVA during early kernel init. It's best to flush the scache 696 * to avoid having secondary cores fetching stale data and lead to 697 * kernel crashes. 698 */ 699 bc_wback_inv(start, (end - start)); 700 __sync(); 701 #endif 702 } 703 704 static inline void local_r4k_flush_icache_range_ipi(void *args) 705 { 706 struct flush_icache_range_args *fir_args = args; 707 unsigned long start = fir_args->start; 708 unsigned long end = fir_args->end; 709 710 local_r4k_flush_icache_range(start, end); 711 } 712 713 static void r4k_flush_icache_range(unsigned long start, unsigned long end) 714 { 715 struct flush_icache_range_args args; 716 717 args.start = start; 718 args.end = end; 719 720 r4k_on_each_cpu(local_r4k_flush_icache_range_ipi, &args); 721 instruction_hazard(); 722 } 723 724 #if defined(CONFIG_DMA_NONCOHERENT) || defined(CONFIG_DMA_MAYBE_COHERENT) 725 726 static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size) 727 { 728 /* Catch bad driver code */ 729 BUG_ON(size == 0); 730 731 preempt_disable(); 732 if (cpu_has_inclusive_pcaches) { 733 if (size >= scache_size) 734 r4k_blast_scache(); 735 else 736 blast_scache_range(addr, addr + size); 737 preempt_enable(); 738 __sync(); 739 return; 740 } 741 742 /* 743 * Either no secondary cache or the available caches don't have the 744 * subset property so we have to flush the primary caches 745 * explicitly 746 */ 747 if (cpu_has_safe_index_cacheops && size >= dcache_size) { 748 r4k_blast_dcache(); 749 } else { 750 R4600_HIT_CACHEOP_WAR_IMPL; 751 blast_dcache_range(addr, addr + size); 752 } 753 preempt_enable(); 754 755 bc_wback_inv(addr, size); 756 __sync(); 757 } 758 759 static void r4k_dma_cache_inv(unsigned long addr, unsigned long size) 760 { 761 /* Catch bad driver code */ 762 BUG_ON(size == 0); 763 764 preempt_disable(); 765 if (cpu_has_inclusive_pcaches) { 766 if (size >= scache_size) 767 r4k_blast_scache(); 768 else { 769 /* 770 * There is no clearly documented alignment requirement 771 * for the cache instruction on MIPS processors and 772 * some processors, among them the RM5200 and RM7000 773 * QED processors will throw an address error for cache 774 * hit ops with insufficient alignment. Solved by 775 * aligning the address to cache line size. 776 */ 777 blast_inv_scache_range(addr, addr + size); 778 } 779 preempt_enable(); 780 __sync(); 781 return; 782 } 783 784 if (cpu_has_safe_index_cacheops && size >= dcache_size) { 785 r4k_blast_dcache(); 786 } else { 787 R4600_HIT_CACHEOP_WAR_IMPL; 788 blast_inv_dcache_range(addr, addr + size); 789 } 790 preempt_enable(); 791 792 bc_inv(addr, size); 793 __sync(); 794 } 795 #endif /* CONFIG_DMA_NONCOHERENT || CONFIG_DMA_MAYBE_COHERENT */ 796 797 /* 798 * While we're protected against bad userland addresses we don't care 799 * very much about what happens in that case. Usually a segmentation 800 * fault will dump the process later on anyway ... 801 */ 802 static void local_r4k_flush_cache_sigtramp(void * arg) 803 { 804 unsigned long ic_lsize = cpu_icache_line_size(); 805 unsigned long dc_lsize = cpu_dcache_line_size(); 806 unsigned long sc_lsize = cpu_scache_line_size(); 807 unsigned long addr = (unsigned long) arg; 808 809 R4600_HIT_CACHEOP_WAR_IMPL; 810 if (dc_lsize) 811 protected_writeback_dcache_line(addr & ~(dc_lsize - 1)); 812 if (!cpu_icache_snoops_remote_store && scache_size) 813 protected_writeback_scache_line(addr & ~(sc_lsize - 1)); 814 if (ic_lsize) 815 protected_flush_icache_line(addr & ~(ic_lsize - 1)); 816 if (MIPS4K_ICACHE_REFILL_WAR) { 817 __asm__ __volatile__ ( 818 ".set push\n\t" 819 ".set noat\n\t" 820 ".set "MIPS_ISA_LEVEL"\n\t" 821 #ifdef CONFIG_32BIT 822 "la $at,1f\n\t" 823 #endif 824 #ifdef CONFIG_64BIT 825 "dla $at,1f\n\t" 826 #endif 827 "cache %0,($at)\n\t" 828 "nop; nop; nop\n" 829 "1:\n\t" 830 ".set pop" 831 : 832 : "i" (Hit_Invalidate_I)); 833 } 834 if (MIPS_CACHE_SYNC_WAR) 835 __asm__ __volatile__ ("sync"); 836 } 837 838 static void r4k_flush_cache_sigtramp(unsigned long addr) 839 { 840 r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr); 841 } 842 843 static void r4k_flush_icache_all(void) 844 { 845 if (cpu_has_vtag_icache) 846 r4k_blast_icache(); 847 } 848 849 struct flush_kernel_vmap_range_args { 850 unsigned long vaddr; 851 int size; 852 }; 853 854 static inline void local_r4k_flush_kernel_vmap_range(void *args) 855 { 856 struct flush_kernel_vmap_range_args *vmra = args; 857 unsigned long vaddr = vmra->vaddr; 858 int size = vmra->size; 859 860 /* 861 * Aliases only affect the primary caches so don't bother with 862 * S-caches or T-caches. 863 */ 864 if (cpu_has_safe_index_cacheops && size >= dcache_size) 865 r4k_blast_dcache(); 866 else { 867 R4600_HIT_CACHEOP_WAR_IMPL; 868 blast_dcache_range(vaddr, vaddr + size); 869 } 870 } 871 872 static void r4k_flush_kernel_vmap_range(unsigned long vaddr, int size) 873 { 874 struct flush_kernel_vmap_range_args args; 875 876 args.vaddr = (unsigned long) vaddr; 877 args.size = size; 878 879 r4k_on_each_cpu(local_r4k_flush_kernel_vmap_range, &args); 880 } 881 882 static inline void rm7k_erratum31(void) 883 { 884 const unsigned long ic_lsize = 32; 885 unsigned long addr; 886 887 /* RM7000 erratum #31. The icache is screwed at startup. */ 888 write_c0_taglo(0); 889 write_c0_taghi(0); 890 891 for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) { 892 __asm__ __volatile__ ( 893 ".set push\n\t" 894 ".set noreorder\n\t" 895 ".set mips3\n\t" 896 "cache\t%1, 0(%0)\n\t" 897 "cache\t%1, 0x1000(%0)\n\t" 898 "cache\t%1, 0x2000(%0)\n\t" 899 "cache\t%1, 0x3000(%0)\n\t" 900 "cache\t%2, 0(%0)\n\t" 901 "cache\t%2, 0x1000(%0)\n\t" 902 "cache\t%2, 0x2000(%0)\n\t" 903 "cache\t%2, 0x3000(%0)\n\t" 904 "cache\t%1, 0(%0)\n\t" 905 "cache\t%1, 0x1000(%0)\n\t" 906 "cache\t%1, 0x2000(%0)\n\t" 907 "cache\t%1, 0x3000(%0)\n\t" 908 ".set pop\n" 909 : 910 : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill)); 911 } 912 } 913 914 static inline int alias_74k_erratum(struct cpuinfo_mips *c) 915 { 916 unsigned int imp = c->processor_id & PRID_IMP_MASK; 917 unsigned int rev = c->processor_id & PRID_REV_MASK; 918 int present = 0; 919 920 /* 921 * Early versions of the 74K do not update the cache tags on a 922 * vtag miss/ptag hit which can occur in the case of KSEG0/KUSEG 923 * aliases. In this case it is better to treat the cache as always 924 * having aliases. Also disable the synonym tag update feature 925 * where available. In this case no opportunistic tag update will 926 * happen where a load causes a virtual address miss but a physical 927 * address hit during a D-cache look-up. 928 */ 929 switch (imp) { 930 case PRID_IMP_74K: 931 if (rev <= PRID_REV_ENCODE_332(2, 4, 0)) 932 present = 1; 933 if (rev == PRID_REV_ENCODE_332(2, 4, 0)) 934 write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND); 935 break; 936 case PRID_IMP_1074K: 937 if (rev <= PRID_REV_ENCODE_332(1, 1, 0)) { 938 present = 1; 939 write_c0_config6(read_c0_config6() | MIPS_CONF6_SYND); 940 } 941 break; 942 default: 943 BUG(); 944 } 945 946 return present; 947 } 948 949 static void b5k_instruction_hazard(void) 950 { 951 __sync(); 952 __sync(); 953 __asm__ __volatile__( 954 " nop; nop; nop; nop; nop; nop; nop; nop\n" 955 " nop; nop; nop; nop; nop; nop; nop; nop\n" 956 " nop; nop; nop; nop; nop; nop; nop; nop\n" 957 " nop; nop; nop; nop; nop; nop; nop; nop\n" 958 : : : "memory"); 959 } 960 961 static char *way_string[] = { NULL, "direct mapped", "2-way", 962 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way", 963 "9-way", "10-way", "11-way", "12-way", 964 "13-way", "14-way", "15-way", "16-way", 965 }; 966 967 static void probe_pcache(void) 968 { 969 struct cpuinfo_mips *c = ¤t_cpu_data; 970 unsigned int config = read_c0_config(); 971 unsigned int prid = read_c0_prid(); 972 int has_74k_erratum = 0; 973 unsigned long config1; 974 unsigned int lsize; 975 976 switch (current_cpu_type()) { 977 case CPU_R4600: /* QED style two way caches? */ 978 case CPU_R4700: 979 case CPU_R5000: 980 case CPU_NEVADA: 981 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 982 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 983 c->icache.ways = 2; 984 c->icache.waybit = __ffs(icache_size/2); 985 986 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 987 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 988 c->dcache.ways = 2; 989 c->dcache.waybit= __ffs(dcache_size/2); 990 991 c->options |= MIPS_CPU_CACHE_CDEX_P; 992 break; 993 994 case CPU_R5432: 995 case CPU_R5500: 996 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 997 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 998 c->icache.ways = 2; 999 c->icache.waybit= 0; 1000 1001 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 1002 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 1003 c->dcache.ways = 2; 1004 c->dcache.waybit = 0; 1005 1006 c->options |= MIPS_CPU_CACHE_CDEX_P | MIPS_CPU_PREFETCH; 1007 break; 1008 1009 case CPU_TX49XX: 1010 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 1011 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 1012 c->icache.ways = 4; 1013 c->icache.waybit= 0; 1014 1015 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 1016 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 1017 c->dcache.ways = 4; 1018 c->dcache.waybit = 0; 1019 1020 c->options |= MIPS_CPU_CACHE_CDEX_P; 1021 c->options |= MIPS_CPU_PREFETCH; 1022 break; 1023 1024 case CPU_R4000PC: 1025 case CPU_R4000SC: 1026 case CPU_R4000MC: 1027 case CPU_R4400PC: 1028 case CPU_R4400SC: 1029 case CPU_R4400MC: 1030 case CPU_R4300: 1031 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 1032 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 1033 c->icache.ways = 1; 1034 c->icache.waybit = 0; /* doesn't matter */ 1035 1036 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 1037 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 1038 c->dcache.ways = 1; 1039 c->dcache.waybit = 0; /* does not matter */ 1040 1041 c->options |= MIPS_CPU_CACHE_CDEX_P; 1042 break; 1043 1044 case CPU_R10000: 1045 case CPU_R12000: 1046 case CPU_R14000: 1047 case CPU_R16000: 1048 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29)); 1049 c->icache.linesz = 64; 1050 c->icache.ways = 2; 1051 c->icache.waybit = 0; 1052 1053 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26)); 1054 c->dcache.linesz = 32; 1055 c->dcache.ways = 2; 1056 c->dcache.waybit = 0; 1057 1058 c->options |= MIPS_CPU_PREFETCH; 1059 break; 1060 1061 case CPU_VR4133: 1062 write_c0_config(config & ~VR41_CONF_P4K); 1063 case CPU_VR4131: 1064 /* Workaround for cache instruction bug of VR4131 */ 1065 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U || 1066 c->processor_id == 0x0c82U) { 1067 config |= 0x00400000U; 1068 if (c->processor_id == 0x0c80U) 1069 config |= VR41_CONF_BP; 1070 write_c0_config(config); 1071 } else 1072 c->options |= MIPS_CPU_CACHE_CDEX_P; 1073 1074 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 1075 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 1076 c->icache.ways = 2; 1077 c->icache.waybit = __ffs(icache_size/2); 1078 1079 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 1080 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 1081 c->dcache.ways = 2; 1082 c->dcache.waybit = __ffs(dcache_size/2); 1083 break; 1084 1085 case CPU_VR41XX: 1086 case CPU_VR4111: 1087 case CPU_VR4121: 1088 case CPU_VR4122: 1089 case CPU_VR4181: 1090 case CPU_VR4181A: 1091 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 1092 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 1093 c->icache.ways = 1; 1094 c->icache.waybit = 0; /* doesn't matter */ 1095 1096 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 1097 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 1098 c->dcache.ways = 1; 1099 c->dcache.waybit = 0; /* does not matter */ 1100 1101 c->options |= MIPS_CPU_CACHE_CDEX_P; 1102 break; 1103 1104 case CPU_RM7000: 1105 rm7k_erratum31(); 1106 1107 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 1108 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 1109 c->icache.ways = 4; 1110 c->icache.waybit = __ffs(icache_size / c->icache.ways); 1111 1112 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 1113 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 1114 c->dcache.ways = 4; 1115 c->dcache.waybit = __ffs(dcache_size / c->dcache.ways); 1116 1117 c->options |= MIPS_CPU_CACHE_CDEX_P; 1118 c->options |= MIPS_CPU_PREFETCH; 1119 break; 1120 1121 case CPU_LOONGSON2: 1122 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 1123 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 1124 if (prid & 0x3) 1125 c->icache.ways = 4; 1126 else 1127 c->icache.ways = 2; 1128 c->icache.waybit = 0; 1129 1130 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 1131 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 1132 if (prid & 0x3) 1133 c->dcache.ways = 4; 1134 else 1135 c->dcache.ways = 2; 1136 c->dcache.waybit = 0; 1137 break; 1138 1139 case CPU_LOONGSON3: 1140 config1 = read_c0_config1(); 1141 lsize = (config1 >> 19) & 7; 1142 if (lsize) 1143 c->icache.linesz = 2 << lsize; 1144 else 1145 c->icache.linesz = 0; 1146 c->icache.sets = 64 << ((config1 >> 22) & 7); 1147 c->icache.ways = 1 + ((config1 >> 16) & 7); 1148 icache_size = c->icache.sets * 1149 c->icache.ways * 1150 c->icache.linesz; 1151 c->icache.waybit = 0; 1152 1153 lsize = (config1 >> 10) & 7; 1154 if (lsize) 1155 c->dcache.linesz = 2 << lsize; 1156 else 1157 c->dcache.linesz = 0; 1158 c->dcache.sets = 64 << ((config1 >> 13) & 7); 1159 c->dcache.ways = 1 + ((config1 >> 7) & 7); 1160 dcache_size = c->dcache.sets * 1161 c->dcache.ways * 1162 c->dcache.linesz; 1163 c->dcache.waybit = 0; 1164 if ((prid & PRID_REV_MASK) >= PRID_REV_LOONGSON3A_R2) 1165 c->options |= MIPS_CPU_PREFETCH; 1166 break; 1167 1168 case CPU_CAVIUM_OCTEON3: 1169 /* For now lie about the number of ways. */ 1170 c->icache.linesz = 128; 1171 c->icache.sets = 16; 1172 c->icache.ways = 8; 1173 c->icache.flags |= MIPS_CACHE_VTAG; 1174 icache_size = c->icache.sets * c->icache.ways * c->icache.linesz; 1175 1176 c->dcache.linesz = 128; 1177 c->dcache.ways = 8; 1178 c->dcache.sets = 8; 1179 dcache_size = c->dcache.sets * c->dcache.ways * c->dcache.linesz; 1180 c->options |= MIPS_CPU_PREFETCH; 1181 break; 1182 1183 default: 1184 if (!(config & MIPS_CONF_M)) 1185 panic("Don't know how to probe P-caches on this cpu."); 1186 1187 /* 1188 * So we seem to be a MIPS32 or MIPS64 CPU 1189 * So let's probe the I-cache ... 1190 */ 1191 config1 = read_c0_config1(); 1192 1193 lsize = (config1 >> 19) & 7; 1194 1195 /* IL == 7 is reserved */ 1196 if (lsize == 7) 1197 panic("Invalid icache line size"); 1198 1199 c->icache.linesz = lsize ? 2 << lsize : 0; 1200 1201 c->icache.sets = 32 << (((config1 >> 22) + 1) & 7); 1202 c->icache.ways = 1 + ((config1 >> 16) & 7); 1203 1204 icache_size = c->icache.sets * 1205 c->icache.ways * 1206 c->icache.linesz; 1207 c->icache.waybit = __ffs(icache_size/c->icache.ways); 1208 1209 if (config & 0x8) /* VI bit */ 1210 c->icache.flags |= MIPS_CACHE_VTAG; 1211 1212 /* 1213 * Now probe the MIPS32 / MIPS64 data cache. 1214 */ 1215 c->dcache.flags = 0; 1216 1217 lsize = (config1 >> 10) & 7; 1218 1219 /* DL == 7 is reserved */ 1220 if (lsize == 7) 1221 panic("Invalid dcache line size"); 1222 1223 c->dcache.linesz = lsize ? 2 << lsize : 0; 1224 1225 c->dcache.sets = 32 << (((config1 >> 13) + 1) & 7); 1226 c->dcache.ways = 1 + ((config1 >> 7) & 7); 1227 1228 dcache_size = c->dcache.sets * 1229 c->dcache.ways * 1230 c->dcache.linesz; 1231 c->dcache.waybit = __ffs(dcache_size/c->dcache.ways); 1232 1233 c->options |= MIPS_CPU_PREFETCH; 1234 break; 1235 } 1236 1237 /* 1238 * Processor configuration sanity check for the R4000SC erratum 1239 * #5. With page sizes larger than 32kB there is no possibility 1240 * to get a VCE exception anymore so we don't care about this 1241 * misconfiguration. The case is rather theoretical anyway; 1242 * presumably no vendor is shipping his hardware in the "bad" 1243 * configuration. 1244 */ 1245 if ((prid & PRID_IMP_MASK) == PRID_IMP_R4000 && 1246 (prid & PRID_REV_MASK) < PRID_REV_R4400 && 1247 !(config & CONF_SC) && c->icache.linesz != 16 && 1248 PAGE_SIZE <= 0x8000) 1249 panic("Improper R4000SC processor configuration detected"); 1250 1251 /* compute a couple of other cache variables */ 1252 c->icache.waysize = icache_size / c->icache.ways; 1253 c->dcache.waysize = dcache_size / c->dcache.ways; 1254 1255 c->icache.sets = c->icache.linesz ? 1256 icache_size / (c->icache.linesz * c->icache.ways) : 0; 1257 c->dcache.sets = c->dcache.linesz ? 1258 dcache_size / (c->dcache.linesz * c->dcache.ways) : 0; 1259 1260 /* 1261 * R1x000 P-caches are odd in a positive way. They're 32kB 2-way 1262 * virtually indexed so normally would suffer from aliases. So 1263 * normally they'd suffer from aliases but magic in the hardware deals 1264 * with that for us so we don't need to take care ourselves. 1265 */ 1266 switch (current_cpu_type()) { 1267 case CPU_20KC: 1268 case CPU_25KF: 1269 case CPU_SB1: 1270 case CPU_SB1A: 1271 case CPU_XLR: 1272 c->dcache.flags |= MIPS_CACHE_PINDEX; 1273 break; 1274 1275 case CPU_R10000: 1276 case CPU_R12000: 1277 case CPU_R14000: 1278 case CPU_R16000: 1279 break; 1280 1281 case CPU_74K: 1282 case CPU_1074K: 1283 has_74k_erratum = alias_74k_erratum(c); 1284 /* Fall through. */ 1285 case CPU_M14KC: 1286 case CPU_M14KEC: 1287 case CPU_24K: 1288 case CPU_34K: 1289 case CPU_1004K: 1290 case CPU_INTERAPTIV: 1291 case CPU_P5600: 1292 case CPU_PROAPTIV: 1293 case CPU_M5150: 1294 case CPU_QEMU_GENERIC: 1295 case CPU_I6400: 1296 case CPU_P6600: 1297 case CPU_M6250: 1298 if (!(read_c0_config7() & MIPS_CONF7_IAR) && 1299 (c->icache.waysize > PAGE_SIZE)) 1300 c->icache.flags |= MIPS_CACHE_ALIASES; 1301 if (!has_74k_erratum && (read_c0_config7() & MIPS_CONF7_AR)) { 1302 /* 1303 * Effectively physically indexed dcache, 1304 * thus no virtual aliases. 1305 */ 1306 c->dcache.flags |= MIPS_CACHE_PINDEX; 1307 break; 1308 } 1309 default: 1310 if (has_74k_erratum || c->dcache.waysize > PAGE_SIZE) 1311 c->dcache.flags |= MIPS_CACHE_ALIASES; 1312 } 1313 1314 switch (current_cpu_type()) { 1315 case CPU_20KC: 1316 /* 1317 * Some older 20Kc chips doesn't have the 'VI' bit in 1318 * the config register. 1319 */ 1320 c->icache.flags |= MIPS_CACHE_VTAG; 1321 break; 1322 1323 case CPU_ALCHEMY: 1324 case CPU_I6400: 1325 c->icache.flags |= MIPS_CACHE_IC_F_DC; 1326 break; 1327 1328 case CPU_BMIPS5000: 1329 c->icache.flags |= MIPS_CACHE_IC_F_DC; 1330 /* Cache aliases are handled in hardware; allow HIGHMEM */ 1331 c->dcache.flags &= ~MIPS_CACHE_ALIASES; 1332 break; 1333 1334 case CPU_LOONGSON2: 1335 /* 1336 * LOONGSON2 has 4 way icache, but when using indexed cache op, 1337 * one op will act on all 4 ways 1338 */ 1339 c->icache.ways = 1; 1340 } 1341 1342 printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n", 1343 icache_size >> 10, 1344 c->icache.flags & MIPS_CACHE_VTAG ? "VIVT" : "VIPT", 1345 way_string[c->icache.ways], c->icache.linesz); 1346 1347 printk("Primary data cache %ldkB, %s, %s, %s, linesize %d bytes\n", 1348 dcache_size >> 10, way_string[c->dcache.ways], 1349 (c->dcache.flags & MIPS_CACHE_PINDEX) ? "PIPT" : "VIPT", 1350 (c->dcache.flags & MIPS_CACHE_ALIASES) ? 1351 "cache aliases" : "no aliases", 1352 c->dcache.linesz); 1353 } 1354 1355 static void probe_vcache(void) 1356 { 1357 struct cpuinfo_mips *c = ¤t_cpu_data; 1358 unsigned int config2, lsize; 1359 1360 if (current_cpu_type() != CPU_LOONGSON3) 1361 return; 1362 1363 config2 = read_c0_config2(); 1364 if ((lsize = ((config2 >> 20) & 15))) 1365 c->vcache.linesz = 2 << lsize; 1366 else 1367 c->vcache.linesz = lsize; 1368 1369 c->vcache.sets = 64 << ((config2 >> 24) & 15); 1370 c->vcache.ways = 1 + ((config2 >> 16) & 15); 1371 1372 vcache_size = c->vcache.sets * c->vcache.ways * c->vcache.linesz; 1373 1374 c->vcache.waybit = 0; 1375 1376 pr_info("Unified victim cache %ldkB %s, linesize %d bytes.\n", 1377 vcache_size >> 10, way_string[c->vcache.ways], c->vcache.linesz); 1378 } 1379 1380 /* 1381 * If you even _breathe_ on this function, look at the gcc output and make sure 1382 * it does not pop things on and off the stack for the cache sizing loop that 1383 * executes in KSEG1 space or else you will crash and burn badly. You have 1384 * been warned. 1385 */ 1386 static int probe_scache(void) 1387 { 1388 unsigned long flags, addr, begin, end, pow2; 1389 unsigned int config = read_c0_config(); 1390 struct cpuinfo_mips *c = ¤t_cpu_data; 1391 1392 if (config & CONF_SC) 1393 return 0; 1394 1395 begin = (unsigned long) &_stext; 1396 begin &= ~((4 * 1024 * 1024) - 1); 1397 end = begin + (4 * 1024 * 1024); 1398 1399 /* 1400 * This is such a bitch, you'd think they would make it easy to do 1401 * this. Away you daemons of stupidity! 1402 */ 1403 local_irq_save(flags); 1404 1405 /* Fill each size-multiple cache line with a valid tag. */ 1406 pow2 = (64 * 1024); 1407 for (addr = begin; addr < end; addr = (begin + pow2)) { 1408 unsigned long *p = (unsigned long *) addr; 1409 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */ 1410 pow2 <<= 1; 1411 } 1412 1413 /* Load first line with zero (therefore invalid) tag. */ 1414 write_c0_taglo(0); 1415 write_c0_taghi(0); 1416 __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */ 1417 cache_op(Index_Store_Tag_I, begin); 1418 cache_op(Index_Store_Tag_D, begin); 1419 cache_op(Index_Store_Tag_SD, begin); 1420 1421 /* Now search for the wrap around point. */ 1422 pow2 = (128 * 1024); 1423 for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) { 1424 cache_op(Index_Load_Tag_SD, addr); 1425 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */ 1426 if (!read_c0_taglo()) 1427 break; 1428 pow2 <<= 1; 1429 } 1430 local_irq_restore(flags); 1431 addr -= begin; 1432 1433 scache_size = addr; 1434 c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22); 1435 c->scache.ways = 1; 1436 c->scache.waybit = 0; /* does not matter */ 1437 1438 return 1; 1439 } 1440 1441 static void __init loongson2_sc_init(void) 1442 { 1443 struct cpuinfo_mips *c = ¤t_cpu_data; 1444 1445 scache_size = 512*1024; 1446 c->scache.linesz = 32; 1447 c->scache.ways = 4; 1448 c->scache.waybit = 0; 1449 c->scache.waysize = scache_size / (c->scache.ways); 1450 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways); 1451 pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n", 1452 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); 1453 1454 c->options |= MIPS_CPU_INCLUSIVE_CACHES; 1455 } 1456 1457 static void __init loongson3_sc_init(void) 1458 { 1459 struct cpuinfo_mips *c = ¤t_cpu_data; 1460 unsigned int config2, lsize; 1461 1462 config2 = read_c0_config2(); 1463 lsize = (config2 >> 4) & 15; 1464 if (lsize) 1465 c->scache.linesz = 2 << lsize; 1466 else 1467 c->scache.linesz = 0; 1468 c->scache.sets = 64 << ((config2 >> 8) & 15); 1469 c->scache.ways = 1 + (config2 & 15); 1470 1471 scache_size = c->scache.sets * 1472 c->scache.ways * 1473 c->scache.linesz; 1474 /* Loongson-3 has 4 cores, 1MB scache for each. scaches are shared */ 1475 scache_size *= 4; 1476 c->scache.waybit = 0; 1477 pr_info("Unified secondary cache %ldkB %s, linesize %d bytes.\n", 1478 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); 1479 if (scache_size) 1480 c->options |= MIPS_CPU_INCLUSIVE_CACHES; 1481 return; 1482 } 1483 1484 extern int r5k_sc_init(void); 1485 extern int rm7k_sc_init(void); 1486 extern int mips_sc_init(void); 1487 1488 static void setup_scache(void) 1489 { 1490 struct cpuinfo_mips *c = ¤t_cpu_data; 1491 unsigned int config = read_c0_config(); 1492 int sc_present = 0; 1493 1494 /* 1495 * Do the probing thing on R4000SC and R4400SC processors. Other 1496 * processors don't have a S-cache that would be relevant to the 1497 * Linux memory management. 1498 */ 1499 switch (current_cpu_type()) { 1500 case CPU_R4000SC: 1501 case CPU_R4000MC: 1502 case CPU_R4400SC: 1503 case CPU_R4400MC: 1504 sc_present = run_uncached(probe_scache); 1505 if (sc_present) 1506 c->options |= MIPS_CPU_CACHE_CDEX_S; 1507 break; 1508 1509 case CPU_R10000: 1510 case CPU_R12000: 1511 case CPU_R14000: 1512 case CPU_R16000: 1513 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16); 1514 c->scache.linesz = 64 << ((config >> 13) & 1); 1515 c->scache.ways = 2; 1516 c->scache.waybit= 0; 1517 sc_present = 1; 1518 break; 1519 1520 case CPU_R5000: 1521 case CPU_NEVADA: 1522 #ifdef CONFIG_R5000_CPU_SCACHE 1523 r5k_sc_init(); 1524 #endif 1525 return; 1526 1527 case CPU_RM7000: 1528 #ifdef CONFIG_RM7000_CPU_SCACHE 1529 rm7k_sc_init(); 1530 #endif 1531 return; 1532 1533 case CPU_LOONGSON2: 1534 loongson2_sc_init(); 1535 return; 1536 1537 case CPU_LOONGSON3: 1538 loongson3_sc_init(); 1539 return; 1540 1541 case CPU_CAVIUM_OCTEON3: 1542 case CPU_XLP: 1543 /* don't need to worry about L2, fully coherent */ 1544 return; 1545 1546 default: 1547 if (c->isa_level & (MIPS_CPU_ISA_M32R1 | MIPS_CPU_ISA_M32R2 | 1548 MIPS_CPU_ISA_M32R6 | MIPS_CPU_ISA_M64R1 | 1549 MIPS_CPU_ISA_M64R2 | MIPS_CPU_ISA_M64R6)) { 1550 #ifdef CONFIG_MIPS_CPU_SCACHE 1551 if (mips_sc_init ()) { 1552 scache_size = c->scache.ways * c->scache.sets * c->scache.linesz; 1553 printk("MIPS secondary cache %ldkB, %s, linesize %d bytes.\n", 1554 scache_size >> 10, 1555 way_string[c->scache.ways], c->scache.linesz); 1556 } 1557 #else 1558 if (!(c->scache.flags & MIPS_CACHE_NOT_PRESENT)) 1559 panic("Dunno how to handle MIPS32 / MIPS64 second level cache"); 1560 #endif 1561 return; 1562 } 1563 sc_present = 0; 1564 } 1565 1566 if (!sc_present) 1567 return; 1568 1569 /* compute a couple of other cache variables */ 1570 c->scache.waysize = scache_size / c->scache.ways; 1571 1572 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways); 1573 1574 printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n", 1575 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); 1576 1577 c->options |= MIPS_CPU_INCLUSIVE_CACHES; 1578 } 1579 1580 void au1x00_fixup_config_od(void) 1581 { 1582 /* 1583 * c0_config.od (bit 19) was write only (and read as 0) 1584 * on the early revisions of Alchemy SOCs. It disables the bus 1585 * transaction overlapping and needs to be set to fix various errata. 1586 */ 1587 switch (read_c0_prid()) { 1588 case 0x00030100: /* Au1000 DA */ 1589 case 0x00030201: /* Au1000 HA */ 1590 case 0x00030202: /* Au1000 HB */ 1591 case 0x01030200: /* Au1500 AB */ 1592 /* 1593 * Au1100 errata actually keeps silence about this bit, so we set it 1594 * just in case for those revisions that require it to be set according 1595 * to the (now gone) cpu table. 1596 */ 1597 case 0x02030200: /* Au1100 AB */ 1598 case 0x02030201: /* Au1100 BA */ 1599 case 0x02030202: /* Au1100 BC */ 1600 set_c0_config(1 << 19); 1601 break; 1602 } 1603 } 1604 1605 /* CP0 hazard avoidance. */ 1606 #define NXP_BARRIER() \ 1607 __asm__ __volatile__( \ 1608 ".set noreorder\n\t" \ 1609 "nop; nop; nop; nop; nop; nop;\n\t" \ 1610 ".set reorder\n\t") 1611 1612 static void nxp_pr4450_fixup_config(void) 1613 { 1614 unsigned long config0; 1615 1616 config0 = read_c0_config(); 1617 1618 /* clear all three cache coherency fields */ 1619 config0 &= ~(0x7 | (7 << 25) | (7 << 28)); 1620 config0 |= (((_page_cachable_default >> _CACHE_SHIFT) << 0) | 1621 ((_page_cachable_default >> _CACHE_SHIFT) << 25) | 1622 ((_page_cachable_default >> _CACHE_SHIFT) << 28)); 1623 write_c0_config(config0); 1624 NXP_BARRIER(); 1625 } 1626 1627 static int cca = -1; 1628 1629 static int __init cca_setup(char *str) 1630 { 1631 get_option(&str, &cca); 1632 1633 return 0; 1634 } 1635 1636 early_param("cca", cca_setup); 1637 1638 static void coherency_setup(void) 1639 { 1640 if (cca < 0 || cca > 7) 1641 cca = read_c0_config() & CONF_CM_CMASK; 1642 _page_cachable_default = cca << _CACHE_SHIFT; 1643 1644 pr_debug("Using cache attribute %d\n", cca); 1645 change_c0_config(CONF_CM_CMASK, cca); 1646 1647 /* 1648 * c0_status.cu=0 specifies that updates by the sc instruction use 1649 * the coherency mode specified by the TLB; 1 means cachable 1650 * coherent update on write will be used. Not all processors have 1651 * this bit and; some wire it to zero, others like Toshiba had the 1652 * silly idea of putting something else there ... 1653 */ 1654 switch (current_cpu_type()) { 1655 case CPU_R4000PC: 1656 case CPU_R4000SC: 1657 case CPU_R4000MC: 1658 case CPU_R4400PC: 1659 case CPU_R4400SC: 1660 case CPU_R4400MC: 1661 clear_c0_config(CONF_CU); 1662 break; 1663 /* 1664 * We need to catch the early Alchemy SOCs with 1665 * the write-only co_config.od bit and set it back to one on: 1666 * Au1000 rev DA, HA, HB; Au1100 AB, BA, BC, Au1500 AB 1667 */ 1668 case CPU_ALCHEMY: 1669 au1x00_fixup_config_od(); 1670 break; 1671 1672 case PRID_IMP_PR4450: 1673 nxp_pr4450_fixup_config(); 1674 break; 1675 } 1676 } 1677 1678 static void r4k_cache_error_setup(void) 1679 { 1680 extern char __weak except_vec2_generic; 1681 extern char __weak except_vec2_sb1; 1682 1683 switch (current_cpu_type()) { 1684 case CPU_SB1: 1685 case CPU_SB1A: 1686 set_uncached_handler(0x100, &except_vec2_sb1, 0x80); 1687 break; 1688 1689 default: 1690 set_uncached_handler(0x100, &except_vec2_generic, 0x80); 1691 break; 1692 } 1693 } 1694 1695 void r4k_cache_init(void) 1696 { 1697 extern void build_clear_page(void); 1698 extern void build_copy_page(void); 1699 struct cpuinfo_mips *c = ¤t_cpu_data; 1700 1701 probe_pcache(); 1702 probe_vcache(); 1703 setup_scache(); 1704 1705 r4k_blast_dcache_page_setup(); 1706 r4k_blast_dcache_page_indexed_setup(); 1707 r4k_blast_dcache_setup(); 1708 r4k_blast_icache_page_setup(); 1709 r4k_blast_icache_page_indexed_setup(); 1710 r4k_blast_icache_setup(); 1711 r4k_blast_scache_page_setup(); 1712 r4k_blast_scache_page_indexed_setup(); 1713 r4k_blast_scache_setup(); 1714 #ifdef CONFIG_EVA 1715 r4k_blast_dcache_user_page_setup(); 1716 r4k_blast_icache_user_page_setup(); 1717 #endif 1718 1719 /* 1720 * Some MIPS32 and MIPS64 processors have physically indexed caches. 1721 * This code supports virtually indexed processors and will be 1722 * unnecessarily inefficient on physically indexed processors. 1723 */ 1724 if (c->dcache.linesz && cpu_has_dc_aliases) 1725 shm_align_mask = max_t( unsigned long, 1726 c->dcache.sets * c->dcache.linesz - 1, 1727 PAGE_SIZE - 1); 1728 else 1729 shm_align_mask = PAGE_SIZE-1; 1730 1731 __flush_cache_vmap = r4k__flush_cache_vmap; 1732 __flush_cache_vunmap = r4k__flush_cache_vunmap; 1733 1734 flush_cache_all = cache_noop; 1735 __flush_cache_all = r4k___flush_cache_all; 1736 flush_cache_mm = r4k_flush_cache_mm; 1737 flush_cache_page = r4k_flush_cache_page; 1738 flush_cache_range = r4k_flush_cache_range; 1739 1740 __flush_kernel_vmap_range = r4k_flush_kernel_vmap_range; 1741 1742 flush_cache_sigtramp = r4k_flush_cache_sigtramp; 1743 flush_icache_all = r4k_flush_icache_all; 1744 local_flush_data_cache_page = local_r4k_flush_data_cache_page; 1745 flush_data_cache_page = r4k_flush_data_cache_page; 1746 flush_icache_range = r4k_flush_icache_range; 1747 local_flush_icache_range = local_r4k_flush_icache_range; 1748 1749 #if defined(CONFIG_DMA_NONCOHERENT) || defined(CONFIG_DMA_MAYBE_COHERENT) 1750 if (coherentio) { 1751 _dma_cache_wback_inv = (void *)cache_noop; 1752 _dma_cache_wback = (void *)cache_noop; 1753 _dma_cache_inv = (void *)cache_noop; 1754 } else { 1755 _dma_cache_wback_inv = r4k_dma_cache_wback_inv; 1756 _dma_cache_wback = r4k_dma_cache_wback_inv; 1757 _dma_cache_inv = r4k_dma_cache_inv; 1758 } 1759 #endif 1760 1761 build_clear_page(); 1762 build_copy_page(); 1763 1764 /* 1765 * We want to run CMP kernels on core with and without coherent 1766 * caches. Therefore, do not use CONFIG_MIPS_CMP to decide whether 1767 * or not to flush caches. 1768 */ 1769 local_r4k___flush_cache_all(NULL); 1770 1771 coherency_setup(); 1772 board_cache_error_setup = r4k_cache_error_setup; 1773 1774 /* 1775 * Per-CPU overrides 1776 */ 1777 switch (current_cpu_type()) { 1778 case CPU_BMIPS4350: 1779 case CPU_BMIPS4380: 1780 /* No IPI is needed because all CPUs share the same D$ */ 1781 flush_data_cache_page = r4k_blast_dcache_page; 1782 break; 1783 case CPU_BMIPS5000: 1784 /* We lose our superpowers if L2 is disabled */ 1785 if (c->scache.flags & MIPS_CACHE_NOT_PRESENT) 1786 break; 1787 1788 /* I$ fills from D$ just by emptying the write buffers */ 1789 flush_cache_page = (void *)b5k_instruction_hazard; 1790 flush_cache_range = (void *)b5k_instruction_hazard; 1791 flush_cache_sigtramp = (void *)b5k_instruction_hazard; 1792 local_flush_data_cache_page = (void *)b5k_instruction_hazard; 1793 flush_data_cache_page = (void *)b5k_instruction_hazard; 1794 flush_icache_range = (void *)b5k_instruction_hazard; 1795 local_flush_icache_range = (void *)b5k_instruction_hazard; 1796 1797 1798 /* Optimization: an L2 flush implicitly flushes the L1 */ 1799 current_cpu_data.options |= MIPS_CPU_INCLUSIVE_CACHES; 1800 break; 1801 case CPU_LOONGSON3: 1802 /* Loongson-3 maintains cache coherency by hardware */ 1803 __flush_cache_all = cache_noop; 1804 __flush_cache_vmap = cache_noop; 1805 __flush_cache_vunmap = cache_noop; 1806 __flush_kernel_vmap_range = (void *)cache_noop; 1807 flush_cache_mm = (void *)cache_noop; 1808 flush_cache_page = (void *)cache_noop; 1809 flush_cache_range = (void *)cache_noop; 1810 flush_cache_sigtramp = (void *)cache_noop; 1811 flush_icache_all = (void *)cache_noop; 1812 flush_data_cache_page = (void *)cache_noop; 1813 local_flush_data_cache_page = (void *)cache_noop; 1814 break; 1815 } 1816 } 1817 1818 static int r4k_cache_pm_notifier(struct notifier_block *self, unsigned long cmd, 1819 void *v) 1820 { 1821 switch (cmd) { 1822 case CPU_PM_ENTER_FAILED: 1823 case CPU_PM_EXIT: 1824 coherency_setup(); 1825 break; 1826 } 1827 1828 return NOTIFY_OK; 1829 } 1830 1831 static struct notifier_block r4k_cache_pm_notifier_block = { 1832 .notifier_call = r4k_cache_pm_notifier, 1833 }; 1834 1835 int __init r4k_cache_init_pm(void) 1836 { 1837 return cpu_pm_register_notifier(&r4k_cache_pm_notifier_block); 1838 } 1839 arch_initcall(r4k_cache_init_pm); 1840