1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) 7 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org) 8 * Copyright (C) 1999, 2000 Silicon Graphics, Inc. 9 */ 10 #include <linux/config.h> 11 #include <linux/init.h> 12 #include <linux/kernel.h> 13 #include <linux/sched.h> 14 #include <linux/mm.h> 15 #include <linux/bitops.h> 16 17 #include <asm/bcache.h> 18 #include <asm/bootinfo.h> 19 #include <asm/cache.h> 20 #include <asm/cacheops.h> 21 #include <asm/cpu.h> 22 #include <asm/cpu-features.h> 23 #include <asm/io.h> 24 #include <asm/page.h> 25 #include <asm/pgtable.h> 26 #include <asm/r4kcache.h> 27 #include <asm/system.h> 28 #include <asm/mmu_context.h> 29 #include <asm/war.h> 30 #include <asm/cacheflush.h> /* for run_uncached() */ 31 32 /* 33 * Must die. 34 */ 35 static unsigned long icache_size __read_mostly; 36 static unsigned long dcache_size __read_mostly; 37 static unsigned long scache_size __read_mostly; 38 39 /* 40 * Dummy cache handling routines for machines without boardcaches 41 */ 42 static void no_sc_noop(void) {} 43 44 static struct bcache_ops no_sc_ops = { 45 .bc_enable = (void *)no_sc_noop, 46 .bc_disable = (void *)no_sc_noop, 47 .bc_wback_inv = (void *)no_sc_noop, 48 .bc_inv = (void *)no_sc_noop 49 }; 50 51 struct bcache_ops *bcops = &no_sc_ops; 52 53 #define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010) 54 #define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020) 55 56 #define R4600_HIT_CACHEOP_WAR_IMPL \ 57 do { \ 58 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) \ 59 *(volatile unsigned long *)CKSEG1; \ 60 if (R4600_V1_HIT_CACHEOP_WAR) \ 61 __asm__ __volatile__("nop;nop;nop;nop"); \ 62 } while (0) 63 64 static void (*r4k_blast_dcache_page)(unsigned long addr); 65 66 static inline void r4k_blast_dcache_page_dc32(unsigned long addr) 67 { 68 R4600_HIT_CACHEOP_WAR_IMPL; 69 blast_dcache32_page(addr); 70 } 71 72 static inline void r4k_blast_dcache_page_setup(void) 73 { 74 unsigned long dc_lsize = cpu_dcache_line_size(); 75 76 if (dc_lsize == 16) 77 r4k_blast_dcache_page = blast_dcache16_page; 78 else if (dc_lsize == 32) 79 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32; 80 } 81 82 static void (* r4k_blast_dcache_page_indexed)(unsigned long addr); 83 84 static inline void r4k_blast_dcache_page_indexed_setup(void) 85 { 86 unsigned long dc_lsize = cpu_dcache_line_size(); 87 88 if (dc_lsize == 16) 89 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed; 90 else if (dc_lsize == 32) 91 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed; 92 } 93 94 static void (* r4k_blast_dcache)(void); 95 96 static inline void r4k_blast_dcache_setup(void) 97 { 98 unsigned long dc_lsize = cpu_dcache_line_size(); 99 100 if (dc_lsize == 16) 101 r4k_blast_dcache = blast_dcache16; 102 else if (dc_lsize == 32) 103 r4k_blast_dcache = blast_dcache32; 104 } 105 106 /* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */ 107 #define JUMP_TO_ALIGN(order) \ 108 __asm__ __volatile__( \ 109 "b\t1f\n\t" \ 110 ".align\t" #order "\n\t" \ 111 "1:\n\t" \ 112 ) 113 #define CACHE32_UNROLL32_ALIGN JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */ 114 #define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11) 115 116 static inline void blast_r4600_v1_icache32(void) 117 { 118 unsigned long flags; 119 120 local_irq_save(flags); 121 blast_icache32(); 122 local_irq_restore(flags); 123 } 124 125 static inline void tx49_blast_icache32(void) 126 { 127 unsigned long start = INDEX_BASE; 128 unsigned long end = start + current_cpu_data.icache.waysize; 129 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 130 unsigned long ws_end = current_cpu_data.icache.ways << 131 current_cpu_data.icache.waybit; 132 unsigned long ws, addr; 133 134 CACHE32_UNROLL32_ALIGN2; 135 /* I'm in even chunk. blast odd chunks */ 136 for (ws = 0; ws < ws_end; ws += ws_inc) 137 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 138 cache32_unroll32(addr|ws,Index_Invalidate_I); 139 CACHE32_UNROLL32_ALIGN; 140 /* I'm in odd chunk. blast even chunks */ 141 for (ws = 0; ws < ws_end; ws += ws_inc) 142 for (addr = start; addr < end; addr += 0x400 * 2) 143 cache32_unroll32(addr|ws,Index_Invalidate_I); 144 } 145 146 static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page) 147 { 148 unsigned long flags; 149 150 local_irq_save(flags); 151 blast_icache32_page_indexed(page); 152 local_irq_restore(flags); 153 } 154 155 static inline void tx49_blast_icache32_page_indexed(unsigned long page) 156 { 157 unsigned long start = page; 158 unsigned long end = start + PAGE_SIZE; 159 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 160 unsigned long ws_end = current_cpu_data.icache.ways << 161 current_cpu_data.icache.waybit; 162 unsigned long ws, addr; 163 164 CACHE32_UNROLL32_ALIGN2; 165 /* I'm in even chunk. blast odd chunks */ 166 for (ws = 0; ws < ws_end; ws += ws_inc) 167 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 168 cache32_unroll32(addr|ws,Index_Invalidate_I); 169 CACHE32_UNROLL32_ALIGN; 170 /* I'm in odd chunk. blast even chunks */ 171 for (ws = 0; ws < ws_end; ws += ws_inc) 172 for (addr = start; addr < end; addr += 0x400 * 2) 173 cache32_unroll32(addr|ws,Index_Invalidate_I); 174 } 175 176 static void (* r4k_blast_icache_page)(unsigned long addr); 177 178 static inline void r4k_blast_icache_page_setup(void) 179 { 180 unsigned long ic_lsize = cpu_icache_line_size(); 181 182 if (ic_lsize == 16) 183 r4k_blast_icache_page = blast_icache16_page; 184 else if (ic_lsize == 32) 185 r4k_blast_icache_page = blast_icache32_page; 186 else if (ic_lsize == 64) 187 r4k_blast_icache_page = blast_icache64_page; 188 } 189 190 191 static void (* r4k_blast_icache_page_indexed)(unsigned long addr); 192 193 static inline void r4k_blast_icache_page_indexed_setup(void) 194 { 195 unsigned long ic_lsize = cpu_icache_line_size(); 196 197 if (ic_lsize == 16) 198 r4k_blast_icache_page_indexed = blast_icache16_page_indexed; 199 else if (ic_lsize == 32) { 200 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 201 r4k_blast_icache_page_indexed = 202 blast_icache32_r4600_v1_page_indexed; 203 else if (TX49XX_ICACHE_INDEX_INV_WAR) 204 r4k_blast_icache_page_indexed = 205 tx49_blast_icache32_page_indexed; 206 else 207 r4k_blast_icache_page_indexed = 208 blast_icache32_page_indexed; 209 } else if (ic_lsize == 64) 210 r4k_blast_icache_page_indexed = blast_icache64_page_indexed; 211 } 212 213 static void (* r4k_blast_icache)(void); 214 215 static inline void r4k_blast_icache_setup(void) 216 { 217 unsigned long ic_lsize = cpu_icache_line_size(); 218 219 if (ic_lsize == 16) 220 r4k_blast_icache = blast_icache16; 221 else if (ic_lsize == 32) { 222 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 223 r4k_blast_icache = blast_r4600_v1_icache32; 224 else if (TX49XX_ICACHE_INDEX_INV_WAR) 225 r4k_blast_icache = tx49_blast_icache32; 226 else 227 r4k_blast_icache = blast_icache32; 228 } else if (ic_lsize == 64) 229 r4k_blast_icache = blast_icache64; 230 } 231 232 static void (* r4k_blast_scache_page)(unsigned long addr); 233 234 static inline void r4k_blast_scache_page_setup(void) 235 { 236 unsigned long sc_lsize = cpu_scache_line_size(); 237 238 if (scache_size == 0) 239 r4k_blast_scache_page = (void *)no_sc_noop; 240 else if (sc_lsize == 16) 241 r4k_blast_scache_page = blast_scache16_page; 242 else if (sc_lsize == 32) 243 r4k_blast_scache_page = blast_scache32_page; 244 else if (sc_lsize == 64) 245 r4k_blast_scache_page = blast_scache64_page; 246 else if (sc_lsize == 128) 247 r4k_blast_scache_page = blast_scache128_page; 248 } 249 250 static void (* r4k_blast_scache_page_indexed)(unsigned long addr); 251 252 static inline void r4k_blast_scache_page_indexed_setup(void) 253 { 254 unsigned long sc_lsize = cpu_scache_line_size(); 255 256 if (scache_size == 0) 257 r4k_blast_scache_page_indexed = (void *)no_sc_noop; 258 else if (sc_lsize == 16) 259 r4k_blast_scache_page_indexed = blast_scache16_page_indexed; 260 else if (sc_lsize == 32) 261 r4k_blast_scache_page_indexed = blast_scache32_page_indexed; 262 else if (sc_lsize == 64) 263 r4k_blast_scache_page_indexed = blast_scache64_page_indexed; 264 else if (sc_lsize == 128) 265 r4k_blast_scache_page_indexed = blast_scache128_page_indexed; 266 } 267 268 static void (* r4k_blast_scache)(void); 269 270 static inline void r4k_blast_scache_setup(void) 271 { 272 unsigned long sc_lsize = cpu_scache_line_size(); 273 274 if (scache_size == 0) 275 r4k_blast_scache = (void *)no_sc_noop; 276 else if (sc_lsize == 16) 277 r4k_blast_scache = blast_scache16; 278 else if (sc_lsize == 32) 279 r4k_blast_scache = blast_scache32; 280 else if (sc_lsize == 64) 281 r4k_blast_scache = blast_scache64; 282 else if (sc_lsize == 128) 283 r4k_blast_scache = blast_scache128; 284 } 285 286 /* 287 * This is former mm's flush_cache_all() which really should be 288 * flush_cache_vunmap these days ... 289 */ 290 static inline void local_r4k_flush_cache_all(void * args) 291 { 292 r4k_blast_dcache(); 293 r4k_blast_icache(); 294 } 295 296 static void r4k_flush_cache_all(void) 297 { 298 if (!cpu_has_dc_aliases) 299 return; 300 301 on_each_cpu(local_r4k_flush_cache_all, NULL, 1, 1); 302 } 303 304 static inline void local_r4k___flush_cache_all(void * args) 305 { 306 r4k_blast_dcache(); 307 r4k_blast_icache(); 308 309 switch (current_cpu_data.cputype) { 310 case CPU_R4000SC: 311 case CPU_R4000MC: 312 case CPU_R4400SC: 313 case CPU_R4400MC: 314 case CPU_R10000: 315 case CPU_R12000: 316 r4k_blast_scache(); 317 } 318 } 319 320 static void r4k___flush_cache_all(void) 321 { 322 on_each_cpu(local_r4k___flush_cache_all, NULL, 1, 1); 323 } 324 325 static inline void local_r4k_flush_cache_range(void * args) 326 { 327 struct vm_area_struct *vma = args; 328 int exec; 329 330 if (!(cpu_context(smp_processor_id(), vma->vm_mm))) 331 return; 332 333 exec = vma->vm_flags & VM_EXEC; 334 if (cpu_has_dc_aliases || exec) 335 r4k_blast_dcache(); 336 if (exec) 337 r4k_blast_icache(); 338 } 339 340 static void r4k_flush_cache_range(struct vm_area_struct *vma, 341 unsigned long start, unsigned long end) 342 { 343 on_each_cpu(local_r4k_flush_cache_range, vma, 1, 1); 344 } 345 346 static inline void local_r4k_flush_cache_mm(void * args) 347 { 348 struct mm_struct *mm = args; 349 350 if (!cpu_context(smp_processor_id(), mm)) 351 return; 352 353 r4k_blast_dcache(); 354 r4k_blast_icache(); 355 356 /* 357 * Kludge alert. For obscure reasons R4000SC and R4400SC go nuts if we 358 * only flush the primary caches but R10000 and R12000 behave sane ... 359 */ 360 if (current_cpu_data.cputype == CPU_R4000SC || 361 current_cpu_data.cputype == CPU_R4000MC || 362 current_cpu_data.cputype == CPU_R4400SC || 363 current_cpu_data.cputype == CPU_R4400MC) 364 r4k_blast_scache(); 365 } 366 367 static void r4k_flush_cache_mm(struct mm_struct *mm) 368 { 369 if (!cpu_has_dc_aliases) 370 return; 371 372 on_each_cpu(local_r4k_flush_cache_mm, mm, 1, 1); 373 } 374 375 struct flush_cache_page_args { 376 struct vm_area_struct *vma; 377 unsigned long addr; 378 unsigned long pfn; 379 }; 380 381 static inline void local_r4k_flush_cache_page(void *args) 382 { 383 struct flush_cache_page_args *fcp_args = args; 384 struct vm_area_struct *vma = fcp_args->vma; 385 unsigned long addr = fcp_args->addr; 386 unsigned long paddr = fcp_args->pfn << PAGE_SHIFT; 387 int exec = vma->vm_flags & VM_EXEC; 388 struct mm_struct *mm = vma->vm_mm; 389 pgd_t *pgdp; 390 pud_t *pudp; 391 pmd_t *pmdp; 392 pte_t *ptep; 393 394 /* 395 * If ownes no valid ASID yet, cannot possibly have gotten 396 * this page into the cache. 397 */ 398 if (cpu_context(smp_processor_id(), mm) == 0) 399 return; 400 401 addr &= PAGE_MASK; 402 pgdp = pgd_offset(mm, addr); 403 pudp = pud_offset(pgdp, addr); 404 pmdp = pmd_offset(pudp, addr); 405 ptep = pte_offset(pmdp, addr); 406 407 /* 408 * If the page isn't marked valid, the page cannot possibly be 409 * in the cache. 410 */ 411 if (!(pte_val(*ptep) & _PAGE_PRESENT)) 412 return; 413 414 /* 415 * Doing flushes for another ASID than the current one is 416 * too difficult since stupid R4k caches do a TLB translation 417 * for every cache flush operation. So we do indexed flushes 418 * in that case, which doesn't overly flush the cache too much. 419 */ 420 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) { 421 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { 422 r4k_blast_dcache_page(addr); 423 if (exec && !cpu_icache_snoops_remote_store) 424 r4k_blast_scache_page(addr); 425 } 426 if (exec) 427 r4k_blast_icache_page(addr); 428 429 return; 430 } 431 432 /* 433 * Do indexed flush, too much work to get the (possible) TLB refills 434 * to work correctly. 435 */ 436 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { 437 r4k_blast_dcache_page_indexed(cpu_has_pindexed_dcache ? 438 paddr : addr); 439 if (exec && !cpu_icache_snoops_remote_store) { 440 r4k_blast_scache_page_indexed(paddr); 441 } 442 } 443 if (exec) { 444 if (cpu_has_vtag_icache) { 445 int cpu = smp_processor_id(); 446 447 if (cpu_context(cpu, mm) != 0) 448 drop_mmu_context(mm, cpu); 449 } else 450 r4k_blast_icache_page_indexed(addr); 451 } 452 } 453 454 static void r4k_flush_cache_page(struct vm_area_struct *vma, 455 unsigned long addr, unsigned long pfn) 456 { 457 struct flush_cache_page_args args; 458 459 args.vma = vma; 460 args.addr = addr; 461 args.pfn = pfn; 462 463 on_each_cpu(local_r4k_flush_cache_page, &args, 1, 1); 464 } 465 466 static inline void local_r4k_flush_data_cache_page(void * addr) 467 { 468 r4k_blast_dcache_page((unsigned long) addr); 469 } 470 471 static void r4k_flush_data_cache_page(unsigned long addr) 472 { 473 on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr, 1, 1); 474 } 475 476 struct flush_icache_range_args { 477 unsigned long start; 478 unsigned long end; 479 }; 480 481 static inline void local_r4k_flush_icache_range(void *args) 482 { 483 struct flush_icache_range_args *fir_args = args; 484 unsigned long start = fir_args->start; 485 unsigned long end = fir_args->end; 486 487 if (!cpu_has_ic_fills_f_dc) { 488 if (end - start > dcache_size) { 489 r4k_blast_dcache(); 490 } else { 491 R4600_HIT_CACHEOP_WAR_IMPL; 492 protected_blast_dcache_range(start, end); 493 } 494 495 if (!cpu_icache_snoops_remote_store && scache_size) { 496 if (end - start > scache_size) 497 r4k_blast_scache(); 498 else 499 protected_blast_scache_range(start, end); 500 } 501 } 502 503 if (end - start > icache_size) 504 r4k_blast_icache(); 505 else 506 protected_blast_icache_range(start, end); 507 } 508 509 static void r4k_flush_icache_range(unsigned long start, unsigned long end) 510 { 511 struct flush_icache_range_args args; 512 513 args.start = start; 514 args.end = end; 515 516 on_each_cpu(local_r4k_flush_icache_range, &args, 1, 1); 517 instruction_hazard(); 518 } 519 520 /* 521 * Ok, this seriously sucks. We use them to flush a user page but don't 522 * know the virtual address, so we have to blast away the whole icache 523 * which is significantly more expensive than the real thing. Otoh we at 524 * least know the kernel address of the page so we can flush it 525 * selectivly. 526 */ 527 528 struct flush_icache_page_args { 529 struct vm_area_struct *vma; 530 struct page *page; 531 }; 532 533 static inline void local_r4k_flush_icache_page(void *args) 534 { 535 struct flush_icache_page_args *fip_args = args; 536 struct vm_area_struct *vma = fip_args->vma; 537 struct page *page = fip_args->page; 538 539 /* 540 * Tricky ... Because we don't know the virtual address we've got the 541 * choice of either invalidating the entire primary and secondary 542 * caches or invalidating the secondary caches also. With the subset 543 * enforcment on R4000SC, R4400SC, R10000 and R12000 invalidating the 544 * secondary cache will result in any entries in the primary caches 545 * also getting invalidated which hopefully is a bit more economical. 546 */ 547 if (cpu_has_subset_pcaches) { 548 unsigned long addr = (unsigned long) page_address(page); 549 550 r4k_blast_scache_page(addr); 551 ClearPageDcacheDirty(page); 552 553 return; 554 } 555 556 if (!cpu_has_ic_fills_f_dc) { 557 unsigned long addr = (unsigned long) page_address(page); 558 r4k_blast_dcache_page(addr); 559 if (!cpu_icache_snoops_remote_store) 560 r4k_blast_scache_page(addr); 561 ClearPageDcacheDirty(page); 562 } 563 564 /* 565 * We're not sure of the virtual address(es) involved here, so 566 * we have to flush the entire I-cache. 567 */ 568 if (cpu_has_vtag_icache) { 569 int cpu = smp_processor_id(); 570 571 if (cpu_context(cpu, vma->vm_mm) != 0) 572 drop_mmu_context(vma->vm_mm, cpu); 573 } else 574 r4k_blast_icache(); 575 } 576 577 static void r4k_flush_icache_page(struct vm_area_struct *vma, 578 struct page *page) 579 { 580 struct flush_icache_page_args args; 581 582 /* 583 * If there's no context yet, or the page isn't executable, no I-cache 584 * flush is needed. 585 */ 586 if (!(vma->vm_flags & VM_EXEC)) 587 return; 588 589 args.vma = vma; 590 args.page = page; 591 592 on_each_cpu(local_r4k_flush_icache_page, &args, 1, 1); 593 } 594 595 596 #ifdef CONFIG_DMA_NONCOHERENT 597 598 static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size) 599 { 600 /* Catch bad driver code */ 601 BUG_ON(size == 0); 602 603 if (cpu_has_subset_pcaches) { 604 if (size >= scache_size) 605 r4k_blast_scache(); 606 else 607 blast_scache_range(addr, addr + size); 608 return; 609 } 610 611 /* 612 * Either no secondary cache or the available caches don't have the 613 * subset property so we have to flush the primary caches 614 * explicitly 615 */ 616 if (size >= dcache_size) { 617 r4k_blast_dcache(); 618 } else { 619 R4600_HIT_CACHEOP_WAR_IMPL; 620 blast_dcache_range(addr, addr + size); 621 } 622 623 bc_wback_inv(addr, size); 624 } 625 626 static void r4k_dma_cache_inv(unsigned long addr, unsigned long size) 627 { 628 /* Catch bad driver code */ 629 BUG_ON(size == 0); 630 631 if (cpu_has_subset_pcaches) { 632 if (size >= scache_size) 633 r4k_blast_scache(); 634 else 635 blast_scache_range(addr, addr + size); 636 return; 637 } 638 639 if (size >= dcache_size) { 640 r4k_blast_dcache(); 641 } else { 642 R4600_HIT_CACHEOP_WAR_IMPL; 643 blast_dcache_range(addr, addr + size); 644 } 645 646 bc_inv(addr, size); 647 } 648 #endif /* CONFIG_DMA_NONCOHERENT */ 649 650 /* 651 * While we're protected against bad userland addresses we don't care 652 * very much about what happens in that case. Usually a segmentation 653 * fault will dump the process later on anyway ... 654 */ 655 static void local_r4k_flush_cache_sigtramp(void * arg) 656 { 657 unsigned long ic_lsize = cpu_icache_line_size(); 658 unsigned long dc_lsize = cpu_dcache_line_size(); 659 unsigned long sc_lsize = cpu_scache_line_size(); 660 unsigned long addr = (unsigned long) arg; 661 662 R4600_HIT_CACHEOP_WAR_IMPL; 663 protected_writeback_dcache_line(addr & ~(dc_lsize - 1)); 664 if (!cpu_icache_snoops_remote_store && scache_size) 665 protected_writeback_scache_line(addr & ~(sc_lsize - 1)); 666 protected_flush_icache_line(addr & ~(ic_lsize - 1)); 667 if (MIPS4K_ICACHE_REFILL_WAR) { 668 __asm__ __volatile__ ( 669 ".set push\n\t" 670 ".set noat\n\t" 671 ".set mips3\n\t" 672 #ifdef CONFIG_32BIT 673 "la $at,1f\n\t" 674 #endif 675 #ifdef CONFIG_64BIT 676 "dla $at,1f\n\t" 677 #endif 678 "cache %0,($at)\n\t" 679 "nop; nop; nop\n" 680 "1:\n\t" 681 ".set pop" 682 : 683 : "i" (Hit_Invalidate_I)); 684 } 685 if (MIPS_CACHE_SYNC_WAR) 686 __asm__ __volatile__ ("sync"); 687 } 688 689 static void r4k_flush_cache_sigtramp(unsigned long addr) 690 { 691 on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr, 1, 1); 692 } 693 694 static void r4k_flush_icache_all(void) 695 { 696 if (cpu_has_vtag_icache) 697 r4k_blast_icache(); 698 } 699 700 static inline void rm7k_erratum31(void) 701 { 702 const unsigned long ic_lsize = 32; 703 unsigned long addr; 704 705 /* RM7000 erratum #31. The icache is screwed at startup. */ 706 write_c0_taglo(0); 707 write_c0_taghi(0); 708 709 for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) { 710 __asm__ __volatile__ ( 711 ".set push\n\t" 712 ".set noreorder\n\t" 713 ".set mips3\n\t" 714 "cache\t%1, 0(%0)\n\t" 715 "cache\t%1, 0x1000(%0)\n\t" 716 "cache\t%1, 0x2000(%0)\n\t" 717 "cache\t%1, 0x3000(%0)\n\t" 718 "cache\t%2, 0(%0)\n\t" 719 "cache\t%2, 0x1000(%0)\n\t" 720 "cache\t%2, 0x2000(%0)\n\t" 721 "cache\t%2, 0x3000(%0)\n\t" 722 "cache\t%1, 0(%0)\n\t" 723 "cache\t%1, 0x1000(%0)\n\t" 724 "cache\t%1, 0x2000(%0)\n\t" 725 "cache\t%1, 0x3000(%0)\n\t" 726 ".set pop\n" 727 : 728 : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill)); 729 } 730 } 731 732 static char *way_string[] __initdata = { NULL, "direct mapped", "2-way", 733 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way" 734 }; 735 736 static void __init probe_pcache(void) 737 { 738 struct cpuinfo_mips *c = ¤t_cpu_data; 739 unsigned int config = read_c0_config(); 740 unsigned int prid = read_c0_prid(); 741 unsigned long config1; 742 unsigned int lsize; 743 744 switch (c->cputype) { 745 case CPU_R4600: /* QED style two way caches? */ 746 case CPU_R4700: 747 case CPU_R5000: 748 case CPU_NEVADA: 749 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 750 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 751 c->icache.ways = 2; 752 c->icache.waybit = ffs(icache_size/2) - 1; 753 754 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 755 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 756 c->dcache.ways = 2; 757 c->dcache.waybit= ffs(dcache_size/2) - 1; 758 759 c->options |= MIPS_CPU_CACHE_CDEX_P; 760 break; 761 762 case CPU_R5432: 763 case CPU_R5500: 764 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 765 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 766 c->icache.ways = 2; 767 c->icache.waybit= 0; 768 769 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 770 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 771 c->dcache.ways = 2; 772 c->dcache.waybit = 0; 773 774 c->options |= MIPS_CPU_CACHE_CDEX_P; 775 break; 776 777 case CPU_TX49XX: 778 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 779 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 780 c->icache.ways = 4; 781 c->icache.waybit= 0; 782 783 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 784 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 785 c->dcache.ways = 4; 786 c->dcache.waybit = 0; 787 788 c->options |= MIPS_CPU_CACHE_CDEX_P; 789 c->options |= MIPS_CPU_PREFETCH; 790 break; 791 792 case CPU_R4000PC: 793 case CPU_R4000SC: 794 case CPU_R4000MC: 795 case CPU_R4400PC: 796 case CPU_R4400SC: 797 case CPU_R4400MC: 798 case CPU_R4300: 799 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 800 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 801 c->icache.ways = 1; 802 c->icache.waybit = 0; /* doesn't matter */ 803 804 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 805 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 806 c->dcache.ways = 1; 807 c->dcache.waybit = 0; /* does not matter */ 808 809 c->options |= MIPS_CPU_CACHE_CDEX_P; 810 break; 811 812 case CPU_R10000: 813 case CPU_R12000: 814 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29)); 815 c->icache.linesz = 64; 816 c->icache.ways = 2; 817 c->icache.waybit = 0; 818 819 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26)); 820 c->dcache.linesz = 32; 821 c->dcache.ways = 2; 822 c->dcache.waybit = 0; 823 824 c->options |= MIPS_CPU_PREFETCH; 825 break; 826 827 case CPU_VR4133: 828 write_c0_config(config & ~CONF_EB); 829 case CPU_VR4131: 830 /* Workaround for cache instruction bug of VR4131 */ 831 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U || 832 c->processor_id == 0x0c82U) { 833 config &= ~0x00000030U; 834 config |= 0x00410000U; 835 write_c0_config(config); 836 } 837 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 838 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 839 c->icache.ways = 2; 840 c->icache.waybit = ffs(icache_size/2) - 1; 841 842 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 843 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 844 c->dcache.ways = 2; 845 c->dcache.waybit = ffs(dcache_size/2) - 1; 846 847 c->options |= MIPS_CPU_CACHE_CDEX_P; 848 break; 849 850 case CPU_VR41XX: 851 case CPU_VR4111: 852 case CPU_VR4121: 853 case CPU_VR4122: 854 case CPU_VR4181: 855 case CPU_VR4181A: 856 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 857 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 858 c->icache.ways = 1; 859 c->icache.waybit = 0; /* doesn't matter */ 860 861 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 862 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 863 c->dcache.ways = 1; 864 c->dcache.waybit = 0; /* does not matter */ 865 866 c->options |= MIPS_CPU_CACHE_CDEX_P; 867 break; 868 869 case CPU_RM7000: 870 rm7k_erratum31(); 871 872 case CPU_RM9000: 873 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 874 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 875 c->icache.ways = 4; 876 c->icache.waybit = ffs(icache_size / c->icache.ways) - 1; 877 878 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 879 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 880 c->dcache.ways = 4; 881 c->dcache.waybit = ffs(dcache_size / c->dcache.ways) - 1; 882 883 #if !defined(CONFIG_SMP) || !defined(RM9000_CDEX_SMP_WAR) 884 c->options |= MIPS_CPU_CACHE_CDEX_P; 885 #endif 886 c->options |= MIPS_CPU_PREFETCH; 887 break; 888 889 default: 890 if (!(config & MIPS_CONF_M)) 891 panic("Don't know how to probe P-caches on this cpu."); 892 893 /* 894 * So we seem to be a MIPS32 or MIPS64 CPU 895 * So let's probe the I-cache ... 896 */ 897 config1 = read_c0_config1(); 898 899 if ((lsize = ((config1 >> 19) & 7))) 900 c->icache.linesz = 2 << lsize; 901 else 902 c->icache.linesz = lsize; 903 c->icache.sets = 64 << ((config1 >> 22) & 7); 904 c->icache.ways = 1 + ((config1 >> 16) & 7); 905 906 icache_size = c->icache.sets * 907 c->icache.ways * 908 c->icache.linesz; 909 c->icache.waybit = ffs(icache_size/c->icache.ways) - 1; 910 911 if (config & 0x8) /* VI bit */ 912 c->icache.flags |= MIPS_CACHE_VTAG; 913 914 /* 915 * Now probe the MIPS32 / MIPS64 data cache. 916 */ 917 c->dcache.flags = 0; 918 919 if ((lsize = ((config1 >> 10) & 7))) 920 c->dcache.linesz = 2 << lsize; 921 else 922 c->dcache.linesz= lsize; 923 c->dcache.sets = 64 << ((config1 >> 13) & 7); 924 c->dcache.ways = 1 + ((config1 >> 7) & 7); 925 926 dcache_size = c->dcache.sets * 927 c->dcache.ways * 928 c->dcache.linesz; 929 c->dcache.waybit = ffs(dcache_size/c->dcache.ways) - 1; 930 931 c->options |= MIPS_CPU_PREFETCH; 932 break; 933 } 934 935 /* 936 * Processor configuration sanity check for the R4000SC erratum 937 * #5. With page sizes larger than 32kB there is no possibility 938 * to get a VCE exception anymore so we don't care about this 939 * misconfiguration. The case is rather theoretical anyway; 940 * presumably no vendor is shipping his hardware in the "bad" 941 * configuration. 942 */ 943 if ((prid & 0xff00) == PRID_IMP_R4000 && (prid & 0xff) < 0x40 && 944 !(config & CONF_SC) && c->icache.linesz != 16 && 945 PAGE_SIZE <= 0x8000) 946 panic("Improper R4000SC processor configuration detected"); 947 948 /* compute a couple of other cache variables */ 949 c->icache.waysize = icache_size / c->icache.ways; 950 c->dcache.waysize = dcache_size / c->dcache.ways; 951 952 c->icache.sets = icache_size / (c->icache.linesz * c->icache.ways); 953 c->dcache.sets = dcache_size / (c->dcache.linesz * c->dcache.ways); 954 955 /* 956 * R10000 and R12000 P-caches are odd in a positive way. They're 32kB 957 * 2-way virtually indexed so normally would suffer from aliases. So 958 * normally they'd suffer from aliases but magic in the hardware deals 959 * with that for us so we don't need to take care ourselves. 960 */ 961 switch (c->cputype) { 962 case CPU_20KC: 963 case CPU_25KF: 964 c->dcache.flags |= MIPS_CACHE_PINDEX; 965 case CPU_R10000: 966 case CPU_R12000: 967 case CPU_SB1: 968 break; 969 case CPU_24K: 970 if (!(read_c0_config7() & (1 << 16))) 971 default: 972 if (c->dcache.waysize > PAGE_SIZE) 973 c->dcache.flags |= MIPS_CACHE_ALIASES; 974 } 975 976 switch (c->cputype) { 977 case CPU_20KC: 978 /* 979 * Some older 20Kc chips doesn't have the 'VI' bit in 980 * the config register. 981 */ 982 c->icache.flags |= MIPS_CACHE_VTAG; 983 break; 984 985 case CPU_AU1000: 986 case CPU_AU1500: 987 case CPU_AU1100: 988 case CPU_AU1550: 989 case CPU_AU1200: 990 c->icache.flags |= MIPS_CACHE_IC_F_DC; 991 break; 992 } 993 994 printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n", 995 icache_size >> 10, 996 cpu_has_vtag_icache ? "virtually tagged" : "physically tagged", 997 way_string[c->icache.ways], c->icache.linesz); 998 999 printk("Primary data cache %ldkB, %s, linesize %d bytes.\n", 1000 dcache_size >> 10, way_string[c->dcache.ways], c->dcache.linesz); 1001 } 1002 1003 /* 1004 * If you even _breathe_ on this function, look at the gcc output and make sure 1005 * it does not pop things on and off the stack for the cache sizing loop that 1006 * executes in KSEG1 space or else you will crash and burn badly. You have 1007 * been warned. 1008 */ 1009 static int __init probe_scache(void) 1010 { 1011 extern unsigned long stext; 1012 unsigned long flags, addr, begin, end, pow2; 1013 unsigned int config = read_c0_config(); 1014 struct cpuinfo_mips *c = ¤t_cpu_data; 1015 int tmp; 1016 1017 if (config & CONF_SC) 1018 return 0; 1019 1020 begin = (unsigned long) &stext; 1021 begin &= ~((4 * 1024 * 1024) - 1); 1022 end = begin + (4 * 1024 * 1024); 1023 1024 /* 1025 * This is such a bitch, you'd think they would make it easy to do 1026 * this. Away you daemons of stupidity! 1027 */ 1028 local_irq_save(flags); 1029 1030 /* Fill each size-multiple cache line with a valid tag. */ 1031 pow2 = (64 * 1024); 1032 for (addr = begin; addr < end; addr = (begin + pow2)) { 1033 unsigned long *p = (unsigned long *) addr; 1034 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */ 1035 pow2 <<= 1; 1036 } 1037 1038 /* Load first line with zero (therefore invalid) tag. */ 1039 write_c0_taglo(0); 1040 write_c0_taghi(0); 1041 __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */ 1042 cache_op(Index_Store_Tag_I, begin); 1043 cache_op(Index_Store_Tag_D, begin); 1044 cache_op(Index_Store_Tag_SD, begin); 1045 1046 /* Now search for the wrap around point. */ 1047 pow2 = (128 * 1024); 1048 tmp = 0; 1049 for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) { 1050 cache_op(Index_Load_Tag_SD, addr); 1051 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */ 1052 if (!read_c0_taglo()) 1053 break; 1054 pow2 <<= 1; 1055 } 1056 local_irq_restore(flags); 1057 addr -= begin; 1058 1059 scache_size = addr; 1060 c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22); 1061 c->scache.ways = 1; 1062 c->dcache.waybit = 0; /* does not matter */ 1063 1064 return 1; 1065 } 1066 1067 extern int r5k_sc_init(void); 1068 extern int rm7k_sc_init(void); 1069 1070 static void __init setup_scache(void) 1071 { 1072 struct cpuinfo_mips *c = ¤t_cpu_data; 1073 unsigned int config = read_c0_config(); 1074 int sc_present = 0; 1075 1076 /* 1077 * Do the probing thing on R4000SC and R4400SC processors. Other 1078 * processors don't have a S-cache that would be relevant to the 1079 * Linux memory managment. 1080 */ 1081 switch (c->cputype) { 1082 case CPU_R4000SC: 1083 case CPU_R4000MC: 1084 case CPU_R4400SC: 1085 case CPU_R4400MC: 1086 sc_present = run_uncached(probe_scache); 1087 if (sc_present) 1088 c->options |= MIPS_CPU_CACHE_CDEX_S; 1089 break; 1090 1091 case CPU_R10000: 1092 case CPU_R12000: 1093 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16); 1094 c->scache.linesz = 64 << ((config >> 13) & 1); 1095 c->scache.ways = 2; 1096 c->scache.waybit= 0; 1097 sc_present = 1; 1098 break; 1099 1100 case CPU_R5000: 1101 case CPU_NEVADA: 1102 #ifdef CONFIG_R5000_CPU_SCACHE 1103 r5k_sc_init(); 1104 #endif 1105 return; 1106 1107 case CPU_RM7000: 1108 case CPU_RM9000: 1109 #ifdef CONFIG_RM7000_CPU_SCACHE 1110 rm7k_sc_init(); 1111 #endif 1112 return; 1113 1114 default: 1115 sc_present = 0; 1116 } 1117 1118 if (!sc_present) 1119 return; 1120 1121 if ((c->isa_level == MIPS_CPU_ISA_M32R1 || 1122 c->isa_level == MIPS_CPU_ISA_M64R1) && 1123 !(c->scache.flags & MIPS_CACHE_NOT_PRESENT)) 1124 panic("Dunno how to handle MIPS32 / MIPS64 second level cache"); 1125 1126 /* compute a couple of other cache variables */ 1127 c->scache.waysize = scache_size / c->scache.ways; 1128 1129 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways); 1130 1131 printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n", 1132 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); 1133 1134 c->options |= MIPS_CPU_SUBSET_CACHES; 1135 } 1136 1137 static inline void coherency_setup(void) 1138 { 1139 change_c0_config(CONF_CM_CMASK, CONF_CM_DEFAULT); 1140 1141 /* 1142 * c0_status.cu=0 specifies that updates by the sc instruction use 1143 * the coherency mode specified by the TLB; 1 means cachable 1144 * coherent update on write will be used. Not all processors have 1145 * this bit and; some wire it to zero, others like Toshiba had the 1146 * silly idea of putting something else there ... 1147 */ 1148 switch (current_cpu_data.cputype) { 1149 case CPU_R4000PC: 1150 case CPU_R4000SC: 1151 case CPU_R4000MC: 1152 case CPU_R4400PC: 1153 case CPU_R4400SC: 1154 case CPU_R4400MC: 1155 clear_c0_config(CONF_CU); 1156 break; 1157 } 1158 } 1159 1160 void __init r4k_cache_init(void) 1161 { 1162 extern void build_clear_page(void); 1163 extern void build_copy_page(void); 1164 extern char except_vec2_generic; 1165 struct cpuinfo_mips *c = ¤t_cpu_data; 1166 1167 /* Default cache error handler for R4000 and R5000 family */ 1168 set_uncached_handler (0x100, &except_vec2_generic, 0x80); 1169 1170 probe_pcache(); 1171 setup_scache(); 1172 1173 r4k_blast_dcache_page_setup(); 1174 r4k_blast_dcache_page_indexed_setup(); 1175 r4k_blast_dcache_setup(); 1176 r4k_blast_icache_page_setup(); 1177 r4k_blast_icache_page_indexed_setup(); 1178 r4k_blast_icache_setup(); 1179 r4k_blast_scache_page_setup(); 1180 r4k_blast_scache_page_indexed_setup(); 1181 r4k_blast_scache_setup(); 1182 1183 /* 1184 * Some MIPS32 and MIPS64 processors have physically indexed caches. 1185 * This code supports virtually indexed processors and will be 1186 * unnecessarily inefficient on physically indexed processors. 1187 */ 1188 shm_align_mask = max_t( unsigned long, 1189 c->dcache.sets * c->dcache.linesz - 1, 1190 PAGE_SIZE - 1); 1191 1192 flush_cache_all = r4k_flush_cache_all; 1193 __flush_cache_all = r4k___flush_cache_all; 1194 flush_cache_mm = r4k_flush_cache_mm; 1195 flush_cache_page = r4k_flush_cache_page; 1196 flush_icache_page = r4k_flush_icache_page; 1197 flush_cache_range = r4k_flush_cache_range; 1198 1199 flush_cache_sigtramp = r4k_flush_cache_sigtramp; 1200 flush_icache_all = r4k_flush_icache_all; 1201 flush_data_cache_page = r4k_flush_data_cache_page; 1202 flush_icache_range = r4k_flush_icache_range; 1203 1204 #ifdef CONFIG_DMA_NONCOHERENT 1205 _dma_cache_wback_inv = r4k_dma_cache_wback_inv; 1206 _dma_cache_wback = r4k_dma_cache_wback_inv; 1207 _dma_cache_inv = r4k_dma_cache_inv; 1208 #endif 1209 1210 build_clear_page(); 1211 build_copy_page(); 1212 local_r4k___flush_cache_all(NULL); 1213 coherency_setup(); 1214 } 1215