1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) 7 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org) 8 * Copyright (C) 1999, 2000 Silicon Graphics, Inc. 9 */ 10 #include <linux/config.h> 11 #include <linux/init.h> 12 #include <linux/kernel.h> 13 #include <linux/sched.h> 14 #include <linux/mm.h> 15 #include <linux/bitops.h> 16 17 #include <asm/bcache.h> 18 #include <asm/bootinfo.h> 19 #include <asm/cache.h> 20 #include <asm/cacheops.h> 21 #include <asm/cpu.h> 22 #include <asm/cpu-features.h> 23 #include <asm/io.h> 24 #include <asm/page.h> 25 #include <asm/pgtable.h> 26 #include <asm/r4kcache.h> 27 #include <asm/system.h> 28 #include <asm/mmu_context.h> 29 #include <asm/war.h> 30 #include <asm/cacheflush.h> /* for run_uncached() */ 31 32 /* 33 * Must die. 34 */ 35 static unsigned long icache_size __read_mostly; 36 static unsigned long dcache_size __read_mostly; 37 static unsigned long scache_size __read_mostly; 38 39 /* 40 * Dummy cache handling routines for machines without boardcaches 41 */ 42 static void no_sc_noop(void) {} 43 44 static struct bcache_ops no_sc_ops = { 45 .bc_enable = (void *)no_sc_noop, 46 .bc_disable = (void *)no_sc_noop, 47 .bc_wback_inv = (void *)no_sc_noop, 48 .bc_inv = (void *)no_sc_noop 49 }; 50 51 struct bcache_ops *bcops = &no_sc_ops; 52 53 #define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010) 54 #define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020) 55 56 #define R4600_HIT_CACHEOP_WAR_IMPL \ 57 do { \ 58 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) \ 59 *(volatile unsigned long *)CKSEG1; \ 60 if (R4600_V1_HIT_CACHEOP_WAR) \ 61 __asm__ __volatile__("nop;nop;nop;nop"); \ 62 } while (0) 63 64 static void (*r4k_blast_dcache_page)(unsigned long addr); 65 66 static inline void r4k_blast_dcache_page_dc32(unsigned long addr) 67 { 68 R4600_HIT_CACHEOP_WAR_IMPL; 69 blast_dcache32_page(addr); 70 } 71 72 static inline void r4k_blast_dcache_page_setup(void) 73 { 74 unsigned long dc_lsize = cpu_dcache_line_size(); 75 76 if (dc_lsize == 16) 77 r4k_blast_dcache_page = blast_dcache16_page; 78 else if (dc_lsize == 32) 79 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32; 80 } 81 82 static void (* r4k_blast_dcache_page_indexed)(unsigned long addr); 83 84 static inline void r4k_blast_dcache_page_indexed_setup(void) 85 { 86 unsigned long dc_lsize = cpu_dcache_line_size(); 87 88 if (dc_lsize == 16) 89 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed; 90 else if (dc_lsize == 32) 91 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed; 92 } 93 94 static void (* r4k_blast_dcache)(void); 95 96 static inline void r4k_blast_dcache_setup(void) 97 { 98 unsigned long dc_lsize = cpu_dcache_line_size(); 99 100 if (dc_lsize == 16) 101 r4k_blast_dcache = blast_dcache16; 102 else if (dc_lsize == 32) 103 r4k_blast_dcache = blast_dcache32; 104 } 105 106 /* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */ 107 #define JUMP_TO_ALIGN(order) \ 108 __asm__ __volatile__( \ 109 "b\t1f\n\t" \ 110 ".align\t" #order "\n\t" \ 111 "1:\n\t" \ 112 ) 113 #define CACHE32_UNROLL32_ALIGN JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */ 114 #define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11) 115 116 static inline void blast_r4600_v1_icache32(void) 117 { 118 unsigned long flags; 119 120 local_irq_save(flags); 121 blast_icache32(); 122 local_irq_restore(flags); 123 } 124 125 static inline void tx49_blast_icache32(void) 126 { 127 unsigned long start = INDEX_BASE; 128 unsigned long end = start + current_cpu_data.icache.waysize; 129 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 130 unsigned long ws_end = current_cpu_data.icache.ways << 131 current_cpu_data.icache.waybit; 132 unsigned long ws, addr; 133 134 CACHE32_UNROLL32_ALIGN2; 135 /* I'm in even chunk. blast odd chunks */ 136 for (ws = 0; ws < ws_end; ws += ws_inc) 137 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 138 cache32_unroll32(addr|ws,Index_Invalidate_I); 139 CACHE32_UNROLL32_ALIGN; 140 /* I'm in odd chunk. blast even chunks */ 141 for (ws = 0; ws < ws_end; ws += ws_inc) 142 for (addr = start; addr < end; addr += 0x400 * 2) 143 cache32_unroll32(addr|ws,Index_Invalidate_I); 144 } 145 146 static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page) 147 { 148 unsigned long flags; 149 150 local_irq_save(flags); 151 blast_icache32_page_indexed(page); 152 local_irq_restore(flags); 153 } 154 155 static inline void tx49_blast_icache32_page_indexed(unsigned long page) 156 { 157 unsigned long indexmask = current_cpu_data.icache.waysize - 1; 158 unsigned long start = INDEX_BASE + (page & indexmask); 159 unsigned long end = start + PAGE_SIZE; 160 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 161 unsigned long ws_end = current_cpu_data.icache.ways << 162 current_cpu_data.icache.waybit; 163 unsigned long ws, addr; 164 165 CACHE32_UNROLL32_ALIGN2; 166 /* I'm in even chunk. blast odd chunks */ 167 for (ws = 0; ws < ws_end; ws += ws_inc) 168 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 169 cache32_unroll32(addr|ws,Index_Invalidate_I); 170 CACHE32_UNROLL32_ALIGN; 171 /* I'm in odd chunk. blast even chunks */ 172 for (ws = 0; ws < ws_end; ws += ws_inc) 173 for (addr = start; addr < end; addr += 0x400 * 2) 174 cache32_unroll32(addr|ws,Index_Invalidate_I); 175 } 176 177 static void (* r4k_blast_icache_page)(unsigned long addr); 178 179 static inline void r4k_blast_icache_page_setup(void) 180 { 181 unsigned long ic_lsize = cpu_icache_line_size(); 182 183 if (ic_lsize == 16) 184 r4k_blast_icache_page = blast_icache16_page; 185 else if (ic_lsize == 32) 186 r4k_blast_icache_page = blast_icache32_page; 187 else if (ic_lsize == 64) 188 r4k_blast_icache_page = blast_icache64_page; 189 } 190 191 192 static void (* r4k_blast_icache_page_indexed)(unsigned long addr); 193 194 static inline void r4k_blast_icache_page_indexed_setup(void) 195 { 196 unsigned long ic_lsize = cpu_icache_line_size(); 197 198 if (ic_lsize == 16) 199 r4k_blast_icache_page_indexed = blast_icache16_page_indexed; 200 else if (ic_lsize == 32) { 201 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 202 r4k_blast_icache_page_indexed = 203 blast_icache32_r4600_v1_page_indexed; 204 else if (TX49XX_ICACHE_INDEX_INV_WAR) 205 r4k_blast_icache_page_indexed = 206 tx49_blast_icache32_page_indexed; 207 else 208 r4k_blast_icache_page_indexed = 209 blast_icache32_page_indexed; 210 } else if (ic_lsize == 64) 211 r4k_blast_icache_page_indexed = blast_icache64_page_indexed; 212 } 213 214 static void (* r4k_blast_icache)(void); 215 216 static inline void r4k_blast_icache_setup(void) 217 { 218 unsigned long ic_lsize = cpu_icache_line_size(); 219 220 if (ic_lsize == 16) 221 r4k_blast_icache = blast_icache16; 222 else if (ic_lsize == 32) { 223 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 224 r4k_blast_icache = blast_r4600_v1_icache32; 225 else if (TX49XX_ICACHE_INDEX_INV_WAR) 226 r4k_blast_icache = tx49_blast_icache32; 227 else 228 r4k_blast_icache = blast_icache32; 229 } else if (ic_lsize == 64) 230 r4k_blast_icache = blast_icache64; 231 } 232 233 static void (* r4k_blast_scache_page)(unsigned long addr); 234 235 static inline void r4k_blast_scache_page_setup(void) 236 { 237 unsigned long sc_lsize = cpu_scache_line_size(); 238 239 if (scache_size == 0) 240 r4k_blast_scache_page = (void *)no_sc_noop; 241 else if (sc_lsize == 16) 242 r4k_blast_scache_page = blast_scache16_page; 243 else if (sc_lsize == 32) 244 r4k_blast_scache_page = blast_scache32_page; 245 else if (sc_lsize == 64) 246 r4k_blast_scache_page = blast_scache64_page; 247 else if (sc_lsize == 128) 248 r4k_blast_scache_page = blast_scache128_page; 249 } 250 251 static void (* r4k_blast_scache_page_indexed)(unsigned long addr); 252 253 static inline void r4k_blast_scache_page_indexed_setup(void) 254 { 255 unsigned long sc_lsize = cpu_scache_line_size(); 256 257 if (scache_size == 0) 258 r4k_blast_scache_page_indexed = (void *)no_sc_noop; 259 else if (sc_lsize == 16) 260 r4k_blast_scache_page_indexed = blast_scache16_page_indexed; 261 else if (sc_lsize == 32) 262 r4k_blast_scache_page_indexed = blast_scache32_page_indexed; 263 else if (sc_lsize == 64) 264 r4k_blast_scache_page_indexed = blast_scache64_page_indexed; 265 else if (sc_lsize == 128) 266 r4k_blast_scache_page_indexed = blast_scache128_page_indexed; 267 } 268 269 static void (* r4k_blast_scache)(void); 270 271 static inline void r4k_blast_scache_setup(void) 272 { 273 unsigned long sc_lsize = cpu_scache_line_size(); 274 275 if (scache_size == 0) 276 r4k_blast_scache = (void *)no_sc_noop; 277 else if (sc_lsize == 16) 278 r4k_blast_scache = blast_scache16; 279 else if (sc_lsize == 32) 280 r4k_blast_scache = blast_scache32; 281 else if (sc_lsize == 64) 282 r4k_blast_scache = blast_scache64; 283 else if (sc_lsize == 128) 284 r4k_blast_scache = blast_scache128; 285 } 286 287 /* 288 * This is former mm's flush_cache_all() which really should be 289 * flush_cache_vunmap these days ... 290 */ 291 static inline void local_r4k_flush_cache_all(void * args) 292 { 293 r4k_blast_dcache(); 294 r4k_blast_icache(); 295 } 296 297 static void r4k_flush_cache_all(void) 298 { 299 if (!cpu_has_dc_aliases) 300 return; 301 302 on_each_cpu(local_r4k_flush_cache_all, NULL, 1, 1); 303 } 304 305 static inline void local_r4k___flush_cache_all(void * args) 306 { 307 r4k_blast_dcache(); 308 r4k_blast_icache(); 309 310 switch (current_cpu_data.cputype) { 311 case CPU_R4000SC: 312 case CPU_R4000MC: 313 case CPU_R4400SC: 314 case CPU_R4400MC: 315 case CPU_R10000: 316 case CPU_R12000: 317 r4k_blast_scache(); 318 } 319 } 320 321 static void r4k___flush_cache_all(void) 322 { 323 on_each_cpu(local_r4k___flush_cache_all, NULL, 1, 1); 324 } 325 326 static inline void local_r4k_flush_cache_range(void * args) 327 { 328 struct vm_area_struct *vma = args; 329 int exec; 330 331 if (!(cpu_context(smp_processor_id(), vma->vm_mm))) 332 return; 333 334 exec = vma->vm_flags & VM_EXEC; 335 if (cpu_has_dc_aliases || exec) 336 r4k_blast_dcache(); 337 if (exec) 338 r4k_blast_icache(); 339 } 340 341 static void r4k_flush_cache_range(struct vm_area_struct *vma, 342 unsigned long start, unsigned long end) 343 { 344 on_each_cpu(local_r4k_flush_cache_range, vma, 1, 1); 345 } 346 347 static inline void local_r4k_flush_cache_mm(void * args) 348 { 349 struct mm_struct *mm = args; 350 351 if (!cpu_context(smp_processor_id(), mm)) 352 return; 353 354 r4k_blast_dcache(); 355 r4k_blast_icache(); 356 357 /* 358 * Kludge alert. For obscure reasons R4000SC and R4400SC go nuts if we 359 * only flush the primary caches but R10000 and R12000 behave sane ... 360 */ 361 if (current_cpu_data.cputype == CPU_R4000SC || 362 current_cpu_data.cputype == CPU_R4000MC || 363 current_cpu_data.cputype == CPU_R4400SC || 364 current_cpu_data.cputype == CPU_R4400MC) 365 r4k_blast_scache(); 366 } 367 368 static void r4k_flush_cache_mm(struct mm_struct *mm) 369 { 370 if (!cpu_has_dc_aliases) 371 return; 372 373 on_each_cpu(local_r4k_flush_cache_mm, mm, 1, 1); 374 } 375 376 struct flush_cache_page_args { 377 struct vm_area_struct *vma; 378 unsigned long addr; 379 unsigned long pfn; 380 }; 381 382 static inline void local_r4k_flush_cache_page(void *args) 383 { 384 struct flush_cache_page_args *fcp_args = args; 385 struct vm_area_struct *vma = fcp_args->vma; 386 unsigned long addr = fcp_args->addr; 387 unsigned long paddr = fcp_args->pfn << PAGE_SHIFT; 388 int exec = vma->vm_flags & VM_EXEC; 389 struct mm_struct *mm = vma->vm_mm; 390 pgd_t *pgdp; 391 pud_t *pudp; 392 pmd_t *pmdp; 393 pte_t *ptep; 394 395 /* 396 * If ownes no valid ASID yet, cannot possibly have gotten 397 * this page into the cache. 398 */ 399 if (cpu_context(smp_processor_id(), mm) == 0) 400 return; 401 402 addr &= PAGE_MASK; 403 pgdp = pgd_offset(mm, addr); 404 pudp = pud_offset(pgdp, addr); 405 pmdp = pmd_offset(pudp, addr); 406 ptep = pte_offset(pmdp, addr); 407 408 /* 409 * If the page isn't marked valid, the page cannot possibly be 410 * in the cache. 411 */ 412 if (!(pte_val(*ptep) & _PAGE_PRESENT)) 413 return; 414 415 /* 416 * Doing flushes for another ASID than the current one is 417 * too difficult since stupid R4k caches do a TLB translation 418 * for every cache flush operation. So we do indexed flushes 419 * in that case, which doesn't overly flush the cache too much. 420 */ 421 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) { 422 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { 423 r4k_blast_dcache_page(addr); 424 if (exec && !cpu_icache_snoops_remote_store) 425 r4k_blast_scache_page(addr); 426 } 427 if (exec) 428 r4k_blast_icache_page(addr); 429 430 return; 431 } 432 433 /* 434 * Do indexed flush, too much work to get the (possible) TLB refills 435 * to work correctly. 436 */ 437 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { 438 r4k_blast_dcache_page_indexed(cpu_has_pindexed_dcache ? 439 paddr : addr); 440 if (exec && !cpu_icache_snoops_remote_store) { 441 r4k_blast_scache_page_indexed(paddr); 442 } 443 } 444 if (exec) { 445 if (cpu_has_vtag_icache) { 446 int cpu = smp_processor_id(); 447 448 if (cpu_context(cpu, mm) != 0) 449 drop_mmu_context(mm, cpu); 450 } else 451 r4k_blast_icache_page_indexed(addr); 452 } 453 } 454 455 static void r4k_flush_cache_page(struct vm_area_struct *vma, 456 unsigned long addr, unsigned long pfn) 457 { 458 struct flush_cache_page_args args; 459 460 args.vma = vma; 461 args.addr = addr; 462 args.pfn = pfn; 463 464 on_each_cpu(local_r4k_flush_cache_page, &args, 1, 1); 465 } 466 467 static inline void local_r4k_flush_data_cache_page(void * addr) 468 { 469 r4k_blast_dcache_page((unsigned long) addr); 470 } 471 472 static void r4k_flush_data_cache_page(unsigned long addr) 473 { 474 on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr, 1, 1); 475 } 476 477 struct flush_icache_range_args { 478 unsigned long start; 479 unsigned long end; 480 }; 481 482 static inline void local_r4k_flush_icache_range(void *args) 483 { 484 struct flush_icache_range_args *fir_args = args; 485 unsigned long start = fir_args->start; 486 unsigned long end = fir_args->end; 487 488 if (!cpu_has_ic_fills_f_dc) { 489 if (end - start > dcache_size) { 490 r4k_blast_dcache(); 491 } else { 492 R4600_HIT_CACHEOP_WAR_IMPL; 493 protected_blast_dcache_range(start, end); 494 } 495 496 if (!cpu_icache_snoops_remote_store && scache_size) { 497 if (end - start > scache_size) 498 r4k_blast_scache(); 499 else 500 protected_blast_scache_range(start, end); 501 } 502 } 503 504 if (end - start > icache_size) 505 r4k_blast_icache(); 506 else 507 protected_blast_icache_range(start, end); 508 } 509 510 static void r4k_flush_icache_range(unsigned long start, unsigned long end) 511 { 512 struct flush_icache_range_args args; 513 514 args.start = start; 515 args.end = end; 516 517 on_each_cpu(local_r4k_flush_icache_range, &args, 1, 1); 518 instruction_hazard(); 519 } 520 521 /* 522 * Ok, this seriously sucks. We use them to flush a user page but don't 523 * know the virtual address, so we have to blast away the whole icache 524 * which is significantly more expensive than the real thing. Otoh we at 525 * least know the kernel address of the page so we can flush it 526 * selectivly. 527 */ 528 529 struct flush_icache_page_args { 530 struct vm_area_struct *vma; 531 struct page *page; 532 }; 533 534 static inline void local_r4k_flush_icache_page(void *args) 535 { 536 struct flush_icache_page_args *fip_args = args; 537 struct vm_area_struct *vma = fip_args->vma; 538 struct page *page = fip_args->page; 539 540 /* 541 * Tricky ... Because we don't know the virtual address we've got the 542 * choice of either invalidating the entire primary and secondary 543 * caches or invalidating the secondary caches also. With the subset 544 * enforcment on R4000SC, R4400SC, R10000 and R12000 invalidating the 545 * secondary cache will result in any entries in the primary caches 546 * also getting invalidated which hopefully is a bit more economical. 547 */ 548 if (cpu_has_subset_pcaches) { 549 unsigned long addr = (unsigned long) page_address(page); 550 551 r4k_blast_scache_page(addr); 552 ClearPageDcacheDirty(page); 553 554 return; 555 } 556 557 if (!cpu_has_ic_fills_f_dc) { 558 unsigned long addr = (unsigned long) page_address(page); 559 r4k_blast_dcache_page(addr); 560 if (!cpu_icache_snoops_remote_store) 561 r4k_blast_scache_page(addr); 562 ClearPageDcacheDirty(page); 563 } 564 565 /* 566 * We're not sure of the virtual address(es) involved here, so 567 * we have to flush the entire I-cache. 568 */ 569 if (cpu_has_vtag_icache) { 570 int cpu = smp_processor_id(); 571 572 if (cpu_context(cpu, vma->vm_mm) != 0) 573 drop_mmu_context(vma->vm_mm, cpu); 574 } else 575 r4k_blast_icache(); 576 } 577 578 static void r4k_flush_icache_page(struct vm_area_struct *vma, 579 struct page *page) 580 { 581 struct flush_icache_page_args args; 582 583 /* 584 * If there's no context yet, or the page isn't executable, no I-cache 585 * flush is needed. 586 */ 587 if (!(vma->vm_flags & VM_EXEC)) 588 return; 589 590 args.vma = vma; 591 args.page = page; 592 593 on_each_cpu(local_r4k_flush_icache_page, &args, 1, 1); 594 } 595 596 597 #ifdef CONFIG_DMA_NONCOHERENT 598 599 static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size) 600 { 601 /* Catch bad driver code */ 602 BUG_ON(size == 0); 603 604 if (cpu_has_subset_pcaches) { 605 if (size >= scache_size) 606 r4k_blast_scache(); 607 else 608 blast_scache_range(addr, addr + size); 609 return; 610 } 611 612 /* 613 * Either no secondary cache or the available caches don't have the 614 * subset property so we have to flush the primary caches 615 * explicitly 616 */ 617 if (size >= dcache_size) { 618 r4k_blast_dcache(); 619 } else { 620 R4600_HIT_CACHEOP_WAR_IMPL; 621 blast_dcache_range(addr, addr + size); 622 } 623 624 bc_wback_inv(addr, size); 625 } 626 627 static void r4k_dma_cache_inv(unsigned long addr, unsigned long size) 628 { 629 /* Catch bad driver code */ 630 BUG_ON(size == 0); 631 632 if (cpu_has_subset_pcaches) { 633 if (size >= scache_size) 634 r4k_blast_scache(); 635 else 636 blast_scache_range(addr, addr + size); 637 return; 638 } 639 640 if (size >= dcache_size) { 641 r4k_blast_dcache(); 642 } else { 643 R4600_HIT_CACHEOP_WAR_IMPL; 644 blast_dcache_range(addr, addr + size); 645 } 646 647 bc_inv(addr, size); 648 } 649 #endif /* CONFIG_DMA_NONCOHERENT */ 650 651 /* 652 * While we're protected against bad userland addresses we don't care 653 * very much about what happens in that case. Usually a segmentation 654 * fault will dump the process later on anyway ... 655 */ 656 static void local_r4k_flush_cache_sigtramp(void * arg) 657 { 658 unsigned long ic_lsize = cpu_icache_line_size(); 659 unsigned long dc_lsize = cpu_dcache_line_size(); 660 unsigned long sc_lsize = cpu_scache_line_size(); 661 unsigned long addr = (unsigned long) arg; 662 663 R4600_HIT_CACHEOP_WAR_IMPL; 664 protected_writeback_dcache_line(addr & ~(dc_lsize - 1)); 665 if (!cpu_icache_snoops_remote_store && scache_size) 666 protected_writeback_scache_line(addr & ~(sc_lsize - 1)); 667 protected_flush_icache_line(addr & ~(ic_lsize - 1)); 668 if (MIPS4K_ICACHE_REFILL_WAR) { 669 __asm__ __volatile__ ( 670 ".set push\n\t" 671 ".set noat\n\t" 672 ".set mips3\n\t" 673 #ifdef CONFIG_32BIT 674 "la $at,1f\n\t" 675 #endif 676 #ifdef CONFIG_64BIT 677 "dla $at,1f\n\t" 678 #endif 679 "cache %0,($at)\n\t" 680 "nop; nop; nop\n" 681 "1:\n\t" 682 ".set pop" 683 : 684 : "i" (Hit_Invalidate_I)); 685 } 686 if (MIPS_CACHE_SYNC_WAR) 687 __asm__ __volatile__ ("sync"); 688 } 689 690 static void r4k_flush_cache_sigtramp(unsigned long addr) 691 { 692 on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr, 1, 1); 693 } 694 695 static void r4k_flush_icache_all(void) 696 { 697 if (cpu_has_vtag_icache) 698 r4k_blast_icache(); 699 } 700 701 static inline void rm7k_erratum31(void) 702 { 703 const unsigned long ic_lsize = 32; 704 unsigned long addr; 705 706 /* RM7000 erratum #31. The icache is screwed at startup. */ 707 write_c0_taglo(0); 708 write_c0_taghi(0); 709 710 for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) { 711 __asm__ __volatile__ ( 712 ".set push\n\t" 713 ".set noreorder\n\t" 714 ".set mips3\n\t" 715 "cache\t%1, 0(%0)\n\t" 716 "cache\t%1, 0x1000(%0)\n\t" 717 "cache\t%1, 0x2000(%0)\n\t" 718 "cache\t%1, 0x3000(%0)\n\t" 719 "cache\t%2, 0(%0)\n\t" 720 "cache\t%2, 0x1000(%0)\n\t" 721 "cache\t%2, 0x2000(%0)\n\t" 722 "cache\t%2, 0x3000(%0)\n\t" 723 "cache\t%1, 0(%0)\n\t" 724 "cache\t%1, 0x1000(%0)\n\t" 725 "cache\t%1, 0x2000(%0)\n\t" 726 "cache\t%1, 0x3000(%0)\n\t" 727 ".set pop\n" 728 : 729 : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill)); 730 } 731 } 732 733 static char *way_string[] __initdata = { NULL, "direct mapped", "2-way", 734 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way" 735 }; 736 737 static void __init probe_pcache(void) 738 { 739 struct cpuinfo_mips *c = ¤t_cpu_data; 740 unsigned int config = read_c0_config(); 741 unsigned int prid = read_c0_prid(); 742 unsigned long config1; 743 unsigned int lsize; 744 745 switch (c->cputype) { 746 case CPU_R4600: /* QED style two way caches? */ 747 case CPU_R4700: 748 case CPU_R5000: 749 case CPU_NEVADA: 750 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 751 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 752 c->icache.ways = 2; 753 c->icache.waybit = __ffs(icache_size/2); 754 755 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 756 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 757 c->dcache.ways = 2; 758 c->dcache.waybit= __ffs(dcache_size/2); 759 760 c->options |= MIPS_CPU_CACHE_CDEX_P; 761 break; 762 763 case CPU_R5432: 764 case CPU_R5500: 765 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 766 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 767 c->icache.ways = 2; 768 c->icache.waybit= 0; 769 770 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 771 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 772 c->dcache.ways = 2; 773 c->dcache.waybit = 0; 774 775 c->options |= MIPS_CPU_CACHE_CDEX_P; 776 break; 777 778 case CPU_TX49XX: 779 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 780 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 781 c->icache.ways = 4; 782 c->icache.waybit= 0; 783 784 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 785 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 786 c->dcache.ways = 4; 787 c->dcache.waybit = 0; 788 789 c->options |= MIPS_CPU_CACHE_CDEX_P; 790 c->options |= MIPS_CPU_PREFETCH; 791 break; 792 793 case CPU_R4000PC: 794 case CPU_R4000SC: 795 case CPU_R4000MC: 796 case CPU_R4400PC: 797 case CPU_R4400SC: 798 case CPU_R4400MC: 799 case CPU_R4300: 800 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 801 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 802 c->icache.ways = 1; 803 c->icache.waybit = 0; /* doesn't matter */ 804 805 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 806 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 807 c->dcache.ways = 1; 808 c->dcache.waybit = 0; /* does not matter */ 809 810 c->options |= MIPS_CPU_CACHE_CDEX_P; 811 break; 812 813 case CPU_R10000: 814 case CPU_R12000: 815 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29)); 816 c->icache.linesz = 64; 817 c->icache.ways = 2; 818 c->icache.waybit = 0; 819 820 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26)); 821 c->dcache.linesz = 32; 822 c->dcache.ways = 2; 823 c->dcache.waybit = 0; 824 825 c->options |= MIPS_CPU_PREFETCH; 826 break; 827 828 case CPU_VR4133: 829 write_c0_config(config & ~CONF_EB); 830 case CPU_VR4131: 831 /* Workaround for cache instruction bug of VR4131 */ 832 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U || 833 c->processor_id == 0x0c82U) { 834 config &= ~0x00000030U; 835 config |= 0x00410000U; 836 write_c0_config(config); 837 } 838 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 839 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 840 c->icache.ways = 2; 841 c->icache.waybit = __ffs(icache_size/2); 842 843 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 844 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 845 c->dcache.ways = 2; 846 c->dcache.waybit = __ffs(dcache_size/2); 847 848 c->options |= MIPS_CPU_CACHE_CDEX_P; 849 break; 850 851 case CPU_VR41XX: 852 case CPU_VR4111: 853 case CPU_VR4121: 854 case CPU_VR4122: 855 case CPU_VR4181: 856 case CPU_VR4181A: 857 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 858 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 859 c->icache.ways = 1; 860 c->icache.waybit = 0; /* doesn't matter */ 861 862 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 863 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 864 c->dcache.ways = 1; 865 c->dcache.waybit = 0; /* does not matter */ 866 867 c->options |= MIPS_CPU_CACHE_CDEX_P; 868 break; 869 870 case CPU_RM7000: 871 rm7k_erratum31(); 872 873 case CPU_RM9000: 874 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 875 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 876 c->icache.ways = 4; 877 c->icache.waybit = __ffs(icache_size / c->icache.ways); 878 879 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 880 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 881 c->dcache.ways = 4; 882 c->dcache.waybit = __ffs(dcache_size / c->dcache.ways); 883 884 #if !defined(CONFIG_SMP) || !defined(RM9000_CDEX_SMP_WAR) 885 c->options |= MIPS_CPU_CACHE_CDEX_P; 886 #endif 887 c->options |= MIPS_CPU_PREFETCH; 888 break; 889 890 default: 891 if (!(config & MIPS_CONF_M)) 892 panic("Don't know how to probe P-caches on this cpu."); 893 894 /* 895 * So we seem to be a MIPS32 or MIPS64 CPU 896 * So let's probe the I-cache ... 897 */ 898 config1 = read_c0_config1(); 899 900 if ((lsize = ((config1 >> 19) & 7))) 901 c->icache.linesz = 2 << lsize; 902 else 903 c->icache.linesz = lsize; 904 c->icache.sets = 64 << ((config1 >> 22) & 7); 905 c->icache.ways = 1 + ((config1 >> 16) & 7); 906 907 icache_size = c->icache.sets * 908 c->icache.ways * 909 c->icache.linesz; 910 c->icache.waybit = __ffs(icache_size/c->icache.ways); 911 912 if (config & 0x8) /* VI bit */ 913 c->icache.flags |= MIPS_CACHE_VTAG; 914 915 /* 916 * Now probe the MIPS32 / MIPS64 data cache. 917 */ 918 c->dcache.flags = 0; 919 920 if ((lsize = ((config1 >> 10) & 7))) 921 c->dcache.linesz = 2 << lsize; 922 else 923 c->dcache.linesz= lsize; 924 c->dcache.sets = 64 << ((config1 >> 13) & 7); 925 c->dcache.ways = 1 + ((config1 >> 7) & 7); 926 927 dcache_size = c->dcache.sets * 928 c->dcache.ways * 929 c->dcache.linesz; 930 c->dcache.waybit = __ffs(dcache_size/c->dcache.ways); 931 932 c->options |= MIPS_CPU_PREFETCH; 933 break; 934 } 935 936 /* 937 * Processor configuration sanity check for the R4000SC erratum 938 * #5. With page sizes larger than 32kB there is no possibility 939 * to get a VCE exception anymore so we don't care about this 940 * misconfiguration. The case is rather theoretical anyway; 941 * presumably no vendor is shipping his hardware in the "bad" 942 * configuration. 943 */ 944 if ((prid & 0xff00) == PRID_IMP_R4000 && (prid & 0xff) < 0x40 && 945 !(config & CONF_SC) && c->icache.linesz != 16 && 946 PAGE_SIZE <= 0x8000) 947 panic("Improper R4000SC processor configuration detected"); 948 949 /* compute a couple of other cache variables */ 950 c->icache.waysize = icache_size / c->icache.ways; 951 c->dcache.waysize = dcache_size / c->dcache.ways; 952 953 c->icache.sets = icache_size / (c->icache.linesz * c->icache.ways); 954 c->dcache.sets = dcache_size / (c->dcache.linesz * c->dcache.ways); 955 956 /* 957 * R10000 and R12000 P-caches are odd in a positive way. They're 32kB 958 * 2-way virtually indexed so normally would suffer from aliases. So 959 * normally they'd suffer from aliases but magic in the hardware deals 960 * with that for us so we don't need to take care ourselves. 961 */ 962 switch (c->cputype) { 963 case CPU_20KC: 964 case CPU_25KF: 965 c->dcache.flags |= MIPS_CACHE_PINDEX; 966 case CPU_R10000: 967 case CPU_R12000: 968 case CPU_SB1: 969 break; 970 case CPU_24K: 971 if (!(read_c0_config7() & (1 << 16))) 972 default: 973 if (c->dcache.waysize > PAGE_SIZE) 974 c->dcache.flags |= MIPS_CACHE_ALIASES; 975 } 976 977 switch (c->cputype) { 978 case CPU_20KC: 979 /* 980 * Some older 20Kc chips doesn't have the 'VI' bit in 981 * the config register. 982 */ 983 c->icache.flags |= MIPS_CACHE_VTAG; 984 break; 985 986 case CPU_AU1000: 987 case CPU_AU1500: 988 case CPU_AU1100: 989 case CPU_AU1550: 990 case CPU_AU1200: 991 c->icache.flags |= MIPS_CACHE_IC_F_DC; 992 break; 993 } 994 995 printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n", 996 icache_size >> 10, 997 cpu_has_vtag_icache ? "virtually tagged" : "physically tagged", 998 way_string[c->icache.ways], c->icache.linesz); 999 1000 printk("Primary data cache %ldkB, %s, linesize %d bytes.\n", 1001 dcache_size >> 10, way_string[c->dcache.ways], c->dcache.linesz); 1002 } 1003 1004 /* 1005 * If you even _breathe_ on this function, look at the gcc output and make sure 1006 * it does not pop things on and off the stack for the cache sizing loop that 1007 * executes in KSEG1 space or else you will crash and burn badly. You have 1008 * been warned. 1009 */ 1010 static int __init probe_scache(void) 1011 { 1012 extern unsigned long stext; 1013 unsigned long flags, addr, begin, end, pow2; 1014 unsigned int config = read_c0_config(); 1015 struct cpuinfo_mips *c = ¤t_cpu_data; 1016 int tmp; 1017 1018 if (config & CONF_SC) 1019 return 0; 1020 1021 begin = (unsigned long) &stext; 1022 begin &= ~((4 * 1024 * 1024) - 1); 1023 end = begin + (4 * 1024 * 1024); 1024 1025 /* 1026 * This is such a bitch, you'd think they would make it easy to do 1027 * this. Away you daemons of stupidity! 1028 */ 1029 local_irq_save(flags); 1030 1031 /* Fill each size-multiple cache line with a valid tag. */ 1032 pow2 = (64 * 1024); 1033 for (addr = begin; addr < end; addr = (begin + pow2)) { 1034 unsigned long *p = (unsigned long *) addr; 1035 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */ 1036 pow2 <<= 1; 1037 } 1038 1039 /* Load first line with zero (therefore invalid) tag. */ 1040 write_c0_taglo(0); 1041 write_c0_taghi(0); 1042 __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */ 1043 cache_op(Index_Store_Tag_I, begin); 1044 cache_op(Index_Store_Tag_D, begin); 1045 cache_op(Index_Store_Tag_SD, begin); 1046 1047 /* Now search for the wrap around point. */ 1048 pow2 = (128 * 1024); 1049 tmp = 0; 1050 for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) { 1051 cache_op(Index_Load_Tag_SD, addr); 1052 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */ 1053 if (!read_c0_taglo()) 1054 break; 1055 pow2 <<= 1; 1056 } 1057 local_irq_restore(flags); 1058 addr -= begin; 1059 1060 scache_size = addr; 1061 c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22); 1062 c->scache.ways = 1; 1063 c->dcache.waybit = 0; /* does not matter */ 1064 1065 return 1; 1066 } 1067 1068 extern int r5k_sc_init(void); 1069 extern int rm7k_sc_init(void); 1070 1071 static void __init setup_scache(void) 1072 { 1073 struct cpuinfo_mips *c = ¤t_cpu_data; 1074 unsigned int config = read_c0_config(); 1075 int sc_present = 0; 1076 1077 /* 1078 * Do the probing thing on R4000SC and R4400SC processors. Other 1079 * processors don't have a S-cache that would be relevant to the 1080 * Linux memory managment. 1081 */ 1082 switch (c->cputype) { 1083 case CPU_R4000SC: 1084 case CPU_R4000MC: 1085 case CPU_R4400SC: 1086 case CPU_R4400MC: 1087 sc_present = run_uncached(probe_scache); 1088 if (sc_present) 1089 c->options |= MIPS_CPU_CACHE_CDEX_S; 1090 break; 1091 1092 case CPU_R10000: 1093 case CPU_R12000: 1094 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16); 1095 c->scache.linesz = 64 << ((config >> 13) & 1); 1096 c->scache.ways = 2; 1097 c->scache.waybit= 0; 1098 sc_present = 1; 1099 break; 1100 1101 case CPU_R5000: 1102 case CPU_NEVADA: 1103 #ifdef CONFIG_R5000_CPU_SCACHE 1104 r5k_sc_init(); 1105 #endif 1106 return; 1107 1108 case CPU_RM7000: 1109 case CPU_RM9000: 1110 #ifdef CONFIG_RM7000_CPU_SCACHE 1111 rm7k_sc_init(); 1112 #endif 1113 return; 1114 1115 default: 1116 sc_present = 0; 1117 } 1118 1119 if (!sc_present) 1120 return; 1121 1122 if ((c->isa_level == MIPS_CPU_ISA_M32R1 || 1123 c->isa_level == MIPS_CPU_ISA_M64R1) && 1124 !(c->scache.flags & MIPS_CACHE_NOT_PRESENT)) 1125 panic("Dunno how to handle MIPS32 / MIPS64 second level cache"); 1126 1127 /* compute a couple of other cache variables */ 1128 c->scache.waysize = scache_size / c->scache.ways; 1129 1130 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways); 1131 1132 printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n", 1133 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); 1134 1135 c->options |= MIPS_CPU_SUBSET_CACHES; 1136 } 1137 1138 static inline void coherency_setup(void) 1139 { 1140 change_c0_config(CONF_CM_CMASK, CONF_CM_DEFAULT); 1141 1142 /* 1143 * c0_status.cu=0 specifies that updates by the sc instruction use 1144 * the coherency mode specified by the TLB; 1 means cachable 1145 * coherent update on write will be used. Not all processors have 1146 * this bit and; some wire it to zero, others like Toshiba had the 1147 * silly idea of putting something else there ... 1148 */ 1149 switch (current_cpu_data.cputype) { 1150 case CPU_R4000PC: 1151 case CPU_R4000SC: 1152 case CPU_R4000MC: 1153 case CPU_R4400PC: 1154 case CPU_R4400SC: 1155 case CPU_R4400MC: 1156 clear_c0_config(CONF_CU); 1157 break; 1158 } 1159 } 1160 1161 void __init r4k_cache_init(void) 1162 { 1163 extern void build_clear_page(void); 1164 extern void build_copy_page(void); 1165 extern char except_vec2_generic; 1166 struct cpuinfo_mips *c = ¤t_cpu_data; 1167 1168 /* Default cache error handler for R4000 and R5000 family */ 1169 set_uncached_handler (0x100, &except_vec2_generic, 0x80); 1170 1171 probe_pcache(); 1172 setup_scache(); 1173 1174 r4k_blast_dcache_page_setup(); 1175 r4k_blast_dcache_page_indexed_setup(); 1176 r4k_blast_dcache_setup(); 1177 r4k_blast_icache_page_setup(); 1178 r4k_blast_icache_page_indexed_setup(); 1179 r4k_blast_icache_setup(); 1180 r4k_blast_scache_page_setup(); 1181 r4k_blast_scache_page_indexed_setup(); 1182 r4k_blast_scache_setup(); 1183 1184 /* 1185 * Some MIPS32 and MIPS64 processors have physically indexed caches. 1186 * This code supports virtually indexed processors and will be 1187 * unnecessarily inefficient on physically indexed processors. 1188 */ 1189 shm_align_mask = max_t( unsigned long, 1190 c->dcache.sets * c->dcache.linesz - 1, 1191 PAGE_SIZE - 1); 1192 1193 flush_cache_all = r4k_flush_cache_all; 1194 __flush_cache_all = r4k___flush_cache_all; 1195 flush_cache_mm = r4k_flush_cache_mm; 1196 flush_cache_page = r4k_flush_cache_page; 1197 flush_icache_page = r4k_flush_icache_page; 1198 flush_cache_range = r4k_flush_cache_range; 1199 1200 flush_cache_sigtramp = r4k_flush_cache_sigtramp; 1201 flush_icache_all = r4k_flush_icache_all; 1202 local_flush_data_cache_page = local_r4k_flush_data_cache_page; 1203 flush_data_cache_page = r4k_flush_data_cache_page; 1204 flush_icache_range = r4k_flush_icache_range; 1205 1206 #ifdef CONFIG_DMA_NONCOHERENT 1207 _dma_cache_wback_inv = r4k_dma_cache_wback_inv; 1208 _dma_cache_wback = r4k_dma_cache_wback_inv; 1209 _dma_cache_inv = r4k_dma_cache_inv; 1210 #endif 1211 1212 build_clear_page(); 1213 build_copy_page(); 1214 local_r4k___flush_cache_all(NULL); 1215 coherency_setup(); 1216 } 1217