1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) 7 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org) 8 * Copyright (C) 1999, 2000 Silicon Graphics, Inc. 9 */ 10 #include <linux/config.h> 11 #include <linux/init.h> 12 #include <linux/kernel.h> 13 #include <linux/sched.h> 14 #include <linux/mm.h> 15 #include <linux/bitops.h> 16 17 #include <asm/bcache.h> 18 #include <asm/bootinfo.h> 19 #include <asm/cache.h> 20 #include <asm/cacheops.h> 21 #include <asm/cpu.h> 22 #include <asm/cpu-features.h> 23 #include <asm/io.h> 24 #include <asm/page.h> 25 #include <asm/pgtable.h> 26 #include <asm/r4kcache.h> 27 #include <asm/system.h> 28 #include <asm/mmu_context.h> 29 #include <asm/war.h> 30 #include <asm/cacheflush.h> /* for run_uncached() */ 31 32 /* 33 * Must die. 34 */ 35 static unsigned long icache_size __read_mostly; 36 static unsigned long dcache_size __read_mostly; 37 static unsigned long scache_size __read_mostly; 38 39 /* 40 * Dummy cache handling routines for machines without boardcaches 41 */ 42 static void no_sc_noop(void) {} 43 44 static struct bcache_ops no_sc_ops = { 45 .bc_enable = (void *)no_sc_noop, 46 .bc_disable = (void *)no_sc_noop, 47 .bc_wback_inv = (void *)no_sc_noop, 48 .bc_inv = (void *)no_sc_noop 49 }; 50 51 struct bcache_ops *bcops = &no_sc_ops; 52 53 #define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010) 54 #define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020) 55 56 #define R4600_HIT_CACHEOP_WAR_IMPL \ 57 do { \ 58 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) \ 59 *(volatile unsigned long *)CKSEG1; \ 60 if (R4600_V1_HIT_CACHEOP_WAR) \ 61 __asm__ __volatile__("nop;nop;nop;nop"); \ 62 } while (0) 63 64 static void (*r4k_blast_dcache_page)(unsigned long addr); 65 66 static inline void r4k_blast_dcache_page_dc32(unsigned long addr) 67 { 68 R4600_HIT_CACHEOP_WAR_IMPL; 69 blast_dcache32_page(addr); 70 } 71 72 static inline void r4k_blast_dcache_page_setup(void) 73 { 74 unsigned long dc_lsize = cpu_dcache_line_size(); 75 76 if (dc_lsize == 16) 77 r4k_blast_dcache_page = blast_dcache16_page; 78 else if (dc_lsize == 32) 79 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32; 80 } 81 82 static void (* r4k_blast_dcache_page_indexed)(unsigned long addr); 83 84 static inline void r4k_blast_dcache_page_indexed_setup(void) 85 { 86 unsigned long dc_lsize = cpu_dcache_line_size(); 87 88 if (dc_lsize == 16) 89 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed; 90 else if (dc_lsize == 32) 91 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed; 92 } 93 94 static void (* r4k_blast_dcache)(void); 95 96 static inline void r4k_blast_dcache_setup(void) 97 { 98 unsigned long dc_lsize = cpu_dcache_line_size(); 99 100 if (dc_lsize == 16) 101 r4k_blast_dcache = blast_dcache16; 102 else if (dc_lsize == 32) 103 r4k_blast_dcache = blast_dcache32; 104 } 105 106 /* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */ 107 #define JUMP_TO_ALIGN(order) \ 108 __asm__ __volatile__( \ 109 "b\t1f\n\t" \ 110 ".align\t" #order "\n\t" \ 111 "1:\n\t" \ 112 ) 113 #define CACHE32_UNROLL32_ALIGN JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */ 114 #define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11) 115 116 static inline void blast_r4600_v1_icache32(void) 117 { 118 unsigned long flags; 119 120 local_irq_save(flags); 121 blast_icache32(); 122 local_irq_restore(flags); 123 } 124 125 static inline void tx49_blast_icache32(void) 126 { 127 unsigned long start = INDEX_BASE; 128 unsigned long end = start + current_cpu_data.icache.waysize; 129 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 130 unsigned long ws_end = current_cpu_data.icache.ways << 131 current_cpu_data.icache.waybit; 132 unsigned long ws, addr; 133 134 CACHE32_UNROLL32_ALIGN2; 135 /* I'm in even chunk. blast odd chunks */ 136 for (ws = 0; ws < ws_end; ws += ws_inc) 137 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 138 cache32_unroll32(addr|ws,Index_Invalidate_I); 139 CACHE32_UNROLL32_ALIGN; 140 /* I'm in odd chunk. blast even chunks */ 141 for (ws = 0; ws < ws_end; ws += ws_inc) 142 for (addr = start; addr < end; addr += 0x400 * 2) 143 cache32_unroll32(addr|ws,Index_Invalidate_I); 144 } 145 146 static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page) 147 { 148 unsigned long flags; 149 150 local_irq_save(flags); 151 blast_icache32_page_indexed(page); 152 local_irq_restore(flags); 153 } 154 155 static inline void tx49_blast_icache32_page_indexed(unsigned long page) 156 { 157 unsigned long start = page; 158 unsigned long end = start + PAGE_SIZE; 159 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 160 unsigned long ws_end = current_cpu_data.icache.ways << 161 current_cpu_data.icache.waybit; 162 unsigned long ws, addr; 163 164 CACHE32_UNROLL32_ALIGN2; 165 /* I'm in even chunk. blast odd chunks */ 166 for (ws = 0; ws < ws_end; ws += ws_inc) 167 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 168 cache32_unroll32(addr|ws,Index_Invalidate_I); 169 CACHE32_UNROLL32_ALIGN; 170 /* I'm in odd chunk. blast even chunks */ 171 for (ws = 0; ws < ws_end; ws += ws_inc) 172 for (addr = start; addr < end; addr += 0x400 * 2) 173 cache32_unroll32(addr|ws,Index_Invalidate_I); 174 } 175 176 static void (* r4k_blast_icache_page)(unsigned long addr); 177 178 static inline void r4k_blast_icache_page_setup(void) 179 { 180 unsigned long ic_lsize = cpu_icache_line_size(); 181 182 if (ic_lsize == 16) 183 r4k_blast_icache_page = blast_icache16_page; 184 else if (ic_lsize == 32) 185 r4k_blast_icache_page = blast_icache32_page; 186 else if (ic_lsize == 64) 187 r4k_blast_icache_page = blast_icache64_page; 188 } 189 190 191 static void (* r4k_blast_icache_page_indexed)(unsigned long addr); 192 193 static inline void r4k_blast_icache_page_indexed_setup(void) 194 { 195 unsigned long ic_lsize = cpu_icache_line_size(); 196 197 if (ic_lsize == 16) 198 r4k_blast_icache_page_indexed = blast_icache16_page_indexed; 199 else if (ic_lsize == 32) { 200 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 201 r4k_blast_icache_page_indexed = 202 blast_icache32_r4600_v1_page_indexed; 203 else if (TX49XX_ICACHE_INDEX_INV_WAR) 204 r4k_blast_icache_page_indexed = 205 tx49_blast_icache32_page_indexed; 206 else 207 r4k_blast_icache_page_indexed = 208 blast_icache32_page_indexed; 209 } else if (ic_lsize == 64) 210 r4k_blast_icache_page_indexed = blast_icache64_page_indexed; 211 } 212 213 static void (* r4k_blast_icache)(void); 214 215 static inline void r4k_blast_icache_setup(void) 216 { 217 unsigned long ic_lsize = cpu_icache_line_size(); 218 219 if (ic_lsize == 16) 220 r4k_blast_icache = blast_icache16; 221 else if (ic_lsize == 32) { 222 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 223 r4k_blast_icache = blast_r4600_v1_icache32; 224 else if (TX49XX_ICACHE_INDEX_INV_WAR) 225 r4k_blast_icache = tx49_blast_icache32; 226 else 227 r4k_blast_icache = blast_icache32; 228 } else if (ic_lsize == 64) 229 r4k_blast_icache = blast_icache64; 230 } 231 232 static void (* r4k_blast_scache_page)(unsigned long addr); 233 234 static inline void r4k_blast_scache_page_setup(void) 235 { 236 unsigned long sc_lsize = cpu_scache_line_size(); 237 238 if (sc_lsize == 16) 239 r4k_blast_scache_page = blast_scache16_page; 240 else if (sc_lsize == 32) 241 r4k_blast_scache_page = blast_scache32_page; 242 else if (sc_lsize == 64) 243 r4k_blast_scache_page = blast_scache64_page; 244 else if (sc_lsize == 128) 245 r4k_blast_scache_page = blast_scache128_page; 246 } 247 248 static void (* r4k_blast_scache_page_indexed)(unsigned long addr); 249 250 static inline void r4k_blast_scache_page_indexed_setup(void) 251 { 252 unsigned long sc_lsize = cpu_scache_line_size(); 253 254 if (sc_lsize == 16) 255 r4k_blast_scache_page_indexed = blast_scache16_page_indexed; 256 else if (sc_lsize == 32) 257 r4k_blast_scache_page_indexed = blast_scache32_page_indexed; 258 else if (sc_lsize == 64) 259 r4k_blast_scache_page_indexed = blast_scache64_page_indexed; 260 else if (sc_lsize == 128) 261 r4k_blast_scache_page_indexed = blast_scache128_page_indexed; 262 } 263 264 static void (* r4k_blast_scache)(void); 265 266 static inline void r4k_blast_scache_setup(void) 267 { 268 unsigned long sc_lsize = cpu_scache_line_size(); 269 270 if (sc_lsize == 16) 271 r4k_blast_scache = blast_scache16; 272 else if (sc_lsize == 32) 273 r4k_blast_scache = blast_scache32; 274 else if (sc_lsize == 64) 275 r4k_blast_scache = blast_scache64; 276 else if (sc_lsize == 128) 277 r4k_blast_scache = blast_scache128; 278 } 279 280 /* 281 * This is former mm's flush_cache_all() which really should be 282 * flush_cache_vunmap these days ... 283 */ 284 static inline void local_r4k_flush_cache_all(void * args) 285 { 286 r4k_blast_dcache(); 287 r4k_blast_icache(); 288 } 289 290 static void r4k_flush_cache_all(void) 291 { 292 if (!cpu_has_dc_aliases) 293 return; 294 295 on_each_cpu(local_r4k_flush_cache_all, NULL, 1, 1); 296 } 297 298 static inline void local_r4k___flush_cache_all(void * args) 299 { 300 r4k_blast_dcache(); 301 r4k_blast_icache(); 302 303 switch (current_cpu_data.cputype) { 304 case CPU_R4000SC: 305 case CPU_R4000MC: 306 case CPU_R4400SC: 307 case CPU_R4400MC: 308 case CPU_R10000: 309 case CPU_R12000: 310 r4k_blast_scache(); 311 } 312 } 313 314 static void r4k___flush_cache_all(void) 315 { 316 on_each_cpu(local_r4k___flush_cache_all, NULL, 1, 1); 317 } 318 319 static inline void local_r4k_flush_cache_range(void * args) 320 { 321 struct vm_area_struct *vma = args; 322 int exec; 323 324 if (!(cpu_context(smp_processor_id(), vma->vm_mm))) 325 return; 326 327 exec = vma->vm_flags & VM_EXEC; 328 if (cpu_has_dc_aliases || exec) 329 r4k_blast_dcache(); 330 if (exec) 331 r4k_blast_icache(); 332 } 333 334 static void r4k_flush_cache_range(struct vm_area_struct *vma, 335 unsigned long start, unsigned long end) 336 { 337 on_each_cpu(local_r4k_flush_cache_range, vma, 1, 1); 338 } 339 340 static inline void local_r4k_flush_cache_mm(void * args) 341 { 342 struct mm_struct *mm = args; 343 344 if (!cpu_context(smp_processor_id(), mm)) 345 return; 346 347 r4k_blast_dcache(); 348 r4k_blast_icache(); 349 350 /* 351 * Kludge alert. For obscure reasons R4000SC and R4400SC go nuts if we 352 * only flush the primary caches but R10000 and R12000 behave sane ... 353 */ 354 if (current_cpu_data.cputype == CPU_R4000SC || 355 current_cpu_data.cputype == CPU_R4000MC || 356 current_cpu_data.cputype == CPU_R4400SC || 357 current_cpu_data.cputype == CPU_R4400MC) 358 r4k_blast_scache(); 359 } 360 361 static void r4k_flush_cache_mm(struct mm_struct *mm) 362 { 363 if (!cpu_has_dc_aliases) 364 return; 365 366 on_each_cpu(local_r4k_flush_cache_mm, mm, 1, 1); 367 } 368 369 struct flush_cache_page_args { 370 struct vm_area_struct *vma; 371 unsigned long addr; 372 }; 373 374 static inline void local_r4k_flush_cache_page(void *args) 375 { 376 struct flush_cache_page_args *fcp_args = args; 377 struct vm_area_struct *vma = fcp_args->vma; 378 unsigned long addr = fcp_args->addr; 379 int exec = vma->vm_flags & VM_EXEC; 380 struct mm_struct *mm = vma->vm_mm; 381 pgd_t *pgdp; 382 pud_t *pudp; 383 pmd_t *pmdp; 384 pte_t *ptep; 385 386 /* 387 * If ownes no valid ASID yet, cannot possibly have gotten 388 * this page into the cache. 389 */ 390 if (cpu_context(smp_processor_id(), mm) == 0) 391 return; 392 393 addr &= PAGE_MASK; 394 pgdp = pgd_offset(mm, addr); 395 pudp = pud_offset(pgdp, addr); 396 pmdp = pmd_offset(pudp, addr); 397 ptep = pte_offset(pmdp, addr); 398 399 /* 400 * If the page isn't marked valid, the page cannot possibly be 401 * in the cache. 402 */ 403 if (!(pte_val(*ptep) & _PAGE_PRESENT)) 404 return; 405 406 /* 407 * Doing flushes for another ASID than the current one is 408 * too difficult since stupid R4k caches do a TLB translation 409 * for every cache flush operation. So we do indexed flushes 410 * in that case, which doesn't overly flush the cache too much. 411 */ 412 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) { 413 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { 414 r4k_blast_dcache_page(addr); 415 if (exec && !cpu_icache_snoops_remote_store) 416 r4k_blast_scache_page(addr); 417 } 418 if (exec) 419 r4k_blast_icache_page(addr); 420 421 return; 422 } 423 424 /* 425 * Do indexed flush, too much work to get the (possible) TLB refills 426 * to work correctly. 427 */ 428 addr = INDEX_BASE + (addr & (dcache_size - 1)); 429 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { 430 r4k_blast_dcache_page_indexed(addr); 431 if (exec && !cpu_icache_snoops_remote_store) 432 r4k_blast_scache_page_indexed(addr); 433 } 434 if (exec) { 435 if (cpu_has_vtag_icache) { 436 int cpu = smp_processor_id(); 437 438 if (cpu_context(cpu, mm) != 0) 439 drop_mmu_context(mm, cpu); 440 } else 441 r4k_blast_icache_page_indexed(addr); 442 } 443 } 444 445 static void r4k_flush_cache_page(struct vm_area_struct *vma, 446 unsigned long addr, unsigned long pfn) 447 { 448 struct flush_cache_page_args args; 449 450 args.vma = vma; 451 args.addr = addr; 452 453 on_each_cpu(local_r4k_flush_cache_page, &args, 1, 1); 454 } 455 456 static inline void local_r4k_flush_data_cache_page(void * addr) 457 { 458 r4k_blast_dcache_page((unsigned long) addr); 459 } 460 461 static void r4k_flush_data_cache_page(unsigned long addr) 462 { 463 on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr, 1, 1); 464 } 465 466 struct flush_icache_range_args { 467 unsigned long __user start; 468 unsigned long __user end; 469 }; 470 471 static inline void local_r4k_flush_icache_range(void *args) 472 { 473 struct flush_icache_range_args *fir_args = args; 474 unsigned long dc_lsize = cpu_dcache_line_size(); 475 unsigned long ic_lsize = cpu_icache_line_size(); 476 unsigned long sc_lsize = cpu_scache_line_size(); 477 unsigned long start = fir_args->start; 478 unsigned long end = fir_args->end; 479 unsigned long addr, aend; 480 481 if (!cpu_has_ic_fills_f_dc) { 482 if (end - start > dcache_size) { 483 r4k_blast_dcache(); 484 } else { 485 R4600_HIT_CACHEOP_WAR_IMPL; 486 addr = start & ~(dc_lsize - 1); 487 aend = (end - 1) & ~(dc_lsize - 1); 488 489 while (1) { 490 /* Hit_Writeback_Inv_D */ 491 protected_writeback_dcache_line(addr); 492 if (addr == aend) 493 break; 494 addr += dc_lsize; 495 } 496 } 497 498 if (!cpu_icache_snoops_remote_store) { 499 if (end - start > scache_size) { 500 r4k_blast_scache(); 501 } else { 502 addr = start & ~(sc_lsize - 1); 503 aend = (end - 1) & ~(sc_lsize - 1); 504 505 while (1) { 506 /* Hit_Writeback_Inv_SD */ 507 protected_writeback_scache_line(addr); 508 if (addr == aend) 509 break; 510 addr += sc_lsize; 511 } 512 } 513 } 514 } 515 516 if (end - start > icache_size) 517 r4k_blast_icache(); 518 else { 519 addr = start & ~(ic_lsize - 1); 520 aend = (end - 1) & ~(ic_lsize - 1); 521 while (1) { 522 /* Hit_Invalidate_I */ 523 protected_flush_icache_line(addr); 524 if (addr == aend) 525 break; 526 addr += ic_lsize; 527 } 528 } 529 } 530 531 static void r4k_flush_icache_range(unsigned long __user start, 532 unsigned long __user end) 533 { 534 struct flush_icache_range_args args; 535 536 args.start = start; 537 args.end = end; 538 539 on_each_cpu(local_r4k_flush_icache_range, &args, 1, 1); 540 instruction_hazard(); 541 } 542 543 /* 544 * Ok, this seriously sucks. We use them to flush a user page but don't 545 * know the virtual address, so we have to blast away the whole icache 546 * which is significantly more expensive than the real thing. Otoh we at 547 * least know the kernel address of the page so we can flush it 548 * selectivly. 549 */ 550 551 struct flush_icache_page_args { 552 struct vm_area_struct *vma; 553 struct page *page; 554 }; 555 556 static inline void local_r4k_flush_icache_page(void *args) 557 { 558 struct flush_icache_page_args *fip_args = args; 559 struct vm_area_struct *vma = fip_args->vma; 560 struct page *page = fip_args->page; 561 562 /* 563 * Tricky ... Because we don't know the virtual address we've got the 564 * choice of either invalidating the entire primary and secondary 565 * caches or invalidating the secondary caches also. With the subset 566 * enforcment on R4000SC, R4400SC, R10000 and R12000 invalidating the 567 * secondary cache will result in any entries in the primary caches 568 * also getting invalidated which hopefully is a bit more economical. 569 */ 570 if (cpu_has_subset_pcaches) { 571 unsigned long addr = (unsigned long) page_address(page); 572 573 r4k_blast_scache_page(addr); 574 ClearPageDcacheDirty(page); 575 576 return; 577 } 578 579 if (!cpu_has_ic_fills_f_dc) { 580 unsigned long addr = (unsigned long) page_address(page); 581 r4k_blast_dcache_page(addr); 582 if (!cpu_icache_snoops_remote_store) 583 r4k_blast_scache_page(addr); 584 ClearPageDcacheDirty(page); 585 } 586 587 /* 588 * We're not sure of the virtual address(es) involved here, so 589 * we have to flush the entire I-cache. 590 */ 591 if (cpu_has_vtag_icache) { 592 int cpu = smp_processor_id(); 593 594 if (cpu_context(cpu, vma->vm_mm) != 0) 595 drop_mmu_context(vma->vm_mm, cpu); 596 } else 597 r4k_blast_icache(); 598 } 599 600 static void r4k_flush_icache_page(struct vm_area_struct *vma, 601 struct page *page) 602 { 603 struct flush_icache_page_args args; 604 605 /* 606 * If there's no context yet, or the page isn't executable, no I-cache 607 * flush is needed. 608 */ 609 if (!(vma->vm_flags & VM_EXEC)) 610 return; 611 612 args.vma = vma; 613 args.page = page; 614 615 on_each_cpu(local_r4k_flush_icache_page, &args, 1, 1); 616 } 617 618 619 #ifdef CONFIG_DMA_NONCOHERENT 620 621 static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size) 622 { 623 unsigned long end, a; 624 625 /* Catch bad driver code */ 626 BUG_ON(size == 0); 627 628 if (cpu_has_subset_pcaches) { 629 unsigned long sc_lsize = cpu_scache_line_size(); 630 631 if (size >= scache_size) { 632 r4k_blast_scache(); 633 return; 634 } 635 636 a = addr & ~(sc_lsize - 1); 637 end = (addr + size - 1) & ~(sc_lsize - 1); 638 while (1) { 639 flush_scache_line(a); /* Hit_Writeback_Inv_SD */ 640 if (a == end) 641 break; 642 a += sc_lsize; 643 } 644 return; 645 } 646 647 /* 648 * Either no secondary cache or the available caches don't have the 649 * subset property so we have to flush the primary caches 650 * explicitly 651 */ 652 if (size >= dcache_size) { 653 r4k_blast_dcache(); 654 } else { 655 unsigned long dc_lsize = cpu_dcache_line_size(); 656 657 R4600_HIT_CACHEOP_WAR_IMPL; 658 a = addr & ~(dc_lsize - 1); 659 end = (addr + size - 1) & ~(dc_lsize - 1); 660 while (1) { 661 flush_dcache_line(a); /* Hit_Writeback_Inv_D */ 662 if (a == end) 663 break; 664 a += dc_lsize; 665 } 666 } 667 668 bc_wback_inv(addr, size); 669 } 670 671 static void r4k_dma_cache_inv(unsigned long addr, unsigned long size) 672 { 673 unsigned long end, a; 674 675 /* Catch bad driver code */ 676 BUG_ON(size == 0); 677 678 if (cpu_has_subset_pcaches) { 679 unsigned long sc_lsize = cpu_scache_line_size(); 680 681 if (size >= scache_size) { 682 r4k_blast_scache(); 683 return; 684 } 685 686 a = addr & ~(sc_lsize - 1); 687 end = (addr + size - 1) & ~(sc_lsize - 1); 688 while (1) { 689 flush_scache_line(a); /* Hit_Writeback_Inv_SD */ 690 if (a == end) 691 break; 692 a += sc_lsize; 693 } 694 return; 695 } 696 697 if (size >= dcache_size) { 698 r4k_blast_dcache(); 699 } else { 700 unsigned long dc_lsize = cpu_dcache_line_size(); 701 702 R4600_HIT_CACHEOP_WAR_IMPL; 703 a = addr & ~(dc_lsize - 1); 704 end = (addr + size - 1) & ~(dc_lsize - 1); 705 while (1) { 706 flush_dcache_line(a); /* Hit_Writeback_Inv_D */ 707 if (a == end) 708 break; 709 a += dc_lsize; 710 } 711 } 712 713 bc_inv(addr, size); 714 } 715 #endif /* CONFIG_DMA_NONCOHERENT */ 716 717 /* 718 * While we're protected against bad userland addresses we don't care 719 * very much about what happens in that case. Usually a segmentation 720 * fault will dump the process later on anyway ... 721 */ 722 static void local_r4k_flush_cache_sigtramp(void * arg) 723 { 724 unsigned long ic_lsize = cpu_icache_line_size(); 725 unsigned long dc_lsize = cpu_dcache_line_size(); 726 unsigned long sc_lsize = cpu_scache_line_size(); 727 unsigned long addr = (unsigned long) arg; 728 729 R4600_HIT_CACHEOP_WAR_IMPL; 730 protected_writeback_dcache_line(addr & ~(dc_lsize - 1)); 731 if (!cpu_icache_snoops_remote_store) 732 protected_writeback_scache_line(addr & ~(sc_lsize - 1)); 733 protected_flush_icache_line(addr & ~(ic_lsize - 1)); 734 if (MIPS4K_ICACHE_REFILL_WAR) { 735 __asm__ __volatile__ ( 736 ".set push\n\t" 737 ".set noat\n\t" 738 ".set mips3\n\t" 739 #ifdef CONFIG_32BIT 740 "la $at,1f\n\t" 741 #endif 742 #ifdef CONFIG_64BIT 743 "dla $at,1f\n\t" 744 #endif 745 "cache %0,($at)\n\t" 746 "nop; nop; nop\n" 747 "1:\n\t" 748 ".set pop" 749 : 750 : "i" (Hit_Invalidate_I)); 751 } 752 if (MIPS_CACHE_SYNC_WAR) 753 __asm__ __volatile__ ("sync"); 754 } 755 756 static void r4k_flush_cache_sigtramp(unsigned long addr) 757 { 758 on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr, 1, 1); 759 } 760 761 static void r4k_flush_icache_all(void) 762 { 763 if (cpu_has_vtag_icache) 764 r4k_blast_icache(); 765 } 766 767 static inline void rm7k_erratum31(void) 768 { 769 const unsigned long ic_lsize = 32; 770 unsigned long addr; 771 772 /* RM7000 erratum #31. The icache is screwed at startup. */ 773 write_c0_taglo(0); 774 write_c0_taghi(0); 775 776 for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) { 777 __asm__ __volatile__ ( 778 ".set push\n\t" 779 ".set noreorder\n\t" 780 ".set mips3\n\t" 781 "cache\t%1, 0(%0)\n\t" 782 "cache\t%1, 0x1000(%0)\n\t" 783 "cache\t%1, 0x2000(%0)\n\t" 784 "cache\t%1, 0x3000(%0)\n\t" 785 "cache\t%2, 0(%0)\n\t" 786 "cache\t%2, 0x1000(%0)\n\t" 787 "cache\t%2, 0x2000(%0)\n\t" 788 "cache\t%2, 0x3000(%0)\n\t" 789 "cache\t%1, 0(%0)\n\t" 790 "cache\t%1, 0x1000(%0)\n\t" 791 "cache\t%1, 0x2000(%0)\n\t" 792 "cache\t%1, 0x3000(%0)\n\t" 793 ".set pop\n" 794 : 795 : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill)); 796 } 797 } 798 799 static char *way_string[] __initdata = { NULL, "direct mapped", "2-way", 800 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way" 801 }; 802 803 static void __init probe_pcache(void) 804 { 805 struct cpuinfo_mips *c = ¤t_cpu_data; 806 unsigned int config = read_c0_config(); 807 unsigned int prid = read_c0_prid(); 808 unsigned long config1; 809 unsigned int lsize; 810 811 switch (c->cputype) { 812 case CPU_R4600: /* QED style two way caches? */ 813 case CPU_R4700: 814 case CPU_R5000: 815 case CPU_NEVADA: 816 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 817 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 818 c->icache.ways = 2; 819 c->icache.waybit = ffs(icache_size/2) - 1; 820 821 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 822 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 823 c->dcache.ways = 2; 824 c->dcache.waybit= ffs(dcache_size/2) - 1; 825 826 c->options |= MIPS_CPU_CACHE_CDEX_P; 827 break; 828 829 case CPU_R5432: 830 case CPU_R5500: 831 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 832 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 833 c->icache.ways = 2; 834 c->icache.waybit= 0; 835 836 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 837 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 838 c->dcache.ways = 2; 839 c->dcache.waybit = 0; 840 841 c->options |= MIPS_CPU_CACHE_CDEX_P; 842 break; 843 844 case CPU_TX49XX: 845 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 846 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 847 c->icache.ways = 4; 848 c->icache.waybit= 0; 849 850 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 851 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 852 c->dcache.ways = 4; 853 c->dcache.waybit = 0; 854 855 c->options |= MIPS_CPU_CACHE_CDEX_P; 856 break; 857 858 case CPU_R4000PC: 859 case CPU_R4000SC: 860 case CPU_R4000MC: 861 case CPU_R4400PC: 862 case CPU_R4400SC: 863 case CPU_R4400MC: 864 case CPU_R4300: 865 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 866 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 867 c->icache.ways = 1; 868 c->icache.waybit = 0; /* doesn't matter */ 869 870 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 871 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 872 c->dcache.ways = 1; 873 c->dcache.waybit = 0; /* does not matter */ 874 875 c->options |= MIPS_CPU_CACHE_CDEX_P; 876 break; 877 878 case CPU_R10000: 879 case CPU_R12000: 880 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29)); 881 c->icache.linesz = 64; 882 c->icache.ways = 2; 883 c->icache.waybit = 0; 884 885 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26)); 886 c->dcache.linesz = 32; 887 c->dcache.ways = 2; 888 c->dcache.waybit = 0; 889 890 c->options |= MIPS_CPU_PREFETCH; 891 break; 892 893 case CPU_VR4133: 894 write_c0_config(config & ~CONF_EB); 895 case CPU_VR4131: 896 /* Workaround for cache instruction bug of VR4131 */ 897 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U || 898 c->processor_id == 0x0c82U) { 899 config &= ~0x00000030U; 900 config |= 0x00410000U; 901 write_c0_config(config); 902 } 903 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 904 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 905 c->icache.ways = 2; 906 c->icache.waybit = ffs(icache_size/2) - 1; 907 908 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 909 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 910 c->dcache.ways = 2; 911 c->dcache.waybit = ffs(dcache_size/2) - 1; 912 913 c->options |= MIPS_CPU_CACHE_CDEX_P; 914 break; 915 916 case CPU_VR41XX: 917 case CPU_VR4111: 918 case CPU_VR4121: 919 case CPU_VR4122: 920 case CPU_VR4181: 921 case CPU_VR4181A: 922 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 923 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 924 c->icache.ways = 1; 925 c->icache.waybit = 0; /* doesn't matter */ 926 927 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 928 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 929 c->dcache.ways = 1; 930 c->dcache.waybit = 0; /* does not matter */ 931 932 c->options |= MIPS_CPU_CACHE_CDEX_P; 933 break; 934 935 case CPU_RM7000: 936 rm7k_erratum31(); 937 938 case CPU_RM9000: 939 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 940 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 941 c->icache.ways = 4; 942 c->icache.waybit = ffs(icache_size / c->icache.ways) - 1; 943 944 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 945 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 946 c->dcache.ways = 4; 947 c->dcache.waybit = ffs(dcache_size / c->dcache.ways) - 1; 948 949 #if !defined(CONFIG_SMP) || !defined(RM9000_CDEX_SMP_WAR) 950 c->options |= MIPS_CPU_CACHE_CDEX_P; 951 #endif 952 c->options |= MIPS_CPU_PREFETCH; 953 break; 954 955 default: 956 if (!(config & MIPS_CONF_M)) 957 panic("Don't know how to probe P-caches on this cpu."); 958 959 /* 960 * So we seem to be a MIPS32 or MIPS64 CPU 961 * So let's probe the I-cache ... 962 */ 963 config1 = read_c0_config1(); 964 965 if ((lsize = ((config1 >> 19) & 7))) 966 c->icache.linesz = 2 << lsize; 967 else 968 c->icache.linesz = lsize; 969 c->icache.sets = 64 << ((config1 >> 22) & 7); 970 c->icache.ways = 1 + ((config1 >> 16) & 7); 971 972 icache_size = c->icache.sets * 973 c->icache.ways * 974 c->icache.linesz; 975 c->icache.waybit = ffs(icache_size/c->icache.ways) - 1; 976 977 if (config & 0x8) /* VI bit */ 978 c->icache.flags |= MIPS_CACHE_VTAG; 979 980 /* 981 * Now probe the MIPS32 / MIPS64 data cache. 982 */ 983 c->dcache.flags = 0; 984 985 if ((lsize = ((config1 >> 10) & 7))) 986 c->dcache.linesz = 2 << lsize; 987 else 988 c->dcache.linesz= lsize; 989 c->dcache.sets = 64 << ((config1 >> 13) & 7); 990 c->dcache.ways = 1 + ((config1 >> 7) & 7); 991 992 dcache_size = c->dcache.sets * 993 c->dcache.ways * 994 c->dcache.linesz; 995 c->dcache.waybit = ffs(dcache_size/c->dcache.ways) - 1; 996 997 c->options |= MIPS_CPU_PREFETCH; 998 break; 999 } 1000 1001 /* 1002 * Processor configuration sanity check for the R4000SC erratum 1003 * #5. With page sizes larger than 32kB there is no possibility 1004 * to get a VCE exception anymore so we don't care about this 1005 * misconfiguration. The case is rather theoretical anyway; 1006 * presumably no vendor is shipping his hardware in the "bad" 1007 * configuration. 1008 */ 1009 if ((prid & 0xff00) == PRID_IMP_R4000 && (prid & 0xff) < 0x40 && 1010 !(config & CONF_SC) && c->icache.linesz != 16 && 1011 PAGE_SIZE <= 0x8000) 1012 panic("Improper R4000SC processor configuration detected"); 1013 1014 /* compute a couple of other cache variables */ 1015 c->icache.waysize = icache_size / c->icache.ways; 1016 c->dcache.waysize = dcache_size / c->dcache.ways; 1017 1018 c->icache.sets = icache_size / (c->icache.linesz * c->icache.ways); 1019 c->dcache.sets = dcache_size / (c->dcache.linesz * c->dcache.ways); 1020 1021 /* 1022 * R10000 and R12000 P-caches are odd in a positive way. They're 32kB 1023 * 2-way virtually indexed so normally would suffer from aliases. So 1024 * normally they'd suffer from aliases but magic in the hardware deals 1025 * with that for us so we don't need to take care ourselves. 1026 */ 1027 switch (c->cputype) { 1028 case CPU_20KC: 1029 case CPU_25KF: 1030 case CPU_R10000: 1031 case CPU_R12000: 1032 case CPU_SB1: 1033 break; 1034 case CPU_24K: 1035 if (!(read_c0_config7() & (1 << 16))) 1036 default: 1037 if (c->dcache.waysize > PAGE_SIZE) 1038 c->dcache.flags |= MIPS_CACHE_ALIASES; 1039 } 1040 1041 switch (c->cputype) { 1042 case CPU_20KC: 1043 /* 1044 * Some older 20Kc chips doesn't have the 'VI' bit in 1045 * the config register. 1046 */ 1047 c->icache.flags |= MIPS_CACHE_VTAG; 1048 break; 1049 1050 case CPU_AU1000: 1051 case CPU_AU1500: 1052 case CPU_AU1100: 1053 case CPU_AU1550: 1054 case CPU_AU1200: 1055 c->icache.flags |= MIPS_CACHE_IC_F_DC; 1056 break; 1057 } 1058 1059 printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n", 1060 icache_size >> 10, 1061 cpu_has_vtag_icache ? "virtually tagged" : "physically tagged", 1062 way_string[c->icache.ways], c->icache.linesz); 1063 1064 printk("Primary data cache %ldkB, %s, linesize %d bytes.\n", 1065 dcache_size >> 10, way_string[c->dcache.ways], c->dcache.linesz); 1066 } 1067 1068 /* 1069 * If you even _breathe_ on this function, look at the gcc output and make sure 1070 * it does not pop things on and off the stack for the cache sizing loop that 1071 * executes in KSEG1 space or else you will crash and burn badly. You have 1072 * been warned. 1073 */ 1074 static int __init probe_scache(void) 1075 { 1076 extern unsigned long stext; 1077 unsigned long flags, addr, begin, end, pow2; 1078 unsigned int config = read_c0_config(); 1079 struct cpuinfo_mips *c = ¤t_cpu_data; 1080 int tmp; 1081 1082 if (config & CONF_SC) 1083 return 0; 1084 1085 begin = (unsigned long) &stext; 1086 begin &= ~((4 * 1024 * 1024) - 1); 1087 end = begin + (4 * 1024 * 1024); 1088 1089 /* 1090 * This is such a bitch, you'd think they would make it easy to do 1091 * this. Away you daemons of stupidity! 1092 */ 1093 local_irq_save(flags); 1094 1095 /* Fill each size-multiple cache line with a valid tag. */ 1096 pow2 = (64 * 1024); 1097 for (addr = begin; addr < end; addr = (begin + pow2)) { 1098 unsigned long *p = (unsigned long *) addr; 1099 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */ 1100 pow2 <<= 1; 1101 } 1102 1103 /* Load first line with zero (therefore invalid) tag. */ 1104 write_c0_taglo(0); 1105 write_c0_taghi(0); 1106 __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */ 1107 cache_op(Index_Store_Tag_I, begin); 1108 cache_op(Index_Store_Tag_D, begin); 1109 cache_op(Index_Store_Tag_SD, begin); 1110 1111 /* Now search for the wrap around point. */ 1112 pow2 = (128 * 1024); 1113 tmp = 0; 1114 for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) { 1115 cache_op(Index_Load_Tag_SD, addr); 1116 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */ 1117 if (!read_c0_taglo()) 1118 break; 1119 pow2 <<= 1; 1120 } 1121 local_irq_restore(flags); 1122 addr -= begin; 1123 1124 scache_size = addr; 1125 c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22); 1126 c->scache.ways = 1; 1127 c->dcache.waybit = 0; /* does not matter */ 1128 1129 return 1; 1130 } 1131 1132 extern int r5k_sc_init(void); 1133 extern int rm7k_sc_init(void); 1134 1135 static void __init setup_scache(void) 1136 { 1137 struct cpuinfo_mips *c = ¤t_cpu_data; 1138 unsigned int config = read_c0_config(); 1139 int sc_present = 0; 1140 1141 /* 1142 * Do the probing thing on R4000SC and R4400SC processors. Other 1143 * processors don't have a S-cache that would be relevant to the 1144 * Linux memory managment. 1145 */ 1146 switch (c->cputype) { 1147 case CPU_R4000SC: 1148 case CPU_R4000MC: 1149 case CPU_R4400SC: 1150 case CPU_R4400MC: 1151 sc_present = run_uncached(probe_scache); 1152 if (sc_present) 1153 c->options |= MIPS_CPU_CACHE_CDEX_S; 1154 break; 1155 1156 case CPU_R10000: 1157 case CPU_R12000: 1158 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16); 1159 c->scache.linesz = 64 << ((config >> 13) & 1); 1160 c->scache.ways = 2; 1161 c->scache.waybit= 0; 1162 sc_present = 1; 1163 break; 1164 1165 case CPU_R5000: 1166 case CPU_NEVADA: 1167 #ifdef CONFIG_R5000_CPU_SCACHE 1168 r5k_sc_init(); 1169 #endif 1170 return; 1171 1172 case CPU_RM7000: 1173 case CPU_RM9000: 1174 #ifdef CONFIG_RM7000_CPU_SCACHE 1175 rm7k_sc_init(); 1176 #endif 1177 return; 1178 1179 default: 1180 sc_present = 0; 1181 } 1182 1183 if (!sc_present) 1184 return; 1185 1186 if ((c->isa_level == MIPS_CPU_ISA_M32 || 1187 c->isa_level == MIPS_CPU_ISA_M64) && 1188 !(c->scache.flags & MIPS_CACHE_NOT_PRESENT)) 1189 panic("Dunno how to handle MIPS32 / MIPS64 second level cache"); 1190 1191 /* compute a couple of other cache variables */ 1192 c->scache.waysize = scache_size / c->scache.ways; 1193 1194 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways); 1195 1196 printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n", 1197 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); 1198 1199 c->options |= MIPS_CPU_SUBSET_CACHES; 1200 } 1201 1202 static inline void coherency_setup(void) 1203 { 1204 change_c0_config(CONF_CM_CMASK, CONF_CM_DEFAULT); 1205 1206 /* 1207 * c0_status.cu=0 specifies that updates by the sc instruction use 1208 * the coherency mode specified by the TLB; 1 means cachable 1209 * coherent update on write will be used. Not all processors have 1210 * this bit and; some wire it to zero, others like Toshiba had the 1211 * silly idea of putting something else there ... 1212 */ 1213 switch (current_cpu_data.cputype) { 1214 case CPU_R4000PC: 1215 case CPU_R4000SC: 1216 case CPU_R4000MC: 1217 case CPU_R4400PC: 1218 case CPU_R4400SC: 1219 case CPU_R4400MC: 1220 clear_c0_config(CONF_CU); 1221 break; 1222 } 1223 } 1224 1225 void __init r4k_cache_init(void) 1226 { 1227 extern void build_clear_page(void); 1228 extern void build_copy_page(void); 1229 extern char except_vec2_generic; 1230 struct cpuinfo_mips *c = ¤t_cpu_data; 1231 1232 /* Default cache error handler for R4000 and R5000 family */ 1233 set_uncached_handler (0x100, &except_vec2_generic, 0x80); 1234 1235 probe_pcache(); 1236 setup_scache(); 1237 1238 r4k_blast_dcache_page_setup(); 1239 r4k_blast_dcache_page_indexed_setup(); 1240 r4k_blast_dcache_setup(); 1241 r4k_blast_icache_page_setup(); 1242 r4k_blast_icache_page_indexed_setup(); 1243 r4k_blast_icache_setup(); 1244 r4k_blast_scache_page_setup(); 1245 r4k_blast_scache_page_indexed_setup(); 1246 r4k_blast_scache_setup(); 1247 1248 /* 1249 * Some MIPS32 and MIPS64 processors have physically indexed caches. 1250 * This code supports virtually indexed processors and will be 1251 * unnecessarily inefficient on physically indexed processors. 1252 */ 1253 shm_align_mask = max_t( unsigned long, 1254 c->dcache.sets * c->dcache.linesz - 1, 1255 PAGE_SIZE - 1); 1256 1257 flush_cache_all = r4k_flush_cache_all; 1258 __flush_cache_all = r4k___flush_cache_all; 1259 flush_cache_mm = r4k_flush_cache_mm; 1260 flush_cache_page = r4k_flush_cache_page; 1261 flush_icache_page = r4k_flush_icache_page; 1262 flush_cache_range = r4k_flush_cache_range; 1263 1264 flush_cache_sigtramp = r4k_flush_cache_sigtramp; 1265 flush_icache_all = r4k_flush_icache_all; 1266 flush_data_cache_page = r4k_flush_data_cache_page; 1267 flush_icache_range = r4k_flush_icache_range; 1268 1269 #ifdef CONFIG_DMA_NONCOHERENT 1270 _dma_cache_wback_inv = r4k_dma_cache_wback_inv; 1271 _dma_cache_wback = r4k_dma_cache_wback_inv; 1272 _dma_cache_inv = r4k_dma_cache_inv; 1273 #endif 1274 1275 build_clear_page(); 1276 build_copy_page(); 1277 local_r4k___flush_cache_all(NULL); 1278 coherency_setup(); 1279 } 1280