1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) 7 * Copyright (C) 1997, 1998, 1999, 2000, 2001, 2002 Ralf Baechle (ralf@gnu.org) 8 * Copyright (C) 1999, 2000 Silicon Graphics, Inc. 9 */ 10 #include <linux/config.h> 11 #include <linux/init.h> 12 #include <linux/kernel.h> 13 #include <linux/sched.h> 14 #include <linux/mm.h> 15 #include <linux/bitops.h> 16 17 #include <asm/bcache.h> 18 #include <asm/bootinfo.h> 19 #include <asm/cache.h> 20 #include <asm/cacheops.h> 21 #include <asm/cpu.h> 22 #include <asm/cpu-features.h> 23 #include <asm/io.h> 24 #include <asm/page.h> 25 #include <asm/pgtable.h> 26 #include <asm/r4kcache.h> 27 #include <asm/system.h> 28 #include <asm/mmu_context.h> 29 #include <asm/war.h> 30 #include <asm/cacheflush.h> /* for run_uncached() */ 31 32 33 /* 34 * Special Variant of smp_call_function for use by cache functions: 35 * 36 * o No return value 37 * o collapses to normal function call on UP kernels 38 * o collapses to normal function call on systems with a single shared 39 * primary cache. 40 */ 41 static inline void r4k_on_each_cpu(void (*func) (void *info), void *info, 42 int retry, int wait) 43 { 44 preempt_disable(); 45 46 #if !defined(CONFIG_MIPS_MT_SMP) && !defined(CONFIG_MIPS_MT_SMTC) 47 smp_call_function(func, info, retry, wait); 48 #endif 49 func(info); 50 preempt_enable(); 51 } 52 53 /* 54 * Must die. 55 */ 56 static unsigned long icache_size __read_mostly; 57 static unsigned long dcache_size __read_mostly; 58 static unsigned long scache_size __read_mostly; 59 60 /* 61 * Dummy cache handling routines for machines without boardcaches 62 */ 63 static void cache_noop(void) {} 64 65 static struct bcache_ops no_sc_ops = { 66 .bc_enable = (void *)cache_noop, 67 .bc_disable = (void *)cache_noop, 68 .bc_wback_inv = (void *)cache_noop, 69 .bc_inv = (void *)cache_noop 70 }; 71 72 struct bcache_ops *bcops = &no_sc_ops; 73 74 #define cpu_is_r4600_v1_x() ((read_c0_prid() & 0xfffffff0) == 0x00002010) 75 #define cpu_is_r4600_v2_x() ((read_c0_prid() & 0xfffffff0) == 0x00002020) 76 77 #define R4600_HIT_CACHEOP_WAR_IMPL \ 78 do { \ 79 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) \ 80 *(volatile unsigned long *)CKSEG1; \ 81 if (R4600_V1_HIT_CACHEOP_WAR) \ 82 __asm__ __volatile__("nop;nop;nop;nop"); \ 83 } while (0) 84 85 static void (*r4k_blast_dcache_page)(unsigned long addr); 86 87 static inline void r4k_blast_dcache_page_dc32(unsigned long addr) 88 { 89 R4600_HIT_CACHEOP_WAR_IMPL; 90 blast_dcache32_page(addr); 91 } 92 93 static inline void r4k_blast_dcache_page_setup(void) 94 { 95 unsigned long dc_lsize = cpu_dcache_line_size(); 96 97 if (dc_lsize == 0) 98 r4k_blast_dcache_page = (void *)cache_noop; 99 else if (dc_lsize == 16) 100 r4k_blast_dcache_page = blast_dcache16_page; 101 else if (dc_lsize == 32) 102 r4k_blast_dcache_page = r4k_blast_dcache_page_dc32; 103 } 104 105 static void (* r4k_blast_dcache_page_indexed)(unsigned long addr); 106 107 static inline void r4k_blast_dcache_page_indexed_setup(void) 108 { 109 unsigned long dc_lsize = cpu_dcache_line_size(); 110 111 if (dc_lsize == 0) 112 r4k_blast_dcache_page_indexed = (void *)cache_noop; 113 else if (dc_lsize == 16) 114 r4k_blast_dcache_page_indexed = blast_dcache16_page_indexed; 115 else if (dc_lsize == 32) 116 r4k_blast_dcache_page_indexed = blast_dcache32_page_indexed; 117 } 118 119 static void (* r4k_blast_dcache)(void); 120 121 static inline void r4k_blast_dcache_setup(void) 122 { 123 unsigned long dc_lsize = cpu_dcache_line_size(); 124 125 if (dc_lsize == 0) 126 r4k_blast_dcache = (void *)cache_noop; 127 else if (dc_lsize == 16) 128 r4k_blast_dcache = blast_dcache16; 129 else if (dc_lsize == 32) 130 r4k_blast_dcache = blast_dcache32; 131 } 132 133 /* force code alignment (used for TX49XX_ICACHE_INDEX_INV_WAR) */ 134 #define JUMP_TO_ALIGN(order) \ 135 __asm__ __volatile__( \ 136 "b\t1f\n\t" \ 137 ".align\t" #order "\n\t" \ 138 "1:\n\t" \ 139 ) 140 #define CACHE32_UNROLL32_ALIGN JUMP_TO_ALIGN(10) /* 32 * 32 = 1024 */ 141 #define CACHE32_UNROLL32_ALIGN2 JUMP_TO_ALIGN(11) 142 143 static inline void blast_r4600_v1_icache32(void) 144 { 145 unsigned long flags; 146 147 local_irq_save(flags); 148 blast_icache32(); 149 local_irq_restore(flags); 150 } 151 152 static inline void tx49_blast_icache32(void) 153 { 154 unsigned long start = INDEX_BASE; 155 unsigned long end = start + current_cpu_data.icache.waysize; 156 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 157 unsigned long ws_end = current_cpu_data.icache.ways << 158 current_cpu_data.icache.waybit; 159 unsigned long ws, addr; 160 161 CACHE32_UNROLL32_ALIGN2; 162 /* I'm in even chunk. blast odd chunks */ 163 for (ws = 0; ws < ws_end; ws += ws_inc) 164 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 165 cache32_unroll32(addr|ws,Index_Invalidate_I); 166 CACHE32_UNROLL32_ALIGN; 167 /* I'm in odd chunk. blast even chunks */ 168 for (ws = 0; ws < ws_end; ws += ws_inc) 169 for (addr = start; addr < end; addr += 0x400 * 2) 170 cache32_unroll32(addr|ws,Index_Invalidate_I); 171 } 172 173 static inline void blast_icache32_r4600_v1_page_indexed(unsigned long page) 174 { 175 unsigned long flags; 176 177 local_irq_save(flags); 178 blast_icache32_page_indexed(page); 179 local_irq_restore(flags); 180 } 181 182 static inline void tx49_blast_icache32_page_indexed(unsigned long page) 183 { 184 unsigned long indexmask = current_cpu_data.icache.waysize - 1; 185 unsigned long start = INDEX_BASE + (page & indexmask); 186 unsigned long end = start + PAGE_SIZE; 187 unsigned long ws_inc = 1UL << current_cpu_data.icache.waybit; 188 unsigned long ws_end = current_cpu_data.icache.ways << 189 current_cpu_data.icache.waybit; 190 unsigned long ws, addr; 191 192 CACHE32_UNROLL32_ALIGN2; 193 /* I'm in even chunk. blast odd chunks */ 194 for (ws = 0; ws < ws_end; ws += ws_inc) 195 for (addr = start + 0x400; addr < end; addr += 0x400 * 2) 196 cache32_unroll32(addr|ws,Index_Invalidate_I); 197 CACHE32_UNROLL32_ALIGN; 198 /* I'm in odd chunk. blast even chunks */ 199 for (ws = 0; ws < ws_end; ws += ws_inc) 200 for (addr = start; addr < end; addr += 0x400 * 2) 201 cache32_unroll32(addr|ws,Index_Invalidate_I); 202 } 203 204 static void (* r4k_blast_icache_page)(unsigned long addr); 205 206 static inline void r4k_blast_icache_page_setup(void) 207 { 208 unsigned long ic_lsize = cpu_icache_line_size(); 209 210 if (ic_lsize == 0) 211 r4k_blast_icache_page = (void *)cache_noop; 212 else if (ic_lsize == 16) 213 r4k_blast_icache_page = blast_icache16_page; 214 else if (ic_lsize == 32) 215 r4k_blast_icache_page = blast_icache32_page; 216 else if (ic_lsize == 64) 217 r4k_blast_icache_page = blast_icache64_page; 218 } 219 220 221 static void (* r4k_blast_icache_page_indexed)(unsigned long addr); 222 223 static inline void r4k_blast_icache_page_indexed_setup(void) 224 { 225 unsigned long ic_lsize = cpu_icache_line_size(); 226 227 if (ic_lsize == 0) 228 r4k_blast_icache_page_indexed = (void *)cache_noop; 229 else if (ic_lsize == 16) 230 r4k_blast_icache_page_indexed = blast_icache16_page_indexed; 231 else if (ic_lsize == 32) { 232 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 233 r4k_blast_icache_page_indexed = 234 blast_icache32_r4600_v1_page_indexed; 235 else if (TX49XX_ICACHE_INDEX_INV_WAR) 236 r4k_blast_icache_page_indexed = 237 tx49_blast_icache32_page_indexed; 238 else 239 r4k_blast_icache_page_indexed = 240 blast_icache32_page_indexed; 241 } else if (ic_lsize == 64) 242 r4k_blast_icache_page_indexed = blast_icache64_page_indexed; 243 } 244 245 static void (* r4k_blast_icache)(void); 246 247 static inline void r4k_blast_icache_setup(void) 248 { 249 unsigned long ic_lsize = cpu_icache_line_size(); 250 251 if (ic_lsize == 0) 252 r4k_blast_icache = (void *)cache_noop; 253 else if (ic_lsize == 16) 254 r4k_blast_icache = blast_icache16; 255 else if (ic_lsize == 32) { 256 if (R4600_V1_INDEX_ICACHEOP_WAR && cpu_is_r4600_v1_x()) 257 r4k_blast_icache = blast_r4600_v1_icache32; 258 else if (TX49XX_ICACHE_INDEX_INV_WAR) 259 r4k_blast_icache = tx49_blast_icache32; 260 else 261 r4k_blast_icache = blast_icache32; 262 } else if (ic_lsize == 64) 263 r4k_blast_icache = blast_icache64; 264 } 265 266 static void (* r4k_blast_scache_page)(unsigned long addr); 267 268 static inline void r4k_blast_scache_page_setup(void) 269 { 270 unsigned long sc_lsize = cpu_scache_line_size(); 271 272 if (scache_size == 0) 273 r4k_blast_scache_page = (void *)cache_noop; 274 else if (sc_lsize == 16) 275 r4k_blast_scache_page = blast_scache16_page; 276 else if (sc_lsize == 32) 277 r4k_blast_scache_page = blast_scache32_page; 278 else if (sc_lsize == 64) 279 r4k_blast_scache_page = blast_scache64_page; 280 else if (sc_lsize == 128) 281 r4k_blast_scache_page = blast_scache128_page; 282 } 283 284 static void (* r4k_blast_scache_page_indexed)(unsigned long addr); 285 286 static inline void r4k_blast_scache_page_indexed_setup(void) 287 { 288 unsigned long sc_lsize = cpu_scache_line_size(); 289 290 if (scache_size == 0) 291 r4k_blast_scache_page_indexed = (void *)cache_noop; 292 else if (sc_lsize == 16) 293 r4k_blast_scache_page_indexed = blast_scache16_page_indexed; 294 else if (sc_lsize == 32) 295 r4k_blast_scache_page_indexed = blast_scache32_page_indexed; 296 else if (sc_lsize == 64) 297 r4k_blast_scache_page_indexed = blast_scache64_page_indexed; 298 else if (sc_lsize == 128) 299 r4k_blast_scache_page_indexed = blast_scache128_page_indexed; 300 } 301 302 static void (* r4k_blast_scache)(void); 303 304 static inline void r4k_blast_scache_setup(void) 305 { 306 unsigned long sc_lsize = cpu_scache_line_size(); 307 308 if (scache_size == 0) 309 r4k_blast_scache = (void *)cache_noop; 310 else if (sc_lsize == 16) 311 r4k_blast_scache = blast_scache16; 312 else if (sc_lsize == 32) 313 r4k_blast_scache = blast_scache32; 314 else if (sc_lsize == 64) 315 r4k_blast_scache = blast_scache64; 316 else if (sc_lsize == 128) 317 r4k_blast_scache = blast_scache128; 318 } 319 320 /* 321 * This is former mm's flush_cache_all() which really should be 322 * flush_cache_vunmap these days ... 323 */ 324 static inline void local_r4k_flush_cache_all(void * args) 325 { 326 r4k_blast_dcache(); 327 r4k_blast_icache(); 328 } 329 330 static void r4k_flush_cache_all(void) 331 { 332 if (!cpu_has_dc_aliases) 333 return; 334 335 r4k_on_each_cpu(local_r4k_flush_cache_all, NULL, 1, 1); 336 } 337 338 static inline void local_r4k___flush_cache_all(void * args) 339 { 340 r4k_blast_dcache(); 341 r4k_blast_icache(); 342 343 switch (current_cpu_data.cputype) { 344 case CPU_R4000SC: 345 case CPU_R4000MC: 346 case CPU_R4400SC: 347 case CPU_R4400MC: 348 case CPU_R10000: 349 case CPU_R12000: 350 case CPU_R14000: 351 r4k_blast_scache(); 352 } 353 } 354 355 static void r4k___flush_cache_all(void) 356 { 357 r4k_on_each_cpu(local_r4k___flush_cache_all, NULL, 1, 1); 358 } 359 360 static inline void local_r4k_flush_cache_range(void * args) 361 { 362 struct vm_area_struct *vma = args; 363 int exec; 364 365 if (!(cpu_context(smp_processor_id(), vma->vm_mm))) 366 return; 367 368 exec = vma->vm_flags & VM_EXEC; 369 if (cpu_has_dc_aliases || exec) 370 r4k_blast_dcache(); 371 if (exec) 372 r4k_blast_icache(); 373 } 374 375 static void r4k_flush_cache_range(struct vm_area_struct *vma, 376 unsigned long start, unsigned long end) 377 { 378 r4k_on_each_cpu(local_r4k_flush_cache_range, vma, 1, 1); 379 } 380 381 static inline void local_r4k_flush_cache_mm(void * args) 382 { 383 struct mm_struct *mm = args; 384 385 if (!cpu_context(smp_processor_id(), mm)) 386 return; 387 388 r4k_blast_dcache(); 389 r4k_blast_icache(); 390 391 /* 392 * Kludge alert. For obscure reasons R4000SC and R4400SC go nuts if we 393 * only flush the primary caches but R10000 and R12000 behave sane ... 394 */ 395 if (current_cpu_data.cputype == CPU_R4000SC || 396 current_cpu_data.cputype == CPU_R4000MC || 397 current_cpu_data.cputype == CPU_R4400SC || 398 current_cpu_data.cputype == CPU_R4400MC) 399 r4k_blast_scache(); 400 } 401 402 static void r4k_flush_cache_mm(struct mm_struct *mm) 403 { 404 if (!cpu_has_dc_aliases) 405 return; 406 407 r4k_on_each_cpu(local_r4k_flush_cache_mm, mm, 1, 1); 408 } 409 410 struct flush_cache_page_args { 411 struct vm_area_struct *vma; 412 unsigned long addr; 413 unsigned long pfn; 414 }; 415 416 static inline void local_r4k_flush_cache_page(void *args) 417 { 418 struct flush_cache_page_args *fcp_args = args; 419 struct vm_area_struct *vma = fcp_args->vma; 420 unsigned long addr = fcp_args->addr; 421 unsigned long paddr = fcp_args->pfn << PAGE_SHIFT; 422 int exec = vma->vm_flags & VM_EXEC; 423 struct mm_struct *mm = vma->vm_mm; 424 pgd_t *pgdp; 425 pud_t *pudp; 426 pmd_t *pmdp; 427 pte_t *ptep; 428 429 /* 430 * If ownes no valid ASID yet, cannot possibly have gotten 431 * this page into the cache. 432 */ 433 if (cpu_context(smp_processor_id(), mm) == 0) 434 return; 435 436 addr &= PAGE_MASK; 437 pgdp = pgd_offset(mm, addr); 438 pudp = pud_offset(pgdp, addr); 439 pmdp = pmd_offset(pudp, addr); 440 ptep = pte_offset(pmdp, addr); 441 442 /* 443 * If the page isn't marked valid, the page cannot possibly be 444 * in the cache. 445 */ 446 if (!(pte_val(*ptep) & _PAGE_PRESENT)) 447 return; 448 449 /* 450 * Doing flushes for another ASID than the current one is 451 * too difficult since stupid R4k caches do a TLB translation 452 * for every cache flush operation. So we do indexed flushes 453 * in that case, which doesn't overly flush the cache too much. 454 */ 455 if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) { 456 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { 457 r4k_blast_dcache_page(addr); 458 if (exec && !cpu_icache_snoops_remote_store) 459 r4k_blast_scache_page(addr); 460 } 461 if (exec) 462 r4k_blast_icache_page(addr); 463 464 return; 465 } 466 467 /* 468 * Do indexed flush, too much work to get the (possible) TLB refills 469 * to work correctly. 470 */ 471 if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { 472 r4k_blast_dcache_page_indexed(cpu_has_pindexed_dcache ? 473 paddr : addr); 474 if (exec && !cpu_icache_snoops_remote_store) { 475 r4k_blast_scache_page_indexed(paddr); 476 } 477 } 478 if (exec) { 479 if (cpu_has_vtag_icache) { 480 int cpu = smp_processor_id(); 481 482 if (cpu_context(cpu, mm) != 0) 483 drop_mmu_context(mm, cpu); 484 } else 485 r4k_blast_icache_page_indexed(addr); 486 } 487 } 488 489 static void r4k_flush_cache_page(struct vm_area_struct *vma, 490 unsigned long addr, unsigned long pfn) 491 { 492 struct flush_cache_page_args args; 493 494 args.vma = vma; 495 args.addr = addr; 496 args.pfn = pfn; 497 498 r4k_on_each_cpu(local_r4k_flush_cache_page, &args, 1, 1); 499 } 500 501 static inline void local_r4k_flush_data_cache_page(void * addr) 502 { 503 r4k_blast_dcache_page((unsigned long) addr); 504 } 505 506 static void r4k_flush_data_cache_page(unsigned long addr) 507 { 508 r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr, 1, 1); 509 } 510 511 struct flush_icache_range_args { 512 unsigned long start; 513 unsigned long end; 514 }; 515 516 static inline void local_r4k_flush_icache_range(void *args) 517 { 518 struct flush_icache_range_args *fir_args = args; 519 unsigned long start = fir_args->start; 520 unsigned long end = fir_args->end; 521 522 if (!cpu_has_ic_fills_f_dc) { 523 if (end - start >= dcache_size) { 524 r4k_blast_dcache(); 525 } else { 526 R4600_HIT_CACHEOP_WAR_IMPL; 527 protected_blast_dcache_range(start, end); 528 } 529 530 if (!cpu_icache_snoops_remote_store && scache_size) { 531 if (end - start > scache_size) 532 r4k_blast_scache(); 533 else 534 protected_blast_scache_range(start, end); 535 } 536 } 537 538 if (end - start > icache_size) 539 r4k_blast_icache(); 540 else 541 protected_blast_icache_range(start, end); 542 } 543 544 static void r4k_flush_icache_range(unsigned long start, unsigned long end) 545 { 546 struct flush_icache_range_args args; 547 548 args.start = start; 549 args.end = end; 550 551 r4k_on_each_cpu(local_r4k_flush_icache_range, &args, 1, 1); 552 instruction_hazard(); 553 } 554 555 /* 556 * Ok, this seriously sucks. We use them to flush a user page but don't 557 * know the virtual address, so we have to blast away the whole icache 558 * which is significantly more expensive than the real thing. Otoh we at 559 * least know the kernel address of the page so we can flush it 560 * selectivly. 561 */ 562 563 struct flush_icache_page_args { 564 struct vm_area_struct *vma; 565 struct page *page; 566 }; 567 568 static inline void local_r4k_flush_icache_page(void *args) 569 { 570 struct flush_icache_page_args *fip_args = args; 571 struct vm_area_struct *vma = fip_args->vma; 572 struct page *page = fip_args->page; 573 574 /* 575 * Tricky ... Because we don't know the virtual address we've got the 576 * choice of either invalidating the entire primary and secondary 577 * caches or invalidating the secondary caches also. With the subset 578 * enforcment on R4000SC, R4400SC, R10000 and R12000 invalidating the 579 * secondary cache will result in any entries in the primary caches 580 * also getting invalidated which hopefully is a bit more economical. 581 */ 582 if (cpu_has_subset_pcaches) { 583 unsigned long addr = (unsigned long) page_address(page); 584 585 r4k_blast_scache_page(addr); 586 ClearPageDcacheDirty(page); 587 588 return; 589 } 590 591 if (!cpu_has_ic_fills_f_dc) { 592 unsigned long addr = (unsigned long) page_address(page); 593 r4k_blast_dcache_page(addr); 594 if (!cpu_icache_snoops_remote_store) 595 r4k_blast_scache_page(addr); 596 ClearPageDcacheDirty(page); 597 } 598 599 /* 600 * We're not sure of the virtual address(es) involved here, so 601 * we have to flush the entire I-cache. 602 */ 603 if (cpu_has_vtag_icache) { 604 int cpu = smp_processor_id(); 605 606 if (cpu_context(cpu, vma->vm_mm) != 0) 607 drop_mmu_context(vma->vm_mm, cpu); 608 } else 609 r4k_blast_icache(); 610 } 611 612 static void r4k_flush_icache_page(struct vm_area_struct *vma, 613 struct page *page) 614 { 615 struct flush_icache_page_args args; 616 617 /* 618 * If there's no context yet, or the page isn't executable, no I-cache 619 * flush is needed. 620 */ 621 if (!(vma->vm_flags & VM_EXEC)) 622 return; 623 624 args.vma = vma; 625 args.page = page; 626 627 r4k_on_each_cpu(local_r4k_flush_icache_page, &args, 1, 1); 628 } 629 630 631 #ifdef CONFIG_DMA_NONCOHERENT 632 633 static void r4k_dma_cache_wback_inv(unsigned long addr, unsigned long size) 634 { 635 /* Catch bad driver code */ 636 BUG_ON(size == 0); 637 638 if (cpu_has_subset_pcaches) { 639 if (size >= scache_size) 640 r4k_blast_scache(); 641 else 642 blast_scache_range(addr, addr + size); 643 return; 644 } 645 646 /* 647 * Either no secondary cache or the available caches don't have the 648 * subset property so we have to flush the primary caches 649 * explicitly 650 */ 651 if (size >= dcache_size) { 652 r4k_blast_dcache(); 653 } else { 654 R4600_HIT_CACHEOP_WAR_IMPL; 655 blast_dcache_range(addr, addr + size); 656 } 657 658 bc_wback_inv(addr, size); 659 } 660 661 static void r4k_dma_cache_inv(unsigned long addr, unsigned long size) 662 { 663 /* Catch bad driver code */ 664 BUG_ON(size == 0); 665 666 if (cpu_has_subset_pcaches) { 667 if (size >= scache_size) 668 r4k_blast_scache(); 669 else 670 blast_scache_range(addr, addr + size); 671 return; 672 } 673 674 if (size >= dcache_size) { 675 r4k_blast_dcache(); 676 } else { 677 R4600_HIT_CACHEOP_WAR_IMPL; 678 blast_dcache_range(addr, addr + size); 679 } 680 681 bc_inv(addr, size); 682 } 683 #endif /* CONFIG_DMA_NONCOHERENT */ 684 685 /* 686 * While we're protected against bad userland addresses we don't care 687 * very much about what happens in that case. Usually a segmentation 688 * fault will dump the process later on anyway ... 689 */ 690 static void local_r4k_flush_cache_sigtramp(void * arg) 691 { 692 unsigned long ic_lsize = cpu_icache_line_size(); 693 unsigned long dc_lsize = cpu_dcache_line_size(); 694 unsigned long sc_lsize = cpu_scache_line_size(); 695 unsigned long addr = (unsigned long) arg; 696 697 R4600_HIT_CACHEOP_WAR_IMPL; 698 if (dc_lsize) 699 protected_writeback_dcache_line(addr & ~(dc_lsize - 1)); 700 if (!cpu_icache_snoops_remote_store && scache_size) 701 protected_writeback_scache_line(addr & ~(sc_lsize - 1)); 702 if (ic_lsize) 703 protected_flush_icache_line(addr & ~(ic_lsize - 1)); 704 if (MIPS4K_ICACHE_REFILL_WAR) { 705 __asm__ __volatile__ ( 706 ".set push\n\t" 707 ".set noat\n\t" 708 ".set mips3\n\t" 709 #ifdef CONFIG_32BIT 710 "la $at,1f\n\t" 711 #endif 712 #ifdef CONFIG_64BIT 713 "dla $at,1f\n\t" 714 #endif 715 "cache %0,($at)\n\t" 716 "nop; nop; nop\n" 717 "1:\n\t" 718 ".set pop" 719 : 720 : "i" (Hit_Invalidate_I)); 721 } 722 if (MIPS_CACHE_SYNC_WAR) 723 __asm__ __volatile__ ("sync"); 724 } 725 726 static void r4k_flush_cache_sigtramp(unsigned long addr) 727 { 728 r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr, 1, 1); 729 } 730 731 static void r4k_flush_icache_all(void) 732 { 733 if (cpu_has_vtag_icache) 734 r4k_blast_icache(); 735 } 736 737 static inline void rm7k_erratum31(void) 738 { 739 const unsigned long ic_lsize = 32; 740 unsigned long addr; 741 742 /* RM7000 erratum #31. The icache is screwed at startup. */ 743 write_c0_taglo(0); 744 write_c0_taghi(0); 745 746 for (addr = INDEX_BASE; addr <= INDEX_BASE + 4096; addr += ic_lsize) { 747 __asm__ __volatile__ ( 748 ".set push\n\t" 749 ".set noreorder\n\t" 750 ".set mips3\n\t" 751 "cache\t%1, 0(%0)\n\t" 752 "cache\t%1, 0x1000(%0)\n\t" 753 "cache\t%1, 0x2000(%0)\n\t" 754 "cache\t%1, 0x3000(%0)\n\t" 755 "cache\t%2, 0(%0)\n\t" 756 "cache\t%2, 0x1000(%0)\n\t" 757 "cache\t%2, 0x2000(%0)\n\t" 758 "cache\t%2, 0x3000(%0)\n\t" 759 "cache\t%1, 0(%0)\n\t" 760 "cache\t%1, 0x1000(%0)\n\t" 761 "cache\t%1, 0x2000(%0)\n\t" 762 "cache\t%1, 0x3000(%0)\n\t" 763 ".set pop\n" 764 : 765 : "r" (addr), "i" (Index_Store_Tag_I), "i" (Fill)); 766 } 767 } 768 769 static char *way_string[] __initdata = { NULL, "direct mapped", "2-way", 770 "3-way", "4-way", "5-way", "6-way", "7-way", "8-way" 771 }; 772 773 static void __init probe_pcache(void) 774 { 775 struct cpuinfo_mips *c = ¤t_cpu_data; 776 unsigned int config = read_c0_config(); 777 unsigned int prid = read_c0_prid(); 778 unsigned long config1; 779 unsigned int lsize; 780 781 switch (c->cputype) { 782 case CPU_R4600: /* QED style two way caches? */ 783 case CPU_R4700: 784 case CPU_R5000: 785 case CPU_NEVADA: 786 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 787 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 788 c->icache.ways = 2; 789 c->icache.waybit = __ffs(icache_size/2); 790 791 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 792 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 793 c->dcache.ways = 2; 794 c->dcache.waybit= __ffs(dcache_size/2); 795 796 c->options |= MIPS_CPU_CACHE_CDEX_P; 797 break; 798 799 case CPU_R5432: 800 case CPU_R5500: 801 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 802 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 803 c->icache.ways = 2; 804 c->icache.waybit= 0; 805 806 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 807 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 808 c->dcache.ways = 2; 809 c->dcache.waybit = 0; 810 811 c->options |= MIPS_CPU_CACHE_CDEX_P; 812 break; 813 814 case CPU_TX49XX: 815 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 816 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 817 c->icache.ways = 4; 818 c->icache.waybit= 0; 819 820 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 821 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 822 c->dcache.ways = 4; 823 c->dcache.waybit = 0; 824 825 c->options |= MIPS_CPU_CACHE_CDEX_P; 826 c->options |= MIPS_CPU_PREFETCH; 827 break; 828 829 case CPU_R4000PC: 830 case CPU_R4000SC: 831 case CPU_R4000MC: 832 case CPU_R4400PC: 833 case CPU_R4400SC: 834 case CPU_R4400MC: 835 case CPU_R4300: 836 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 837 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 838 c->icache.ways = 1; 839 c->icache.waybit = 0; /* doesn't matter */ 840 841 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 842 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 843 c->dcache.ways = 1; 844 c->dcache.waybit = 0; /* does not matter */ 845 846 c->options |= MIPS_CPU_CACHE_CDEX_P; 847 break; 848 849 case CPU_R10000: 850 case CPU_R12000: 851 case CPU_R14000: 852 icache_size = 1 << (12 + ((config & R10K_CONF_IC) >> 29)); 853 c->icache.linesz = 64; 854 c->icache.ways = 2; 855 c->icache.waybit = 0; 856 857 dcache_size = 1 << (12 + ((config & R10K_CONF_DC) >> 26)); 858 c->dcache.linesz = 32; 859 c->dcache.ways = 2; 860 c->dcache.waybit = 0; 861 862 c->options |= MIPS_CPU_PREFETCH; 863 break; 864 865 case CPU_VR4133: 866 write_c0_config(config & ~CONF_EB); 867 case CPU_VR4131: 868 /* Workaround for cache instruction bug of VR4131 */ 869 if (c->processor_id == 0x0c80U || c->processor_id == 0x0c81U || 870 c->processor_id == 0x0c82U) { 871 config &= ~0x00000030U; 872 config |= 0x00410000U; 873 write_c0_config(config); 874 } 875 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 876 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 877 c->icache.ways = 2; 878 c->icache.waybit = __ffs(icache_size/2); 879 880 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 881 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 882 c->dcache.ways = 2; 883 c->dcache.waybit = __ffs(dcache_size/2); 884 885 c->options |= MIPS_CPU_CACHE_CDEX_P; 886 break; 887 888 case CPU_VR41XX: 889 case CPU_VR4111: 890 case CPU_VR4121: 891 case CPU_VR4122: 892 case CPU_VR4181: 893 case CPU_VR4181A: 894 icache_size = 1 << (10 + ((config & CONF_IC) >> 9)); 895 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 896 c->icache.ways = 1; 897 c->icache.waybit = 0; /* doesn't matter */ 898 899 dcache_size = 1 << (10 + ((config & CONF_DC) >> 6)); 900 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 901 c->dcache.ways = 1; 902 c->dcache.waybit = 0; /* does not matter */ 903 904 c->options |= MIPS_CPU_CACHE_CDEX_P; 905 break; 906 907 case CPU_RM7000: 908 rm7k_erratum31(); 909 910 case CPU_RM9000: 911 icache_size = 1 << (12 + ((config & CONF_IC) >> 9)); 912 c->icache.linesz = 16 << ((config & CONF_IB) >> 5); 913 c->icache.ways = 4; 914 c->icache.waybit = __ffs(icache_size / c->icache.ways); 915 916 dcache_size = 1 << (12 + ((config & CONF_DC) >> 6)); 917 c->dcache.linesz = 16 << ((config & CONF_DB) >> 4); 918 c->dcache.ways = 4; 919 c->dcache.waybit = __ffs(dcache_size / c->dcache.ways); 920 921 #if !defined(CONFIG_SMP) || !defined(RM9000_CDEX_SMP_WAR) 922 c->options |= MIPS_CPU_CACHE_CDEX_P; 923 #endif 924 c->options |= MIPS_CPU_PREFETCH; 925 break; 926 927 default: 928 if (!(config & MIPS_CONF_M)) 929 panic("Don't know how to probe P-caches on this cpu."); 930 931 /* 932 * So we seem to be a MIPS32 or MIPS64 CPU 933 * So let's probe the I-cache ... 934 */ 935 config1 = read_c0_config1(); 936 937 if ((lsize = ((config1 >> 19) & 7))) 938 c->icache.linesz = 2 << lsize; 939 else 940 c->icache.linesz = lsize; 941 c->icache.sets = 64 << ((config1 >> 22) & 7); 942 c->icache.ways = 1 + ((config1 >> 16) & 7); 943 944 icache_size = c->icache.sets * 945 c->icache.ways * 946 c->icache.linesz; 947 c->icache.waybit = __ffs(icache_size/c->icache.ways); 948 949 if (config & 0x8) /* VI bit */ 950 c->icache.flags |= MIPS_CACHE_VTAG; 951 952 /* 953 * Now probe the MIPS32 / MIPS64 data cache. 954 */ 955 c->dcache.flags = 0; 956 957 if ((lsize = ((config1 >> 10) & 7))) 958 c->dcache.linesz = 2 << lsize; 959 else 960 c->dcache.linesz= lsize; 961 c->dcache.sets = 64 << ((config1 >> 13) & 7); 962 c->dcache.ways = 1 + ((config1 >> 7) & 7); 963 964 dcache_size = c->dcache.sets * 965 c->dcache.ways * 966 c->dcache.linesz; 967 c->dcache.waybit = __ffs(dcache_size/c->dcache.ways); 968 969 c->options |= MIPS_CPU_PREFETCH; 970 break; 971 } 972 973 /* 974 * Processor configuration sanity check for the R4000SC erratum 975 * #5. With page sizes larger than 32kB there is no possibility 976 * to get a VCE exception anymore so we don't care about this 977 * misconfiguration. The case is rather theoretical anyway; 978 * presumably no vendor is shipping his hardware in the "bad" 979 * configuration. 980 */ 981 if ((prid & 0xff00) == PRID_IMP_R4000 && (prid & 0xff) < 0x40 && 982 !(config & CONF_SC) && c->icache.linesz != 16 && 983 PAGE_SIZE <= 0x8000) 984 panic("Improper R4000SC processor configuration detected"); 985 986 /* compute a couple of other cache variables */ 987 c->icache.waysize = icache_size / c->icache.ways; 988 c->dcache.waysize = dcache_size / c->dcache.ways; 989 990 c->icache.sets = c->icache.linesz ? 991 icache_size / (c->icache.linesz * c->icache.ways) : 0; 992 c->dcache.sets = c->dcache.linesz ? 993 dcache_size / (c->dcache.linesz * c->dcache.ways) : 0; 994 995 /* 996 * R10000 and R12000 P-caches are odd in a positive way. They're 32kB 997 * 2-way virtually indexed so normally would suffer from aliases. So 998 * normally they'd suffer from aliases but magic in the hardware deals 999 * with that for us so we don't need to take care ourselves. 1000 */ 1001 switch (c->cputype) { 1002 case CPU_20KC: 1003 case CPU_25KF: 1004 c->dcache.flags |= MIPS_CACHE_PINDEX; 1005 case CPU_R10000: 1006 case CPU_R12000: 1007 case CPU_R14000: 1008 case CPU_SB1: 1009 break; 1010 case CPU_24K: 1011 case CPU_34K: 1012 case CPU_74K: 1013 if ((read_c0_config7() & (1 << 16))) { 1014 /* effectively physically indexed dcache, 1015 thus no virtual aliases. */ 1016 c->dcache.flags |= MIPS_CACHE_PINDEX; 1017 break; 1018 } 1019 default: 1020 if (c->dcache.waysize > PAGE_SIZE) 1021 c->dcache.flags |= MIPS_CACHE_ALIASES; 1022 } 1023 1024 switch (c->cputype) { 1025 case CPU_20KC: 1026 /* 1027 * Some older 20Kc chips doesn't have the 'VI' bit in 1028 * the config register. 1029 */ 1030 c->icache.flags |= MIPS_CACHE_VTAG; 1031 break; 1032 1033 case CPU_AU1000: 1034 case CPU_AU1500: 1035 case CPU_AU1100: 1036 case CPU_AU1550: 1037 case CPU_AU1200: 1038 c->icache.flags |= MIPS_CACHE_IC_F_DC; 1039 break; 1040 } 1041 1042 printk("Primary instruction cache %ldkB, %s, %s, linesize %d bytes.\n", 1043 icache_size >> 10, 1044 cpu_has_vtag_icache ? "virtually tagged" : "physically tagged", 1045 way_string[c->icache.ways], c->icache.linesz); 1046 1047 printk("Primary data cache %ldkB, %s, linesize %d bytes.\n", 1048 dcache_size >> 10, way_string[c->dcache.ways], c->dcache.linesz); 1049 } 1050 1051 /* 1052 * If you even _breathe_ on this function, look at the gcc output and make sure 1053 * it does not pop things on and off the stack for the cache sizing loop that 1054 * executes in KSEG1 space or else you will crash and burn badly. You have 1055 * been warned. 1056 */ 1057 static int __init probe_scache(void) 1058 { 1059 extern unsigned long stext; 1060 unsigned long flags, addr, begin, end, pow2; 1061 unsigned int config = read_c0_config(); 1062 struct cpuinfo_mips *c = ¤t_cpu_data; 1063 int tmp; 1064 1065 if (config & CONF_SC) 1066 return 0; 1067 1068 begin = (unsigned long) &stext; 1069 begin &= ~((4 * 1024 * 1024) - 1); 1070 end = begin + (4 * 1024 * 1024); 1071 1072 /* 1073 * This is such a bitch, you'd think they would make it easy to do 1074 * this. Away you daemons of stupidity! 1075 */ 1076 local_irq_save(flags); 1077 1078 /* Fill each size-multiple cache line with a valid tag. */ 1079 pow2 = (64 * 1024); 1080 for (addr = begin; addr < end; addr = (begin + pow2)) { 1081 unsigned long *p = (unsigned long *) addr; 1082 __asm__ __volatile__("nop" : : "r" (*p)); /* whee... */ 1083 pow2 <<= 1; 1084 } 1085 1086 /* Load first line with zero (therefore invalid) tag. */ 1087 write_c0_taglo(0); 1088 write_c0_taghi(0); 1089 __asm__ __volatile__("nop; nop; nop; nop;"); /* avoid the hazard */ 1090 cache_op(Index_Store_Tag_I, begin); 1091 cache_op(Index_Store_Tag_D, begin); 1092 cache_op(Index_Store_Tag_SD, begin); 1093 1094 /* Now search for the wrap around point. */ 1095 pow2 = (128 * 1024); 1096 tmp = 0; 1097 for (addr = begin + (128 * 1024); addr < end; addr = begin + pow2) { 1098 cache_op(Index_Load_Tag_SD, addr); 1099 __asm__ __volatile__("nop; nop; nop; nop;"); /* hazard... */ 1100 if (!read_c0_taglo()) 1101 break; 1102 pow2 <<= 1; 1103 } 1104 local_irq_restore(flags); 1105 addr -= begin; 1106 1107 scache_size = addr; 1108 c->scache.linesz = 16 << ((config & R4K_CONF_SB) >> 22); 1109 c->scache.ways = 1; 1110 c->dcache.waybit = 0; /* does not matter */ 1111 1112 return 1; 1113 } 1114 1115 extern int r5k_sc_init(void); 1116 extern int rm7k_sc_init(void); 1117 extern int mips_sc_init(void); 1118 1119 static void __init setup_scache(void) 1120 { 1121 struct cpuinfo_mips *c = ¤t_cpu_data; 1122 unsigned int config = read_c0_config(); 1123 int sc_present = 0; 1124 1125 /* 1126 * Do the probing thing on R4000SC and R4400SC processors. Other 1127 * processors don't have a S-cache that would be relevant to the 1128 * Linux memory managment. 1129 */ 1130 switch (c->cputype) { 1131 case CPU_R4000SC: 1132 case CPU_R4000MC: 1133 case CPU_R4400SC: 1134 case CPU_R4400MC: 1135 sc_present = run_uncached(probe_scache); 1136 if (sc_present) 1137 c->options |= MIPS_CPU_CACHE_CDEX_S; 1138 break; 1139 1140 case CPU_R10000: 1141 case CPU_R12000: 1142 case CPU_R14000: 1143 scache_size = 0x80000 << ((config & R10K_CONF_SS) >> 16); 1144 c->scache.linesz = 64 << ((config >> 13) & 1); 1145 c->scache.ways = 2; 1146 c->scache.waybit= 0; 1147 sc_present = 1; 1148 break; 1149 1150 case CPU_R5000: 1151 case CPU_NEVADA: 1152 #ifdef CONFIG_R5000_CPU_SCACHE 1153 r5k_sc_init(); 1154 #endif 1155 return; 1156 1157 case CPU_RM7000: 1158 case CPU_RM9000: 1159 #ifdef CONFIG_RM7000_CPU_SCACHE 1160 rm7k_sc_init(); 1161 #endif 1162 return; 1163 1164 default: 1165 if (c->isa_level == MIPS_CPU_ISA_M32R1 || 1166 c->isa_level == MIPS_CPU_ISA_M32R2 || 1167 c->isa_level == MIPS_CPU_ISA_M64R1 || 1168 c->isa_level == MIPS_CPU_ISA_M64R2) { 1169 #ifdef CONFIG_MIPS_CPU_SCACHE 1170 if (mips_sc_init ()) { 1171 scache_size = c->scache.ways * c->scache.sets * c->scache.linesz; 1172 printk("MIPS secondary cache %ldkB, %s, linesize %d bytes.\n", 1173 scache_size >> 10, 1174 way_string[c->scache.ways], c->scache.linesz); 1175 } 1176 #else 1177 if (!(c->scache.flags & MIPS_CACHE_NOT_PRESENT)) 1178 panic("Dunno how to handle MIPS32 / MIPS64 second level cache"); 1179 #endif 1180 return; 1181 } 1182 sc_present = 0; 1183 } 1184 1185 if (!sc_present) 1186 return; 1187 1188 /* compute a couple of other cache variables */ 1189 c->scache.waysize = scache_size / c->scache.ways; 1190 1191 c->scache.sets = scache_size / (c->scache.linesz * c->scache.ways); 1192 1193 printk("Unified secondary cache %ldkB %s, linesize %d bytes.\n", 1194 scache_size >> 10, way_string[c->scache.ways], c->scache.linesz); 1195 1196 c->options |= MIPS_CPU_SUBSET_CACHES; 1197 } 1198 1199 void au1x00_fixup_config_od(void) 1200 { 1201 /* 1202 * c0_config.od (bit 19) was write only (and read as 0) 1203 * on the early revisions of Alchemy SOCs. It disables the bus 1204 * transaction overlapping and needs to be set to fix various errata. 1205 */ 1206 switch (read_c0_prid()) { 1207 case 0x00030100: /* Au1000 DA */ 1208 case 0x00030201: /* Au1000 HA */ 1209 case 0x00030202: /* Au1000 HB */ 1210 case 0x01030200: /* Au1500 AB */ 1211 /* 1212 * Au1100 errata actually keeps silence about this bit, so we set it 1213 * just in case for those revisions that require it to be set according 1214 * to arch/mips/au1000/common/cputable.c 1215 */ 1216 case 0x02030200: /* Au1100 AB */ 1217 case 0x02030201: /* Au1100 BA */ 1218 case 0x02030202: /* Au1100 BC */ 1219 set_c0_config(1 << 19); 1220 break; 1221 } 1222 } 1223 1224 static inline void coherency_setup(void) 1225 { 1226 change_c0_config(CONF_CM_CMASK, CONF_CM_DEFAULT); 1227 1228 /* 1229 * c0_status.cu=0 specifies that updates by the sc instruction use 1230 * the coherency mode specified by the TLB; 1 means cachable 1231 * coherent update on write will be used. Not all processors have 1232 * this bit and; some wire it to zero, others like Toshiba had the 1233 * silly idea of putting something else there ... 1234 */ 1235 switch (current_cpu_data.cputype) { 1236 case CPU_R4000PC: 1237 case CPU_R4000SC: 1238 case CPU_R4000MC: 1239 case CPU_R4400PC: 1240 case CPU_R4400SC: 1241 case CPU_R4400MC: 1242 clear_c0_config(CONF_CU); 1243 break; 1244 /* 1245 * We need to catch the ealry Alchemy SOCs with 1246 * the write-only co_config.od bit and set it back to one... 1247 */ 1248 case CPU_AU1000: /* rev. DA, HA, HB */ 1249 case CPU_AU1100: /* rev. AB, BA, BC ?? */ 1250 case CPU_AU1500: /* rev. AB */ 1251 au1x00_fixup_config_od(); 1252 break; 1253 } 1254 } 1255 1256 void __init r4k_cache_init(void) 1257 { 1258 extern void build_clear_page(void); 1259 extern void build_copy_page(void); 1260 extern char except_vec2_generic; 1261 struct cpuinfo_mips *c = ¤t_cpu_data; 1262 1263 /* Default cache error handler for R4000 and R5000 family */ 1264 set_uncached_handler (0x100, &except_vec2_generic, 0x80); 1265 1266 probe_pcache(); 1267 setup_scache(); 1268 1269 r4k_blast_dcache_page_setup(); 1270 r4k_blast_dcache_page_indexed_setup(); 1271 r4k_blast_dcache_setup(); 1272 r4k_blast_icache_page_setup(); 1273 r4k_blast_icache_page_indexed_setup(); 1274 r4k_blast_icache_setup(); 1275 r4k_blast_scache_page_setup(); 1276 r4k_blast_scache_page_indexed_setup(); 1277 r4k_blast_scache_setup(); 1278 1279 /* 1280 * Some MIPS32 and MIPS64 processors have physically indexed caches. 1281 * This code supports virtually indexed processors and will be 1282 * unnecessarily inefficient on physically indexed processors. 1283 */ 1284 if (c->dcache.linesz) 1285 shm_align_mask = max_t( unsigned long, 1286 c->dcache.sets * c->dcache.linesz - 1, 1287 PAGE_SIZE - 1); 1288 else 1289 shm_align_mask = PAGE_SIZE-1; 1290 flush_cache_all = r4k_flush_cache_all; 1291 __flush_cache_all = r4k___flush_cache_all; 1292 flush_cache_mm = r4k_flush_cache_mm; 1293 flush_cache_page = r4k_flush_cache_page; 1294 flush_icache_page = r4k_flush_icache_page; 1295 flush_cache_range = r4k_flush_cache_range; 1296 1297 flush_cache_sigtramp = r4k_flush_cache_sigtramp; 1298 flush_icache_all = r4k_flush_icache_all; 1299 local_flush_data_cache_page = local_r4k_flush_data_cache_page; 1300 flush_data_cache_page = r4k_flush_data_cache_page; 1301 flush_icache_range = r4k_flush_icache_range; 1302 1303 #ifdef CONFIG_DMA_NONCOHERENT 1304 _dma_cache_wback_inv = r4k_dma_cache_wback_inv; 1305 _dma_cache_wback = r4k_dma_cache_wback_inv; 1306 _dma_cache_inv = r4k_dma_cache_inv; 1307 #endif 1308 1309 build_clear_page(); 1310 build_copy_page(); 1311 local_r4k___flush_cache_all(NULL); 1312 coherency_setup(); 1313 } 1314