1 /* 2 * ARC Cache Management 3 * 4 * Copyright (C) 2014-15 Synopsys, Inc. (www.synopsys.com) 5 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 12 #include <linux/module.h> 13 #include <linux/mm.h> 14 #include <linux/sched.h> 15 #include <linux/cache.h> 16 #include <linux/mmu_context.h> 17 #include <linux/syscalls.h> 18 #include <linux/uaccess.h> 19 #include <linux/pagemap.h> 20 #include <asm/cacheflush.h> 21 #include <asm/cachectl.h> 22 #include <asm/setup.h> 23 24 #ifdef CONFIG_ISA_ARCV2 25 #define USE_RGN_FLSH 1 26 #endif 27 28 static int l2_line_sz; 29 static int ioc_exists; 30 int slc_enable = 1, ioc_enable = 1; 31 unsigned long perip_base = ARC_UNCACHED_ADDR_SPACE; /* legacy value for boot */ 32 unsigned long perip_end = 0xFFFFFFFF; /* legacy value */ 33 34 void (*_cache_line_loop_ic_fn)(phys_addr_t paddr, unsigned long vaddr, 35 unsigned long sz, const int op, const int full_page); 36 37 void (*__dma_cache_wback_inv)(phys_addr_t start, unsigned long sz); 38 void (*__dma_cache_inv)(phys_addr_t start, unsigned long sz); 39 void (*__dma_cache_wback)(phys_addr_t start, unsigned long sz); 40 41 char *arc_cache_mumbojumbo(int c, char *buf, int len) 42 { 43 int n = 0; 44 struct cpuinfo_arc_cache *p; 45 46 #define PR_CACHE(p, cfg, str) \ 47 if (!(p)->line_len) \ 48 n += scnprintf(buf + n, len - n, str"\t\t: N/A\n"); \ 49 else \ 50 n += scnprintf(buf + n, len - n, \ 51 str"\t\t: %uK, %dway/set, %uB Line, %s%s%s\n", \ 52 (p)->sz_k, (p)->assoc, (p)->line_len, \ 53 (p)->vipt ? "VIPT" : "PIPT", \ 54 (p)->alias ? " aliasing" : "", \ 55 IS_USED_CFG(cfg)); 56 57 PR_CACHE(&cpuinfo_arc700[c].icache, CONFIG_ARC_HAS_ICACHE, "I-Cache"); 58 PR_CACHE(&cpuinfo_arc700[c].dcache, CONFIG_ARC_HAS_DCACHE, "D-Cache"); 59 60 p = &cpuinfo_arc700[c].slc; 61 if (p->line_len) 62 n += scnprintf(buf + n, len - n, 63 "SLC\t\t: %uK, %uB Line%s\n", 64 p->sz_k, p->line_len, IS_USED_RUN(slc_enable)); 65 66 n += scnprintf(buf + n, len - n, "Peripherals\t: %#lx%s%s\n", 67 perip_base, 68 IS_AVAIL3(ioc_exists, ioc_enable, ", IO-Coherency ")); 69 70 return buf; 71 } 72 73 /* 74 * Read the Cache Build Confuration Registers, Decode them and save into 75 * the cpuinfo structure for later use. 76 * No Validation done here, simply read/convert the BCRs 77 */ 78 static void read_decode_cache_bcr_arcv2(int cpu) 79 { 80 struct cpuinfo_arc_cache *p_slc = &cpuinfo_arc700[cpu].slc; 81 struct bcr_generic sbcr; 82 83 struct bcr_slc_cfg { 84 #ifdef CONFIG_CPU_BIG_ENDIAN 85 unsigned int pad:24, way:2, lsz:2, sz:4; 86 #else 87 unsigned int sz:4, lsz:2, way:2, pad:24; 88 #endif 89 } slc_cfg; 90 91 struct bcr_clust_cfg { 92 #ifdef CONFIG_CPU_BIG_ENDIAN 93 unsigned int pad:7, c:1, num_entries:8, num_cores:8, ver:8; 94 #else 95 unsigned int ver:8, num_cores:8, num_entries:8, c:1, pad:7; 96 #endif 97 } cbcr; 98 99 struct bcr_volatile { 100 #ifdef CONFIG_CPU_BIG_ENDIAN 101 unsigned int start:4, limit:4, pad:22, order:1, disable:1; 102 #else 103 unsigned int disable:1, order:1, pad:22, limit:4, start:4; 104 #endif 105 } vol; 106 107 108 READ_BCR(ARC_REG_SLC_BCR, sbcr); 109 if (sbcr.ver) { 110 READ_BCR(ARC_REG_SLC_CFG, slc_cfg); 111 p_slc->sz_k = 128 << slc_cfg.sz; 112 l2_line_sz = p_slc->line_len = (slc_cfg.lsz == 0) ? 128 : 64; 113 } 114 115 READ_BCR(ARC_REG_CLUSTER_BCR, cbcr); 116 if (cbcr.c) 117 ioc_exists = 1; 118 else 119 ioc_enable = 0; 120 121 /* HS 2.0 didn't have AUX_VOL */ 122 if (cpuinfo_arc700[cpu].core.family > 0x51) { 123 READ_BCR(AUX_VOL, vol); 124 perip_base = vol.start << 28; 125 /* HS 3.0 has limit and strict-ordering fields */ 126 if (cpuinfo_arc700[cpu].core.family > 0x52) 127 perip_end = (vol.limit << 28) - 1; 128 } 129 } 130 131 void read_decode_cache_bcr(void) 132 { 133 struct cpuinfo_arc_cache *p_ic, *p_dc; 134 unsigned int cpu = smp_processor_id(); 135 struct bcr_cache { 136 #ifdef CONFIG_CPU_BIG_ENDIAN 137 unsigned int pad:12, line_len:4, sz:4, config:4, ver:8; 138 #else 139 unsigned int ver:8, config:4, sz:4, line_len:4, pad:12; 140 #endif 141 } ibcr, dbcr; 142 143 p_ic = &cpuinfo_arc700[cpu].icache; 144 READ_BCR(ARC_REG_IC_BCR, ibcr); 145 146 if (!ibcr.ver) 147 goto dc_chk; 148 149 if (ibcr.ver <= 3) { 150 BUG_ON(ibcr.config != 3); 151 p_ic->assoc = 2; /* Fixed to 2w set assoc */ 152 } else if (ibcr.ver >= 4) { 153 p_ic->assoc = 1 << ibcr.config; /* 1,2,4,8 */ 154 } 155 156 p_ic->line_len = 8 << ibcr.line_len; 157 p_ic->sz_k = 1 << (ibcr.sz - 1); 158 p_ic->vipt = 1; 159 p_ic->alias = p_ic->sz_k/p_ic->assoc/TO_KB(PAGE_SIZE) > 1; 160 161 dc_chk: 162 p_dc = &cpuinfo_arc700[cpu].dcache; 163 READ_BCR(ARC_REG_DC_BCR, dbcr); 164 165 if (!dbcr.ver) 166 goto slc_chk; 167 168 if (dbcr.ver <= 3) { 169 BUG_ON(dbcr.config != 2); 170 p_dc->assoc = 4; /* Fixed to 4w set assoc */ 171 p_dc->vipt = 1; 172 p_dc->alias = p_dc->sz_k/p_dc->assoc/TO_KB(PAGE_SIZE) > 1; 173 } else if (dbcr.ver >= 4) { 174 p_dc->assoc = 1 << dbcr.config; /* 1,2,4,8 */ 175 p_dc->vipt = 0; 176 p_dc->alias = 0; /* PIPT so can't VIPT alias */ 177 } 178 179 p_dc->line_len = 16 << dbcr.line_len; 180 p_dc->sz_k = 1 << (dbcr.sz - 1); 181 182 slc_chk: 183 if (is_isa_arcv2()) 184 read_decode_cache_bcr_arcv2(cpu); 185 } 186 187 /* 188 * Line Operation on {I,D}-Cache 189 */ 190 191 #define OP_INV 0x1 192 #define OP_FLUSH 0x2 193 #define OP_FLUSH_N_INV 0x3 194 #define OP_INV_IC 0x4 195 196 /* 197 * I-Cache Aliasing in ARC700 VIPT caches (MMU v1-v3) 198 * 199 * ARC VIPT I-cache uses vaddr to index into cache and paddr to match the tag. 200 * The orig Cache Management Module "CDU" only required paddr to invalidate a 201 * certain line since it sufficed as index in Non-Aliasing VIPT cache-geometry. 202 * Infact for distinct V1,V2,P: all of {V1-P},{V2-P},{P-P} would end up fetching 203 * the exact same line. 204 * 205 * However for larger Caches (way-size > page-size) - i.e. in Aliasing config, 206 * paddr alone could not be used to correctly index the cache. 207 * 208 * ------------------ 209 * MMU v1/v2 (Fixed Page Size 8k) 210 * ------------------ 211 * The solution was to provide CDU with these additonal vaddr bits. These 212 * would be bits [x:13], x would depend on cache-geometry, 13 comes from 213 * standard page size of 8k. 214 * H/w folks chose [17:13] to be a future safe range, and moreso these 5 bits 215 * of vaddr could easily be "stuffed" in the paddr as bits [4:0] since the 216 * orig 5 bits of paddr were anyways ignored by CDU line ops, as they 217 * represent the offset within cache-line. The adv of using this "clumsy" 218 * interface for additional info was no new reg was needed in CDU programming 219 * model. 220 * 221 * 17:13 represented the max num of bits passable, actual bits needed were 222 * fewer, based on the num-of-aliases possible. 223 * -for 2 alias possibility, only bit 13 needed (32K cache) 224 * -for 4 alias possibility, bits 14:13 needed (64K cache) 225 * 226 * ------------------ 227 * MMU v3 228 * ------------------ 229 * This ver of MMU supports variable page sizes (1k-16k): although Linux will 230 * only support 8k (default), 16k and 4k. 231 * However from hardware perspective, smaller page sizes aggravate aliasing 232 * meaning more vaddr bits needed to disambiguate the cache-line-op ; 233 * the existing scheme of piggybacking won't work for certain configurations. 234 * Two new registers IC_PTAG and DC_PTAG inttoduced. 235 * "tag" bits are provided in PTAG, index bits in existing IVIL/IVDL/FLDL regs 236 */ 237 238 static inline 239 void __cache_line_loop_v2(phys_addr_t paddr, unsigned long vaddr, 240 unsigned long sz, const int op, const int full_page) 241 { 242 unsigned int aux_cmd; 243 int num_lines; 244 245 if (op == OP_INV_IC) { 246 aux_cmd = ARC_REG_IC_IVIL; 247 } else { 248 /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 249 aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 250 } 251 252 /* Ensure we properly floor/ceil the non-line aligned/sized requests 253 * and have @paddr - aligned to cache line and integral @num_lines. 254 * This however can be avoided for page sized since: 255 * -@paddr will be cache-line aligned already (being page aligned) 256 * -@sz will be integral multiple of line size (being page sized). 257 */ 258 if (!full_page) { 259 sz += paddr & ~CACHE_LINE_MASK; 260 paddr &= CACHE_LINE_MASK; 261 vaddr &= CACHE_LINE_MASK; 262 } 263 264 num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 265 266 /* MMUv2 and before: paddr contains stuffed vaddrs bits */ 267 paddr |= (vaddr >> PAGE_SHIFT) & 0x1F; 268 269 while (num_lines-- > 0) { 270 write_aux_reg(aux_cmd, paddr); 271 paddr += L1_CACHE_BYTES; 272 } 273 } 274 275 /* 276 * For ARC700 MMUv3 I-cache and D-cache flushes 277 * - ARC700 programming model requires paddr and vaddr be passed in seperate 278 * AUX registers (*_IV*L and *_PTAG respectively) irrespective of whether the 279 * caches actually alias or not. 280 * - For HS38, only the aliasing I-cache configuration uses the PTAG reg 281 * (non aliasing I-cache version doesn't; while D-cache can't possibly alias) 282 */ 283 static inline 284 void __cache_line_loop_v3(phys_addr_t paddr, unsigned long vaddr, 285 unsigned long sz, const int op, const int full_page) 286 { 287 unsigned int aux_cmd, aux_tag; 288 int num_lines; 289 290 if (op == OP_INV_IC) { 291 aux_cmd = ARC_REG_IC_IVIL; 292 aux_tag = ARC_REG_IC_PTAG; 293 } else { 294 aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 295 aux_tag = ARC_REG_DC_PTAG; 296 } 297 298 /* Ensure we properly floor/ceil the non-line aligned/sized requests 299 * and have @paddr - aligned to cache line and integral @num_lines. 300 * This however can be avoided for page sized since: 301 * -@paddr will be cache-line aligned already (being page aligned) 302 * -@sz will be integral multiple of line size (being page sized). 303 */ 304 if (!full_page) { 305 sz += paddr & ~CACHE_LINE_MASK; 306 paddr &= CACHE_LINE_MASK; 307 vaddr &= CACHE_LINE_MASK; 308 } 309 num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 310 311 /* 312 * MMUv3, cache ops require paddr in PTAG reg 313 * if V-P const for loop, PTAG can be written once outside loop 314 */ 315 if (full_page) 316 write_aux_reg(aux_tag, paddr); 317 318 /* 319 * This is technically for MMU v4, using the MMU v3 programming model 320 * Special work for HS38 aliasing I-cache configuration with PAE40 321 * - upper 8 bits of paddr need to be written into PTAG_HI 322 * - (and needs to be written before the lower 32 bits) 323 * Note that PTAG_HI is hoisted outside the line loop 324 */ 325 if (is_pae40_enabled() && op == OP_INV_IC) 326 write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 327 328 while (num_lines-- > 0) { 329 if (!full_page) { 330 write_aux_reg(aux_tag, paddr); 331 paddr += L1_CACHE_BYTES; 332 } 333 334 write_aux_reg(aux_cmd, vaddr); 335 vaddr += L1_CACHE_BYTES; 336 } 337 } 338 339 #ifndef USE_RGN_FLSH 340 341 /* 342 * In HS38x (MMU v4), I-cache is VIPT (can alias), D-cache is PIPT 343 * Here's how cache ops are implemented 344 * 345 * - D-cache: only paddr needed (in DC_IVDL/DC_FLDL) 346 * - I-cache Non Aliasing: Despite VIPT, only paddr needed (in IC_IVIL) 347 * - I-cache Aliasing: Both vaddr and paddr needed (in IC_IVIL, IC_PTAG 348 * respectively, similar to MMU v3 programming model, hence 349 * __cache_line_loop_v3() is used) 350 * 351 * If PAE40 is enabled, independent of aliasing considerations, the higher bits 352 * needs to be written into PTAG_HI 353 */ 354 static inline 355 void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, 356 unsigned long sz, const int op, const int full_page) 357 { 358 unsigned int aux_cmd; 359 int num_lines; 360 361 if (op == OP_INV_IC) { 362 aux_cmd = ARC_REG_IC_IVIL; 363 } else { 364 /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 365 aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 366 } 367 368 /* Ensure we properly floor/ceil the non-line aligned/sized requests 369 * and have @paddr - aligned to cache line and integral @num_lines. 370 * This however can be avoided for page sized since: 371 * -@paddr will be cache-line aligned already (being page aligned) 372 * -@sz will be integral multiple of line size (being page sized). 373 */ 374 if (!full_page) { 375 sz += paddr & ~CACHE_LINE_MASK; 376 paddr &= CACHE_LINE_MASK; 377 } 378 379 num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 380 381 /* 382 * For HS38 PAE40 configuration 383 * - upper 8 bits of paddr need to be written into PTAG_HI 384 * - (and needs to be written before the lower 32 bits) 385 */ 386 if (is_pae40_enabled()) { 387 if (op == OP_INV_IC) 388 /* 389 * Non aliasing I-cache in HS38, 390 * aliasing I-cache handled in __cache_line_loop_v3() 391 */ 392 write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 393 else 394 write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32); 395 } 396 397 while (num_lines-- > 0) { 398 write_aux_reg(aux_cmd, paddr); 399 paddr += L1_CACHE_BYTES; 400 } 401 } 402 403 #else 404 405 /* 406 * optimized flush operation which takes a region as opposed to iterating per line 407 */ 408 static inline 409 void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, 410 unsigned long sz, const int op, const int full_page) 411 { 412 unsigned int s, e; 413 414 /* Only for Non aliasing I-cache in HS38 */ 415 if (op == OP_INV_IC) { 416 s = ARC_REG_IC_IVIR; 417 e = ARC_REG_IC_ENDR; 418 } else { 419 s = ARC_REG_DC_STARTR; 420 e = ARC_REG_DC_ENDR; 421 } 422 423 if (!full_page) { 424 /* for any leading gap between @paddr and start of cache line */ 425 sz += paddr & ~CACHE_LINE_MASK; 426 paddr &= CACHE_LINE_MASK; 427 428 /* 429 * account for any trailing gap to end of cache line 430 * this is equivalent to DIV_ROUND_UP() in line ops above 431 */ 432 sz += L1_CACHE_BYTES - 1; 433 } 434 435 if (is_pae40_enabled()) { 436 /* TBD: check if crossing 4TB boundary */ 437 if (op == OP_INV_IC) 438 write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 439 else 440 write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32); 441 } 442 443 /* ENDR needs to be set ahead of START */ 444 write_aux_reg(e, paddr + sz); /* ENDR is exclusive */ 445 write_aux_reg(s, paddr); 446 447 /* caller waits on DC_CTRL.FS */ 448 } 449 450 #endif 451 452 #if (CONFIG_ARC_MMU_VER < 3) 453 #define __cache_line_loop __cache_line_loop_v2 454 #elif (CONFIG_ARC_MMU_VER == 3) 455 #define __cache_line_loop __cache_line_loop_v3 456 #elif (CONFIG_ARC_MMU_VER > 3) 457 #define __cache_line_loop __cache_line_loop_v4 458 #endif 459 460 #ifdef CONFIG_ARC_HAS_DCACHE 461 462 /*************************************************************** 463 * Machine specific helpers for Entire D-Cache or Per Line ops 464 */ 465 466 #ifndef USE_RGN_FLSH 467 /* 468 * this version avoids extra read/write of DC_CTRL for flush or invalid ops 469 * in the non region flush regime (such as for ARCompact) 470 */ 471 static inline void __before_dc_op(const int op) 472 { 473 if (op == OP_FLUSH_N_INV) { 474 /* Dcache provides 2 cmd: FLUSH or INV 475 * INV inturn has sub-modes: DISCARD or FLUSH-BEFORE 476 * flush-n-inv is achieved by INV cmd but with IM=1 477 * So toggle INV sub-mode depending on op request and default 478 */ 479 const unsigned int ctl = ARC_REG_DC_CTRL; 480 write_aux_reg(ctl, read_aux_reg(ctl) | DC_CTRL_INV_MODE_FLUSH); 481 } 482 } 483 484 #else 485 486 static inline void __before_dc_op(const int op) 487 { 488 const unsigned int ctl = ARC_REG_DC_CTRL; 489 unsigned int val = read_aux_reg(ctl); 490 491 if (op == OP_FLUSH_N_INV) { 492 val |= DC_CTRL_INV_MODE_FLUSH; 493 } 494 495 if (op != OP_INV_IC) { 496 /* 497 * Flush / Invalidate is provided by DC_CTRL.RNG_OP 0 or 1 498 * combined Flush-n-invalidate uses DC_CTRL.IM = 1 set above 499 */ 500 val &= ~DC_CTRL_RGN_OP_MSK; 501 if (op & OP_INV) 502 val |= DC_CTRL_RGN_OP_INV; 503 } 504 write_aux_reg(ctl, val); 505 } 506 507 #endif 508 509 510 static inline void __after_dc_op(const int op) 511 { 512 if (op & OP_FLUSH) { 513 const unsigned int ctl = ARC_REG_DC_CTRL; 514 unsigned int reg; 515 516 /* flush / flush-n-inv both wait */ 517 while ((reg = read_aux_reg(ctl)) & DC_CTRL_FLUSH_STATUS) 518 ; 519 520 /* Switch back to default Invalidate mode */ 521 if (op == OP_FLUSH_N_INV) 522 write_aux_reg(ctl, reg & ~DC_CTRL_INV_MODE_FLUSH); 523 } 524 } 525 526 /* 527 * Operation on Entire D-Cache 528 * @op = {OP_INV, OP_FLUSH, OP_FLUSH_N_INV} 529 * Note that constant propagation ensures all the checks are gone 530 * in generated code 531 */ 532 static inline void __dc_entire_op(const int op) 533 { 534 int aux; 535 536 __before_dc_op(op); 537 538 if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 539 aux = ARC_REG_DC_IVDC; 540 else 541 aux = ARC_REG_DC_FLSH; 542 543 write_aux_reg(aux, 0x1); 544 545 __after_dc_op(op); 546 } 547 548 static inline void __dc_disable(void) 549 { 550 const int r = ARC_REG_DC_CTRL; 551 552 __dc_entire_op(OP_FLUSH_N_INV); 553 write_aux_reg(r, read_aux_reg(r) | DC_CTRL_DIS); 554 } 555 556 static void __dc_enable(void) 557 { 558 const int r = ARC_REG_DC_CTRL; 559 560 write_aux_reg(r, read_aux_reg(r) & ~DC_CTRL_DIS); 561 } 562 563 /* For kernel mappings cache operation: index is same as paddr */ 564 #define __dc_line_op_k(p, sz, op) __dc_line_op(p, p, sz, op) 565 566 /* 567 * D-Cache Line ops: Per Line INV (discard or wback+discard) or FLUSH (wback) 568 */ 569 static inline void __dc_line_op(phys_addr_t paddr, unsigned long vaddr, 570 unsigned long sz, const int op) 571 { 572 const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; 573 unsigned long flags; 574 575 local_irq_save(flags); 576 577 __before_dc_op(op); 578 579 __cache_line_loop(paddr, vaddr, sz, op, full_page); 580 581 __after_dc_op(op); 582 583 local_irq_restore(flags); 584 } 585 586 #else 587 588 #define __dc_entire_op(op) 589 #define __dc_disable() 590 #define __dc_enable() 591 #define __dc_line_op(paddr, vaddr, sz, op) 592 #define __dc_line_op_k(paddr, sz, op) 593 594 #endif /* CONFIG_ARC_HAS_DCACHE */ 595 596 #ifdef CONFIG_ARC_HAS_ICACHE 597 598 static inline void __ic_entire_inv(void) 599 { 600 write_aux_reg(ARC_REG_IC_IVIC, 1); 601 read_aux_reg(ARC_REG_IC_CTRL); /* blocks */ 602 } 603 604 static inline void 605 __ic_line_inv_vaddr_local(phys_addr_t paddr, unsigned long vaddr, 606 unsigned long sz) 607 { 608 const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; 609 unsigned long flags; 610 611 local_irq_save(flags); 612 (*_cache_line_loop_ic_fn)(paddr, vaddr, sz, OP_INV_IC, full_page); 613 local_irq_restore(flags); 614 } 615 616 #ifndef CONFIG_SMP 617 618 #define __ic_line_inv_vaddr(p, v, s) __ic_line_inv_vaddr_local(p, v, s) 619 620 #else 621 622 struct ic_inv_args { 623 phys_addr_t paddr, vaddr; 624 int sz; 625 }; 626 627 static void __ic_line_inv_vaddr_helper(void *info) 628 { 629 struct ic_inv_args *ic_inv = info; 630 631 __ic_line_inv_vaddr_local(ic_inv->paddr, ic_inv->vaddr, ic_inv->sz); 632 } 633 634 static void __ic_line_inv_vaddr(phys_addr_t paddr, unsigned long vaddr, 635 unsigned long sz) 636 { 637 struct ic_inv_args ic_inv = { 638 .paddr = paddr, 639 .vaddr = vaddr, 640 .sz = sz 641 }; 642 643 on_each_cpu(__ic_line_inv_vaddr_helper, &ic_inv, 1); 644 } 645 646 #endif /* CONFIG_SMP */ 647 648 #else /* !CONFIG_ARC_HAS_ICACHE */ 649 650 #define __ic_entire_inv() 651 #define __ic_line_inv_vaddr(pstart, vstart, sz) 652 653 #endif /* CONFIG_ARC_HAS_ICACHE */ 654 655 noinline void slc_op_rgn(phys_addr_t paddr, unsigned long sz, const int op) 656 { 657 #ifdef CONFIG_ISA_ARCV2 658 /* 659 * SLC is shared between all cores and concurrent aux operations from 660 * multiple cores need to be serialized using a spinlock 661 * A concurrent operation can be silently ignored and/or the old/new 662 * operation can remain incomplete forever (lockup in SLC_CTRL_BUSY loop 663 * below) 664 */ 665 static DEFINE_SPINLOCK(lock); 666 unsigned long flags; 667 unsigned int ctrl; 668 phys_addr_t end; 669 670 spin_lock_irqsave(&lock, flags); 671 672 /* 673 * The Region Flush operation is specified by CTRL.RGN_OP[11..9] 674 * - b'000 (default) is Flush, 675 * - b'001 is Invalidate if CTRL.IM == 0 676 * - b'001 is Flush-n-Invalidate if CTRL.IM == 1 677 */ 678 ctrl = read_aux_reg(ARC_REG_SLC_CTRL); 679 680 /* Don't rely on default value of IM bit */ 681 if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 682 ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 683 else 684 ctrl |= SLC_CTRL_IM; 685 686 if (op & OP_INV) 687 ctrl |= SLC_CTRL_RGN_OP_INV; /* Inv or flush-n-inv */ 688 else 689 ctrl &= ~SLC_CTRL_RGN_OP_INV; 690 691 write_aux_reg(ARC_REG_SLC_CTRL, ctrl); 692 693 /* 694 * Lower bits are ignored, no need to clip 695 * END needs to be setup before START (latter triggers the operation) 696 * END can't be same as START, so add (l2_line_sz - 1) to sz 697 */ 698 end = paddr + sz + l2_line_sz - 1; 699 if (is_pae40_enabled()) 700 write_aux_reg(ARC_REG_SLC_RGN_END1, upper_32_bits(end)); 701 702 write_aux_reg(ARC_REG_SLC_RGN_END, lower_32_bits(end)); 703 704 if (is_pae40_enabled()) 705 write_aux_reg(ARC_REG_SLC_RGN_START1, upper_32_bits(paddr)); 706 707 write_aux_reg(ARC_REG_SLC_RGN_START, lower_32_bits(paddr)); 708 709 /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ 710 read_aux_reg(ARC_REG_SLC_CTRL); 711 712 while (read_aux_reg(ARC_REG_SLC_CTRL) & SLC_CTRL_BUSY); 713 714 spin_unlock_irqrestore(&lock, flags); 715 #endif 716 } 717 718 noinline void slc_op_line(phys_addr_t paddr, unsigned long sz, const int op) 719 { 720 #ifdef CONFIG_ISA_ARCV2 721 /* 722 * SLC is shared between all cores and concurrent aux operations from 723 * multiple cores need to be serialized using a spinlock 724 * A concurrent operation can be silently ignored and/or the old/new 725 * operation can remain incomplete forever (lockup in SLC_CTRL_BUSY loop 726 * below) 727 */ 728 static DEFINE_SPINLOCK(lock); 729 730 const unsigned long SLC_LINE_MASK = ~(l2_line_sz - 1); 731 unsigned int ctrl, cmd; 732 unsigned long flags; 733 int num_lines; 734 735 spin_lock_irqsave(&lock, flags); 736 737 ctrl = read_aux_reg(ARC_REG_SLC_CTRL); 738 739 /* Don't rely on default value of IM bit */ 740 if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 741 ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 742 else 743 ctrl |= SLC_CTRL_IM; 744 745 write_aux_reg(ARC_REG_SLC_CTRL, ctrl); 746 747 cmd = op & OP_INV ? ARC_AUX_SLC_IVDL : ARC_AUX_SLC_FLDL; 748 749 sz += paddr & ~SLC_LINE_MASK; 750 paddr &= SLC_LINE_MASK; 751 752 num_lines = DIV_ROUND_UP(sz, l2_line_sz); 753 754 while (num_lines-- > 0) { 755 write_aux_reg(cmd, paddr); 756 paddr += l2_line_sz; 757 } 758 759 /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ 760 read_aux_reg(ARC_REG_SLC_CTRL); 761 762 while (read_aux_reg(ARC_REG_SLC_CTRL) & SLC_CTRL_BUSY); 763 764 spin_unlock_irqrestore(&lock, flags); 765 #endif 766 } 767 768 #define slc_op(paddr, sz, op) slc_op_rgn(paddr, sz, op) 769 770 noinline static void slc_entire_op(const int op) 771 { 772 unsigned int ctrl, r = ARC_REG_SLC_CTRL; 773 774 ctrl = read_aux_reg(r); 775 776 if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 777 ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 778 else 779 ctrl |= SLC_CTRL_IM; 780 781 write_aux_reg(r, ctrl); 782 783 write_aux_reg(ARC_REG_SLC_INVALIDATE, 1); 784 785 /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ 786 read_aux_reg(r); 787 788 /* Important to wait for flush to complete */ 789 while (read_aux_reg(r) & SLC_CTRL_BUSY); 790 } 791 792 static inline void arc_slc_disable(void) 793 { 794 const int r = ARC_REG_SLC_CTRL; 795 796 slc_entire_op(OP_FLUSH_N_INV); 797 write_aux_reg(r, read_aux_reg(r) | SLC_CTRL_DIS); 798 } 799 800 static inline void arc_slc_enable(void) 801 { 802 const int r = ARC_REG_SLC_CTRL; 803 804 write_aux_reg(r, read_aux_reg(r) & ~SLC_CTRL_DIS); 805 } 806 807 /*********************************************************** 808 * Exported APIs 809 */ 810 811 /* 812 * Handle cache congruency of kernel and userspace mappings of page when kernel 813 * writes-to/reads-from 814 * 815 * The idea is to defer flushing of kernel mapping after a WRITE, possible if: 816 * -dcache is NOT aliasing, hence any U/K-mappings of page are congruent 817 * -U-mapping doesn't exist yet for page (finalised in update_mmu_cache) 818 * -In SMP, if hardware caches are coherent 819 * 820 * There's a corollary case, where kernel READs from a userspace mapped page. 821 * If the U-mapping is not congruent to to K-mapping, former needs flushing. 822 */ 823 void flush_dcache_page(struct page *page) 824 { 825 struct address_space *mapping; 826 827 if (!cache_is_vipt_aliasing()) { 828 clear_bit(PG_dc_clean, &page->flags); 829 return; 830 } 831 832 /* don't handle anon pages here */ 833 mapping = page_mapping(page); 834 if (!mapping) 835 return; 836 837 /* 838 * pagecache page, file not yet mapped to userspace 839 * Make a note that K-mapping is dirty 840 */ 841 if (!mapping_mapped(mapping)) { 842 clear_bit(PG_dc_clean, &page->flags); 843 } else if (page_mapcount(page)) { 844 845 /* kernel reading from page with U-mapping */ 846 phys_addr_t paddr = (unsigned long)page_address(page); 847 unsigned long vaddr = page->index << PAGE_SHIFT; 848 849 if (addr_not_cache_congruent(paddr, vaddr)) 850 __flush_dcache_page(paddr, vaddr); 851 } 852 } 853 EXPORT_SYMBOL(flush_dcache_page); 854 855 /* 856 * DMA ops for systems with L1 cache only 857 * Make memory coherent with L1 cache by flushing/invalidating L1 lines 858 */ 859 static void __dma_cache_wback_inv_l1(phys_addr_t start, unsigned long sz) 860 { 861 __dc_line_op_k(start, sz, OP_FLUSH_N_INV); 862 } 863 864 static void __dma_cache_inv_l1(phys_addr_t start, unsigned long sz) 865 { 866 __dc_line_op_k(start, sz, OP_INV); 867 } 868 869 static void __dma_cache_wback_l1(phys_addr_t start, unsigned long sz) 870 { 871 __dc_line_op_k(start, sz, OP_FLUSH); 872 } 873 874 /* 875 * DMA ops for systems with both L1 and L2 caches, but without IOC 876 * Both L1 and L2 lines need to be explicitly flushed/invalidated 877 */ 878 static void __dma_cache_wback_inv_slc(phys_addr_t start, unsigned long sz) 879 { 880 __dc_line_op_k(start, sz, OP_FLUSH_N_INV); 881 slc_op(start, sz, OP_FLUSH_N_INV); 882 } 883 884 static void __dma_cache_inv_slc(phys_addr_t start, unsigned long sz) 885 { 886 __dc_line_op_k(start, sz, OP_INV); 887 slc_op(start, sz, OP_INV); 888 } 889 890 static void __dma_cache_wback_slc(phys_addr_t start, unsigned long sz) 891 { 892 __dc_line_op_k(start, sz, OP_FLUSH); 893 slc_op(start, sz, OP_FLUSH); 894 } 895 896 /* 897 * DMA ops for systems with IOC 898 * IOC hardware snoops all DMA traffic keeping the caches consistent with 899 * memory - eliding need for any explicit cache maintenance of DMA buffers 900 */ 901 static void __dma_cache_wback_inv_ioc(phys_addr_t start, unsigned long sz) {} 902 static void __dma_cache_inv_ioc(phys_addr_t start, unsigned long sz) {} 903 static void __dma_cache_wback_ioc(phys_addr_t start, unsigned long sz) {} 904 905 /* 906 * Exported DMA API 907 */ 908 void dma_cache_wback_inv(phys_addr_t start, unsigned long sz) 909 { 910 __dma_cache_wback_inv(start, sz); 911 } 912 EXPORT_SYMBOL(dma_cache_wback_inv); 913 914 void dma_cache_inv(phys_addr_t start, unsigned long sz) 915 { 916 __dma_cache_inv(start, sz); 917 } 918 EXPORT_SYMBOL(dma_cache_inv); 919 920 void dma_cache_wback(phys_addr_t start, unsigned long sz) 921 { 922 __dma_cache_wback(start, sz); 923 } 924 EXPORT_SYMBOL(dma_cache_wback); 925 926 /* 927 * This is API for making I/D Caches consistent when modifying 928 * kernel code (loadable modules, kprobes, kgdb...) 929 * This is called on insmod, with kernel virtual address for CODE of 930 * the module. ARC cache maintenance ops require PHY address thus we 931 * need to convert vmalloc addr to PHY addr 932 */ 933 void flush_icache_range(unsigned long kstart, unsigned long kend) 934 { 935 unsigned int tot_sz; 936 937 WARN(kstart < TASK_SIZE, "%s() can't handle user vaddr", __func__); 938 939 /* Shortcut for bigger flush ranges. 940 * Here we don't care if this was kernel virtual or phy addr 941 */ 942 tot_sz = kend - kstart; 943 if (tot_sz > PAGE_SIZE) { 944 flush_cache_all(); 945 return; 946 } 947 948 /* Case: Kernel Phy addr (0x8000_0000 onwards) */ 949 if (likely(kstart > PAGE_OFFSET)) { 950 /* 951 * The 2nd arg despite being paddr will be used to index icache 952 * This is OK since no alternate virtual mappings will exist 953 * given the callers for this case: kprobe/kgdb in built-in 954 * kernel code only. 955 */ 956 __sync_icache_dcache(kstart, kstart, kend - kstart); 957 return; 958 } 959 960 /* 961 * Case: Kernel Vaddr (0x7000_0000 to 0x7fff_ffff) 962 * (1) ARC Cache Maintenance ops only take Phy addr, hence special 963 * handling of kernel vaddr. 964 * 965 * (2) Despite @tot_sz being < PAGE_SIZE (bigger cases handled already), 966 * it still needs to handle a 2 page scenario, where the range 967 * straddles across 2 virtual pages and hence need for loop 968 */ 969 while (tot_sz > 0) { 970 unsigned int off, sz; 971 unsigned long phy, pfn; 972 973 off = kstart % PAGE_SIZE; 974 pfn = vmalloc_to_pfn((void *)kstart); 975 phy = (pfn << PAGE_SHIFT) + off; 976 sz = min_t(unsigned int, tot_sz, PAGE_SIZE - off); 977 __sync_icache_dcache(phy, kstart, sz); 978 kstart += sz; 979 tot_sz -= sz; 980 } 981 } 982 EXPORT_SYMBOL(flush_icache_range); 983 984 /* 985 * General purpose helper to make I and D cache lines consistent. 986 * @paddr is phy addr of region 987 * @vaddr is typically user vaddr (breakpoint) or kernel vaddr (vmalloc) 988 * However in one instance, when called by kprobe (for a breakpt in 989 * builtin kernel code) @vaddr will be paddr only, meaning CDU operation will 990 * use a paddr to index the cache (despite VIPT). This is fine since since a 991 * builtin kernel page will not have any virtual mappings. 992 * kprobe on loadable module will be kernel vaddr. 993 */ 994 void __sync_icache_dcache(phys_addr_t paddr, unsigned long vaddr, int len) 995 { 996 __dc_line_op(paddr, vaddr, len, OP_FLUSH_N_INV); 997 __ic_line_inv_vaddr(paddr, vaddr, len); 998 } 999 1000 /* wrapper to compile time eliminate alignment checks in flush loop */ 1001 void __inv_icache_page(phys_addr_t paddr, unsigned long vaddr) 1002 { 1003 __ic_line_inv_vaddr(paddr, vaddr, PAGE_SIZE); 1004 } 1005 1006 /* 1007 * wrapper to clearout kernel or userspace mappings of a page 1008 * For kernel mappings @vaddr == @paddr 1009 */ 1010 void __flush_dcache_page(phys_addr_t paddr, unsigned long vaddr) 1011 { 1012 __dc_line_op(paddr, vaddr & PAGE_MASK, PAGE_SIZE, OP_FLUSH_N_INV); 1013 } 1014 1015 noinline void flush_cache_all(void) 1016 { 1017 unsigned long flags; 1018 1019 local_irq_save(flags); 1020 1021 __ic_entire_inv(); 1022 __dc_entire_op(OP_FLUSH_N_INV); 1023 1024 local_irq_restore(flags); 1025 1026 } 1027 1028 #ifdef CONFIG_ARC_CACHE_VIPT_ALIASING 1029 1030 void flush_cache_mm(struct mm_struct *mm) 1031 { 1032 flush_cache_all(); 1033 } 1034 1035 void flush_cache_page(struct vm_area_struct *vma, unsigned long u_vaddr, 1036 unsigned long pfn) 1037 { 1038 unsigned int paddr = pfn << PAGE_SHIFT; 1039 1040 u_vaddr &= PAGE_MASK; 1041 1042 __flush_dcache_page(paddr, u_vaddr); 1043 1044 if (vma->vm_flags & VM_EXEC) 1045 __inv_icache_page(paddr, u_vaddr); 1046 } 1047 1048 void flush_cache_range(struct vm_area_struct *vma, unsigned long start, 1049 unsigned long end) 1050 { 1051 flush_cache_all(); 1052 } 1053 1054 void flush_anon_page(struct vm_area_struct *vma, struct page *page, 1055 unsigned long u_vaddr) 1056 { 1057 /* TBD: do we really need to clear the kernel mapping */ 1058 __flush_dcache_page(page_address(page), u_vaddr); 1059 __flush_dcache_page(page_address(page), page_address(page)); 1060 1061 } 1062 1063 #endif 1064 1065 void copy_user_highpage(struct page *to, struct page *from, 1066 unsigned long u_vaddr, struct vm_area_struct *vma) 1067 { 1068 void *kfrom = kmap_atomic(from); 1069 void *kto = kmap_atomic(to); 1070 int clean_src_k_mappings = 0; 1071 1072 /* 1073 * If SRC page was already mapped in userspace AND it's U-mapping is 1074 * not congruent with K-mapping, sync former to physical page so that 1075 * K-mapping in memcpy below, sees the right data 1076 * 1077 * Note that while @u_vaddr refers to DST page's userspace vaddr, it is 1078 * equally valid for SRC page as well 1079 * 1080 * For !VIPT cache, all of this gets compiled out as 1081 * addr_not_cache_congruent() is 0 1082 */ 1083 if (page_mapcount(from) && addr_not_cache_congruent(kfrom, u_vaddr)) { 1084 __flush_dcache_page((unsigned long)kfrom, u_vaddr); 1085 clean_src_k_mappings = 1; 1086 } 1087 1088 copy_page(kto, kfrom); 1089 1090 /* 1091 * Mark DST page K-mapping as dirty for a later finalization by 1092 * update_mmu_cache(). Although the finalization could have been done 1093 * here as well (given that both vaddr/paddr are available). 1094 * But update_mmu_cache() already has code to do that for other 1095 * non copied user pages (e.g. read faults which wire in pagecache page 1096 * directly). 1097 */ 1098 clear_bit(PG_dc_clean, &to->flags); 1099 1100 /* 1101 * if SRC was already usermapped and non-congruent to kernel mapping 1102 * sync the kernel mapping back to physical page 1103 */ 1104 if (clean_src_k_mappings) { 1105 __flush_dcache_page((unsigned long)kfrom, (unsigned long)kfrom); 1106 set_bit(PG_dc_clean, &from->flags); 1107 } else { 1108 clear_bit(PG_dc_clean, &from->flags); 1109 } 1110 1111 kunmap_atomic(kto); 1112 kunmap_atomic(kfrom); 1113 } 1114 1115 void clear_user_page(void *to, unsigned long u_vaddr, struct page *page) 1116 { 1117 clear_page(to); 1118 clear_bit(PG_dc_clean, &page->flags); 1119 } 1120 1121 1122 /********************************************************************** 1123 * Explicit Cache flush request from user space via syscall 1124 * Needed for JITs which generate code on the fly 1125 */ 1126 SYSCALL_DEFINE3(cacheflush, uint32_t, start, uint32_t, sz, uint32_t, flags) 1127 { 1128 /* TBD: optimize this */ 1129 flush_cache_all(); 1130 return 0; 1131 } 1132 1133 /* 1134 * IO-Coherency (IOC) setup rules: 1135 * 1136 * 1. Needs to be at system level, so only once by Master core 1137 * Non-Masters need not be accessing caches at that time 1138 * - They are either HALT_ON_RESET and kick started much later or 1139 * - if run on reset, need to ensure that arc_platform_smp_wait_to_boot() 1140 * doesn't perturb caches or coherency unit 1141 * 1142 * 2. caches (L1 and SLC) need to be purged (flush+inv) before setting up IOC, 1143 * otherwise any straggler data might behave strangely post IOC enabling 1144 * 1145 * 3. All Caches need to be disabled when setting up IOC to elide any in-flight 1146 * Coherency transactions 1147 */ 1148 noinline void __init arc_ioc_setup(void) 1149 { 1150 unsigned int ioc_base, mem_sz; 1151 1152 /* Flush + invalidate + disable L1 dcache */ 1153 __dc_disable(); 1154 1155 /* Flush + invalidate SLC */ 1156 if (read_aux_reg(ARC_REG_SLC_BCR)) 1157 slc_entire_op(OP_FLUSH_N_INV); 1158 1159 /* 1160 * currently IOC Aperture covers entire DDR 1161 * TBD: fix for PGU + 1GB of low mem 1162 * TBD: fix for PAE 1163 */ 1164 mem_sz = arc_get_mem_sz(); 1165 1166 if (!is_power_of_2(mem_sz) || mem_sz < 4096) 1167 panic("IOC Aperture size must be power of 2 larger than 4KB"); 1168 1169 /* 1170 * IOC Aperture size decoded as 2 ^ (SIZE + 2) KB, 1171 * so setting 0x11 implies 512MB, 0x12 implies 1GB... 1172 */ 1173 write_aux_reg(ARC_REG_IO_COH_AP0_SIZE, order_base_2(mem_sz >> 10) - 2); 1174 1175 /* for now assume kernel base is start of IOC aperture */ 1176 ioc_base = CONFIG_LINUX_RAM_BASE; 1177 1178 if (ioc_base % mem_sz != 0) 1179 panic("IOC Aperture start must be aligned to the size of the aperture"); 1180 1181 write_aux_reg(ARC_REG_IO_COH_AP0_BASE, ioc_base >> 12); 1182 write_aux_reg(ARC_REG_IO_COH_PARTIAL, 1); 1183 write_aux_reg(ARC_REG_IO_COH_ENABLE, 1); 1184 1185 /* Re-enable L1 dcache */ 1186 __dc_enable(); 1187 } 1188 1189 /* 1190 * Cache related boot time checks/setups only needed on master CPU: 1191 * - Geometry checks (kernel build and hardware agree: e.g. L1_CACHE_BYTES) 1192 * Assume SMP only, so all cores will have same cache config. A check on 1193 * one core suffices for all 1194 * - IOC setup / dma callbacks only need to be done once 1195 */ 1196 void __init arc_cache_init_master(void) 1197 { 1198 unsigned int __maybe_unused cpu = smp_processor_id(); 1199 1200 if (IS_ENABLED(CONFIG_ARC_HAS_ICACHE)) { 1201 struct cpuinfo_arc_cache *ic = &cpuinfo_arc700[cpu].icache; 1202 1203 if (!ic->line_len) 1204 panic("cache support enabled but non-existent cache\n"); 1205 1206 if (ic->line_len != L1_CACHE_BYTES) 1207 panic("ICache line [%d] != kernel Config [%d]", 1208 ic->line_len, L1_CACHE_BYTES); 1209 1210 /* 1211 * In MMU v4 (HS38x) the aliasing icache config uses IVIL/PTAG 1212 * pair to provide vaddr/paddr respectively, just as in MMU v3 1213 */ 1214 if (is_isa_arcv2() && ic->alias) 1215 _cache_line_loop_ic_fn = __cache_line_loop_v3; 1216 else 1217 _cache_line_loop_ic_fn = __cache_line_loop; 1218 } 1219 1220 if (IS_ENABLED(CONFIG_ARC_HAS_DCACHE)) { 1221 struct cpuinfo_arc_cache *dc = &cpuinfo_arc700[cpu].dcache; 1222 1223 if (!dc->line_len) 1224 panic("cache support enabled but non-existent cache\n"); 1225 1226 if (dc->line_len != L1_CACHE_BYTES) 1227 panic("DCache line [%d] != kernel Config [%d]", 1228 dc->line_len, L1_CACHE_BYTES); 1229 1230 /* check for D-Cache aliasing on ARCompact: ARCv2 has PIPT */ 1231 if (is_isa_arcompact()) { 1232 int handled = IS_ENABLED(CONFIG_ARC_CACHE_VIPT_ALIASING); 1233 int num_colors = dc->sz_k/dc->assoc/TO_KB(PAGE_SIZE); 1234 1235 if (dc->alias) { 1236 if (!handled) 1237 panic("Enable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); 1238 if (CACHE_COLORS_NUM != num_colors) 1239 panic("CACHE_COLORS_NUM not optimized for config\n"); 1240 } else if (!dc->alias && handled) { 1241 panic("Disable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); 1242 } 1243 } 1244 } 1245 1246 /* Note that SLC disable not formally supported till HS 3.0 */ 1247 if (is_isa_arcv2() && l2_line_sz && !slc_enable) 1248 arc_slc_disable(); 1249 1250 if (is_isa_arcv2() && ioc_enable) 1251 arc_ioc_setup(); 1252 1253 if (is_isa_arcv2() && ioc_enable) { 1254 __dma_cache_wback_inv = __dma_cache_wback_inv_ioc; 1255 __dma_cache_inv = __dma_cache_inv_ioc; 1256 __dma_cache_wback = __dma_cache_wback_ioc; 1257 } else if (is_isa_arcv2() && l2_line_sz && slc_enable) { 1258 __dma_cache_wback_inv = __dma_cache_wback_inv_slc; 1259 __dma_cache_inv = __dma_cache_inv_slc; 1260 __dma_cache_wback = __dma_cache_wback_slc; 1261 } else { 1262 __dma_cache_wback_inv = __dma_cache_wback_inv_l1; 1263 __dma_cache_inv = __dma_cache_inv_l1; 1264 __dma_cache_wback = __dma_cache_wback_l1; 1265 } 1266 } 1267 1268 void __ref arc_cache_init(void) 1269 { 1270 unsigned int __maybe_unused cpu = smp_processor_id(); 1271 char str[256]; 1272 1273 pr_info("%s", arc_cache_mumbojumbo(0, str, sizeof(str))); 1274 1275 if (!cpu) 1276 arc_cache_init_master(); 1277 1278 /* 1279 * In PAE regime, TLB and cache maintenance ops take wider addresses 1280 * And even if PAE is not enabled in kernel, the upper 32-bits still need 1281 * to be zeroed to keep the ops sane. 1282 * As an optimization for more common !PAE enabled case, zero them out 1283 * once at init, rather than checking/setting to 0 for every runtime op 1284 */ 1285 if (is_isa_arcv2() && pae40_exist_but_not_enab()) { 1286 1287 if (IS_ENABLED(CONFIG_ARC_HAS_ICACHE)) 1288 write_aux_reg(ARC_REG_IC_PTAG_HI, 0); 1289 1290 if (IS_ENABLED(CONFIG_ARC_HAS_DCACHE)) 1291 write_aux_reg(ARC_REG_DC_PTAG_HI, 0); 1292 1293 if (l2_line_sz) { 1294 write_aux_reg(ARC_REG_SLC_RGN_END1, 0); 1295 write_aux_reg(ARC_REG_SLC_RGN_START1, 0); 1296 } 1297 } 1298 } 1299