1 /* 2 * ARC Cache Management 3 * 4 * Copyright (C) 2014-15 Synopsys, Inc. (www.synopsys.com) 5 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 */ 11 12 #include <linux/module.h> 13 #include <linux/mm.h> 14 #include <linux/sched.h> 15 #include <linux/cache.h> 16 #include <linux/mmu_context.h> 17 #include <linux/syscalls.h> 18 #include <linux/uaccess.h> 19 #include <linux/pagemap.h> 20 #include <asm/cacheflush.h> 21 #include <asm/cachectl.h> 22 #include <asm/setup.h> 23 24 #ifdef CONFIG_ISA_ARCV2 25 #define USE_RGN_FLSH 1 26 #endif 27 28 static int l2_line_sz; 29 static int ioc_exists; 30 int slc_enable = 1, ioc_enable = 1; 31 unsigned long perip_base = ARC_UNCACHED_ADDR_SPACE; /* legacy value for boot */ 32 unsigned long perip_end = 0xFFFFFFFF; /* legacy value */ 33 34 void (*_cache_line_loop_ic_fn)(phys_addr_t paddr, unsigned long vaddr, 35 unsigned long sz, const int op, const int full_page); 36 37 void (*__dma_cache_wback_inv)(phys_addr_t start, unsigned long sz); 38 void (*__dma_cache_inv)(phys_addr_t start, unsigned long sz); 39 void (*__dma_cache_wback)(phys_addr_t start, unsigned long sz); 40 41 char *arc_cache_mumbojumbo(int c, char *buf, int len) 42 { 43 int n = 0; 44 struct cpuinfo_arc_cache *p; 45 46 #define PR_CACHE(p, cfg, str) \ 47 if (!(p)->line_len) \ 48 n += scnprintf(buf + n, len - n, str"\t\t: N/A\n"); \ 49 else \ 50 n += scnprintf(buf + n, len - n, \ 51 str"\t\t: %uK, %dway/set, %uB Line, %s%s%s\n", \ 52 (p)->sz_k, (p)->assoc, (p)->line_len, \ 53 (p)->vipt ? "VIPT" : "PIPT", \ 54 (p)->alias ? " aliasing" : "", \ 55 IS_USED_CFG(cfg)); 56 57 PR_CACHE(&cpuinfo_arc700[c].icache, CONFIG_ARC_HAS_ICACHE, "I-Cache"); 58 PR_CACHE(&cpuinfo_arc700[c].dcache, CONFIG_ARC_HAS_DCACHE, "D-Cache"); 59 60 p = &cpuinfo_arc700[c].slc; 61 if (p->line_len) 62 n += scnprintf(buf + n, len - n, 63 "SLC\t\t: %uK, %uB Line%s\n", 64 p->sz_k, p->line_len, IS_USED_RUN(slc_enable)); 65 66 n += scnprintf(buf + n, len - n, "Peripherals\t: %#lx%s%s\n", 67 perip_base, 68 IS_AVAIL3(ioc_exists, ioc_enable, ", IO-Coherency ")); 69 70 return buf; 71 } 72 73 /* 74 * Read the Cache Build Confuration Registers, Decode them and save into 75 * the cpuinfo structure for later use. 76 * No Validation done here, simply read/convert the BCRs 77 */ 78 static void read_decode_cache_bcr_arcv2(int cpu) 79 { 80 struct cpuinfo_arc_cache *p_slc = &cpuinfo_arc700[cpu].slc; 81 struct bcr_generic sbcr; 82 83 struct bcr_slc_cfg { 84 #ifdef CONFIG_CPU_BIG_ENDIAN 85 unsigned int pad:24, way:2, lsz:2, sz:4; 86 #else 87 unsigned int sz:4, lsz:2, way:2, pad:24; 88 #endif 89 } slc_cfg; 90 91 struct bcr_clust_cfg { 92 #ifdef CONFIG_CPU_BIG_ENDIAN 93 unsigned int pad:7, c:1, num_entries:8, num_cores:8, ver:8; 94 #else 95 unsigned int ver:8, num_cores:8, num_entries:8, c:1, pad:7; 96 #endif 97 } cbcr; 98 99 struct bcr_volatile { 100 #ifdef CONFIG_CPU_BIG_ENDIAN 101 unsigned int start:4, limit:4, pad:22, order:1, disable:1; 102 #else 103 unsigned int disable:1, order:1, pad:22, limit:4, start:4; 104 #endif 105 } vol; 106 107 108 READ_BCR(ARC_REG_SLC_BCR, sbcr); 109 if (sbcr.ver) { 110 READ_BCR(ARC_REG_SLC_CFG, slc_cfg); 111 p_slc->sz_k = 128 << slc_cfg.sz; 112 l2_line_sz = p_slc->line_len = (slc_cfg.lsz == 0) ? 128 : 64; 113 } 114 115 READ_BCR(ARC_REG_CLUSTER_BCR, cbcr); 116 if (cbcr.c) 117 ioc_exists = 1; 118 else 119 ioc_enable = 0; 120 121 /* HS 2.0 didn't have AUX_VOL */ 122 if (cpuinfo_arc700[cpu].core.family > 0x51) { 123 READ_BCR(AUX_VOL, vol); 124 perip_base = vol.start << 28; 125 /* HS 3.0 has limit and strict-ordering fields */ 126 if (cpuinfo_arc700[cpu].core.family > 0x52) 127 perip_end = (vol.limit << 28) - 1; 128 } 129 } 130 131 void read_decode_cache_bcr(void) 132 { 133 struct cpuinfo_arc_cache *p_ic, *p_dc; 134 unsigned int cpu = smp_processor_id(); 135 struct bcr_cache { 136 #ifdef CONFIG_CPU_BIG_ENDIAN 137 unsigned int pad:12, line_len:4, sz:4, config:4, ver:8; 138 #else 139 unsigned int ver:8, config:4, sz:4, line_len:4, pad:12; 140 #endif 141 } ibcr, dbcr; 142 143 p_ic = &cpuinfo_arc700[cpu].icache; 144 READ_BCR(ARC_REG_IC_BCR, ibcr); 145 146 if (!ibcr.ver) 147 goto dc_chk; 148 149 if (ibcr.ver <= 3) { 150 BUG_ON(ibcr.config != 3); 151 p_ic->assoc = 2; /* Fixed to 2w set assoc */ 152 } else if (ibcr.ver >= 4) { 153 p_ic->assoc = 1 << ibcr.config; /* 1,2,4,8 */ 154 } 155 156 p_ic->line_len = 8 << ibcr.line_len; 157 p_ic->sz_k = 1 << (ibcr.sz - 1); 158 p_ic->vipt = 1; 159 p_ic->alias = p_ic->sz_k/p_ic->assoc/TO_KB(PAGE_SIZE) > 1; 160 161 dc_chk: 162 p_dc = &cpuinfo_arc700[cpu].dcache; 163 READ_BCR(ARC_REG_DC_BCR, dbcr); 164 165 if (!dbcr.ver) 166 goto slc_chk; 167 168 if (dbcr.ver <= 3) { 169 BUG_ON(dbcr.config != 2); 170 p_dc->assoc = 4; /* Fixed to 4w set assoc */ 171 p_dc->vipt = 1; 172 p_dc->alias = p_dc->sz_k/p_dc->assoc/TO_KB(PAGE_SIZE) > 1; 173 } else if (dbcr.ver >= 4) { 174 p_dc->assoc = 1 << dbcr.config; /* 1,2,4,8 */ 175 p_dc->vipt = 0; 176 p_dc->alias = 0; /* PIPT so can't VIPT alias */ 177 } 178 179 p_dc->line_len = 16 << dbcr.line_len; 180 p_dc->sz_k = 1 << (dbcr.sz - 1); 181 182 slc_chk: 183 if (is_isa_arcv2()) 184 read_decode_cache_bcr_arcv2(cpu); 185 } 186 187 /* 188 * Line Operation on {I,D}-Cache 189 */ 190 191 #define OP_INV 0x1 192 #define OP_FLUSH 0x2 193 #define OP_FLUSH_N_INV 0x3 194 #define OP_INV_IC 0x4 195 196 /* 197 * I-Cache Aliasing in ARC700 VIPT caches (MMU v1-v3) 198 * 199 * ARC VIPT I-cache uses vaddr to index into cache and paddr to match the tag. 200 * The orig Cache Management Module "CDU" only required paddr to invalidate a 201 * certain line since it sufficed as index in Non-Aliasing VIPT cache-geometry. 202 * Infact for distinct V1,V2,P: all of {V1-P},{V2-P},{P-P} would end up fetching 203 * the exact same line. 204 * 205 * However for larger Caches (way-size > page-size) - i.e. in Aliasing config, 206 * paddr alone could not be used to correctly index the cache. 207 * 208 * ------------------ 209 * MMU v1/v2 (Fixed Page Size 8k) 210 * ------------------ 211 * The solution was to provide CDU with these additonal vaddr bits. These 212 * would be bits [x:13], x would depend on cache-geometry, 13 comes from 213 * standard page size of 8k. 214 * H/w folks chose [17:13] to be a future safe range, and moreso these 5 bits 215 * of vaddr could easily be "stuffed" in the paddr as bits [4:0] since the 216 * orig 5 bits of paddr were anyways ignored by CDU line ops, as they 217 * represent the offset within cache-line. The adv of using this "clumsy" 218 * interface for additional info was no new reg was needed in CDU programming 219 * model. 220 * 221 * 17:13 represented the max num of bits passable, actual bits needed were 222 * fewer, based on the num-of-aliases possible. 223 * -for 2 alias possibility, only bit 13 needed (32K cache) 224 * -for 4 alias possibility, bits 14:13 needed (64K cache) 225 * 226 * ------------------ 227 * MMU v3 228 * ------------------ 229 * This ver of MMU supports variable page sizes (1k-16k): although Linux will 230 * only support 8k (default), 16k and 4k. 231 * However from hardware perspective, smaller page sizes aggravate aliasing 232 * meaning more vaddr bits needed to disambiguate the cache-line-op ; 233 * the existing scheme of piggybacking won't work for certain configurations. 234 * Two new registers IC_PTAG and DC_PTAG inttoduced. 235 * "tag" bits are provided in PTAG, index bits in existing IVIL/IVDL/FLDL regs 236 */ 237 238 static inline 239 void __cache_line_loop_v2(phys_addr_t paddr, unsigned long vaddr, 240 unsigned long sz, const int op, const int full_page) 241 { 242 unsigned int aux_cmd; 243 int num_lines; 244 245 if (op == OP_INV_IC) { 246 aux_cmd = ARC_REG_IC_IVIL; 247 } else { 248 /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 249 aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 250 } 251 252 /* Ensure we properly floor/ceil the non-line aligned/sized requests 253 * and have @paddr - aligned to cache line and integral @num_lines. 254 * This however can be avoided for page sized since: 255 * -@paddr will be cache-line aligned already (being page aligned) 256 * -@sz will be integral multiple of line size (being page sized). 257 */ 258 if (!full_page) { 259 sz += paddr & ~CACHE_LINE_MASK; 260 paddr &= CACHE_LINE_MASK; 261 vaddr &= CACHE_LINE_MASK; 262 } 263 264 num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 265 266 /* MMUv2 and before: paddr contains stuffed vaddrs bits */ 267 paddr |= (vaddr >> PAGE_SHIFT) & 0x1F; 268 269 while (num_lines-- > 0) { 270 write_aux_reg(aux_cmd, paddr); 271 paddr += L1_CACHE_BYTES; 272 } 273 } 274 275 /* 276 * For ARC700 MMUv3 I-cache and D-cache flushes 277 * - ARC700 programming model requires paddr and vaddr be passed in seperate 278 * AUX registers (*_IV*L and *_PTAG respectively) irrespective of whether the 279 * caches actually alias or not. 280 * - For HS38, only the aliasing I-cache configuration uses the PTAG reg 281 * (non aliasing I-cache version doesn't; while D-cache can't possibly alias) 282 */ 283 static inline 284 void __cache_line_loop_v3(phys_addr_t paddr, unsigned long vaddr, 285 unsigned long sz, const int op, const int full_page) 286 { 287 unsigned int aux_cmd, aux_tag; 288 int num_lines; 289 290 if (op == OP_INV_IC) { 291 aux_cmd = ARC_REG_IC_IVIL; 292 aux_tag = ARC_REG_IC_PTAG; 293 } else { 294 aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 295 aux_tag = ARC_REG_DC_PTAG; 296 } 297 298 /* Ensure we properly floor/ceil the non-line aligned/sized requests 299 * and have @paddr - aligned to cache line and integral @num_lines. 300 * This however can be avoided for page sized since: 301 * -@paddr will be cache-line aligned already (being page aligned) 302 * -@sz will be integral multiple of line size (being page sized). 303 */ 304 if (!full_page) { 305 sz += paddr & ~CACHE_LINE_MASK; 306 paddr &= CACHE_LINE_MASK; 307 vaddr &= CACHE_LINE_MASK; 308 } 309 num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 310 311 /* 312 * MMUv3, cache ops require paddr in PTAG reg 313 * if V-P const for loop, PTAG can be written once outside loop 314 */ 315 if (full_page) 316 write_aux_reg(aux_tag, paddr); 317 318 /* 319 * This is technically for MMU v4, using the MMU v3 programming model 320 * Special work for HS38 aliasing I-cache configuration with PAE40 321 * - upper 8 bits of paddr need to be written into PTAG_HI 322 * - (and needs to be written before the lower 32 bits) 323 * Note that PTAG_HI is hoisted outside the line loop 324 */ 325 if (is_pae40_enabled() && op == OP_INV_IC) 326 write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 327 328 while (num_lines-- > 0) { 329 if (!full_page) { 330 write_aux_reg(aux_tag, paddr); 331 paddr += L1_CACHE_BYTES; 332 } 333 334 write_aux_reg(aux_cmd, vaddr); 335 vaddr += L1_CACHE_BYTES; 336 } 337 } 338 339 #ifndef USE_RGN_FLSH 340 341 /* 342 * In HS38x (MMU v4), I-cache is VIPT (can alias), D-cache is PIPT 343 * Here's how cache ops are implemented 344 * 345 * - D-cache: only paddr needed (in DC_IVDL/DC_FLDL) 346 * - I-cache Non Aliasing: Despite VIPT, only paddr needed (in IC_IVIL) 347 * - I-cache Aliasing: Both vaddr and paddr needed (in IC_IVIL, IC_PTAG 348 * respectively, similar to MMU v3 programming model, hence 349 * __cache_line_loop_v3() is used) 350 * 351 * If PAE40 is enabled, independent of aliasing considerations, the higher bits 352 * needs to be written into PTAG_HI 353 */ 354 static inline 355 void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, 356 unsigned long sz, const int op, const int full_page) 357 { 358 unsigned int aux_cmd; 359 int num_lines; 360 361 if (op == OP_INV_IC) { 362 aux_cmd = ARC_REG_IC_IVIL; 363 } else { 364 /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 365 aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 366 } 367 368 /* Ensure we properly floor/ceil the non-line aligned/sized requests 369 * and have @paddr - aligned to cache line and integral @num_lines. 370 * This however can be avoided for page sized since: 371 * -@paddr will be cache-line aligned already (being page aligned) 372 * -@sz will be integral multiple of line size (being page sized). 373 */ 374 if (!full_page) { 375 sz += paddr & ~CACHE_LINE_MASK; 376 paddr &= CACHE_LINE_MASK; 377 } 378 379 num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 380 381 /* 382 * For HS38 PAE40 configuration 383 * - upper 8 bits of paddr need to be written into PTAG_HI 384 * - (and needs to be written before the lower 32 bits) 385 */ 386 if (is_pae40_enabled()) { 387 if (op == OP_INV_IC) 388 /* 389 * Non aliasing I-cache in HS38, 390 * aliasing I-cache handled in __cache_line_loop_v3() 391 */ 392 write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 393 else 394 write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32); 395 } 396 397 while (num_lines-- > 0) { 398 write_aux_reg(aux_cmd, paddr); 399 paddr += L1_CACHE_BYTES; 400 } 401 } 402 403 #else 404 405 /* 406 * optimized flush operation which takes a region as opposed to iterating per line 407 */ 408 static inline 409 void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, 410 unsigned long sz, const int op, const int full_page) 411 { 412 unsigned int s, e; 413 414 /* Only for Non aliasing I-cache in HS38 */ 415 if (op == OP_INV_IC) { 416 s = ARC_REG_IC_IVIR; 417 e = ARC_REG_IC_ENDR; 418 } else { 419 s = ARC_REG_DC_STARTR; 420 e = ARC_REG_DC_ENDR; 421 } 422 423 if (!full_page) { 424 /* for any leading gap between @paddr and start of cache line */ 425 sz += paddr & ~CACHE_LINE_MASK; 426 paddr &= CACHE_LINE_MASK; 427 428 /* 429 * account for any trailing gap to end of cache line 430 * this is equivalent to DIV_ROUND_UP() in line ops above 431 */ 432 sz += L1_CACHE_BYTES - 1; 433 } 434 435 if (is_pae40_enabled()) { 436 /* TBD: check if crossing 4TB boundary */ 437 if (op == OP_INV_IC) 438 write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 439 else 440 write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32); 441 } 442 443 /* ENDR needs to be set ahead of START */ 444 write_aux_reg(e, paddr + sz); /* ENDR is exclusive */ 445 write_aux_reg(s, paddr); 446 447 /* caller waits on DC_CTRL.FS */ 448 } 449 450 #endif 451 452 #if (CONFIG_ARC_MMU_VER < 3) 453 #define __cache_line_loop __cache_line_loop_v2 454 #elif (CONFIG_ARC_MMU_VER == 3) 455 #define __cache_line_loop __cache_line_loop_v3 456 #elif (CONFIG_ARC_MMU_VER > 3) 457 #define __cache_line_loop __cache_line_loop_v4 458 #endif 459 460 #ifdef CONFIG_ARC_HAS_DCACHE 461 462 /*************************************************************** 463 * Machine specific helpers for Entire D-Cache or Per Line ops 464 */ 465 466 #ifndef USE_RGN_FLSH 467 /* 468 * this version avoids extra read/write of DC_CTRL for flush or invalid ops 469 * in the non region flush regime (such as for ARCompact) 470 */ 471 static inline void __before_dc_op(const int op) 472 { 473 if (op == OP_FLUSH_N_INV) { 474 /* Dcache provides 2 cmd: FLUSH or INV 475 * INV inturn has sub-modes: DISCARD or FLUSH-BEFORE 476 * flush-n-inv is achieved by INV cmd but with IM=1 477 * So toggle INV sub-mode depending on op request and default 478 */ 479 const unsigned int ctl = ARC_REG_DC_CTRL; 480 write_aux_reg(ctl, read_aux_reg(ctl) | DC_CTRL_INV_MODE_FLUSH); 481 } 482 } 483 484 #else 485 486 static inline void __before_dc_op(const int op) 487 { 488 const unsigned int ctl = ARC_REG_DC_CTRL; 489 unsigned int val = read_aux_reg(ctl); 490 491 if (op == OP_FLUSH_N_INV) { 492 val |= DC_CTRL_INV_MODE_FLUSH; 493 } 494 495 if (op != OP_INV_IC) { 496 /* 497 * Flush / Invalidate is provided by DC_CTRL.RNG_OP 0 or 1 498 * combined Flush-n-invalidate uses DC_CTRL.IM = 1 set above 499 */ 500 val &= ~DC_CTRL_RGN_OP_MSK; 501 if (op & OP_INV) 502 val |= DC_CTRL_RGN_OP_INV; 503 } 504 write_aux_reg(ctl, val); 505 } 506 507 #endif 508 509 510 static inline void __after_dc_op(const int op) 511 { 512 if (op & OP_FLUSH) { 513 const unsigned int ctl = ARC_REG_DC_CTRL; 514 unsigned int reg; 515 516 /* flush / flush-n-inv both wait */ 517 while ((reg = read_aux_reg(ctl)) & DC_CTRL_FLUSH_STATUS) 518 ; 519 520 /* Switch back to default Invalidate mode */ 521 if (op == OP_FLUSH_N_INV) 522 write_aux_reg(ctl, reg & ~DC_CTRL_INV_MODE_FLUSH); 523 } 524 } 525 526 /* 527 * Operation on Entire D-Cache 528 * @op = {OP_INV, OP_FLUSH, OP_FLUSH_N_INV} 529 * Note that constant propagation ensures all the checks are gone 530 * in generated code 531 */ 532 static inline void __dc_entire_op(const int op) 533 { 534 int aux; 535 536 __before_dc_op(op); 537 538 if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 539 aux = ARC_REG_DC_IVDC; 540 else 541 aux = ARC_REG_DC_FLSH; 542 543 write_aux_reg(aux, 0x1); 544 545 __after_dc_op(op); 546 } 547 548 static inline void __dc_disable(void) 549 { 550 const int r = ARC_REG_DC_CTRL; 551 552 __dc_entire_op(OP_FLUSH_N_INV); 553 write_aux_reg(r, read_aux_reg(r) | DC_CTRL_DIS); 554 } 555 556 static void __dc_enable(void) 557 { 558 const int r = ARC_REG_DC_CTRL; 559 560 write_aux_reg(r, read_aux_reg(r) & ~DC_CTRL_DIS); 561 } 562 563 /* For kernel mappings cache operation: index is same as paddr */ 564 #define __dc_line_op_k(p, sz, op) __dc_line_op(p, p, sz, op) 565 566 /* 567 * D-Cache Line ops: Per Line INV (discard or wback+discard) or FLUSH (wback) 568 */ 569 static inline void __dc_line_op(phys_addr_t paddr, unsigned long vaddr, 570 unsigned long sz, const int op) 571 { 572 const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; 573 unsigned long flags; 574 575 local_irq_save(flags); 576 577 __before_dc_op(op); 578 579 __cache_line_loop(paddr, vaddr, sz, op, full_page); 580 581 __after_dc_op(op); 582 583 local_irq_restore(flags); 584 } 585 586 #else 587 588 #define __dc_entire_op(op) 589 #define __dc_disable() 590 #define __dc_enable() 591 #define __dc_line_op(paddr, vaddr, sz, op) 592 #define __dc_line_op_k(paddr, sz, op) 593 594 #endif /* CONFIG_ARC_HAS_DCACHE */ 595 596 #ifdef CONFIG_ARC_HAS_ICACHE 597 598 static inline void __ic_entire_inv(void) 599 { 600 write_aux_reg(ARC_REG_IC_IVIC, 1); 601 read_aux_reg(ARC_REG_IC_CTRL); /* blocks */ 602 } 603 604 static inline void 605 __ic_line_inv_vaddr_local(phys_addr_t paddr, unsigned long vaddr, 606 unsigned long sz) 607 { 608 const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; 609 unsigned long flags; 610 611 local_irq_save(flags); 612 (*_cache_line_loop_ic_fn)(paddr, vaddr, sz, OP_INV_IC, full_page); 613 local_irq_restore(flags); 614 } 615 616 #ifndef CONFIG_SMP 617 618 #define __ic_line_inv_vaddr(p, v, s) __ic_line_inv_vaddr_local(p, v, s) 619 620 #else 621 622 struct ic_inv_args { 623 phys_addr_t paddr, vaddr; 624 int sz; 625 }; 626 627 static void __ic_line_inv_vaddr_helper(void *info) 628 { 629 struct ic_inv_args *ic_inv = info; 630 631 __ic_line_inv_vaddr_local(ic_inv->paddr, ic_inv->vaddr, ic_inv->sz); 632 } 633 634 static void __ic_line_inv_vaddr(phys_addr_t paddr, unsigned long vaddr, 635 unsigned long sz) 636 { 637 struct ic_inv_args ic_inv = { 638 .paddr = paddr, 639 .vaddr = vaddr, 640 .sz = sz 641 }; 642 643 on_each_cpu(__ic_line_inv_vaddr_helper, &ic_inv, 1); 644 } 645 646 #endif /* CONFIG_SMP */ 647 648 #else /* !CONFIG_ARC_HAS_ICACHE */ 649 650 #define __ic_entire_inv() 651 #define __ic_line_inv_vaddr(pstart, vstart, sz) 652 653 #endif /* CONFIG_ARC_HAS_ICACHE */ 654 655 noinline void slc_op(phys_addr_t paddr, unsigned long sz, const int op) 656 { 657 #ifdef CONFIG_ISA_ARCV2 658 /* 659 * SLC is shared between all cores and concurrent aux operations from 660 * multiple cores need to be serialized using a spinlock 661 * A concurrent operation can be silently ignored and/or the old/new 662 * operation can remain incomplete forever (lockup in SLC_CTRL_BUSY loop 663 * below) 664 */ 665 static DEFINE_SPINLOCK(lock); 666 unsigned long flags; 667 unsigned int ctrl; 668 669 spin_lock_irqsave(&lock, flags); 670 671 /* 672 * The Region Flush operation is specified by CTRL.RGN_OP[11..9] 673 * - b'000 (default) is Flush, 674 * - b'001 is Invalidate if CTRL.IM == 0 675 * - b'001 is Flush-n-Invalidate if CTRL.IM == 1 676 */ 677 ctrl = read_aux_reg(ARC_REG_SLC_CTRL); 678 679 /* Don't rely on default value of IM bit */ 680 if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 681 ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 682 else 683 ctrl |= SLC_CTRL_IM; 684 685 if (op & OP_INV) 686 ctrl |= SLC_CTRL_RGN_OP_INV; /* Inv or flush-n-inv */ 687 else 688 ctrl &= ~SLC_CTRL_RGN_OP_INV; 689 690 write_aux_reg(ARC_REG_SLC_CTRL, ctrl); 691 692 /* 693 * Lower bits are ignored, no need to clip 694 * END needs to be setup before START (latter triggers the operation) 695 * END can't be same as START, so add (l2_line_sz - 1) to sz 696 */ 697 write_aux_reg(ARC_REG_SLC_RGN_END, (paddr + sz + l2_line_sz - 1)); 698 write_aux_reg(ARC_REG_SLC_RGN_START, paddr); 699 700 while (read_aux_reg(ARC_REG_SLC_CTRL) & SLC_CTRL_BUSY); 701 702 spin_unlock_irqrestore(&lock, flags); 703 #endif 704 } 705 706 noinline static void slc_entire_op(const int op) 707 { 708 unsigned int ctrl, r = ARC_REG_SLC_CTRL; 709 710 ctrl = read_aux_reg(r); 711 712 if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 713 ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 714 else 715 ctrl |= SLC_CTRL_IM; 716 717 write_aux_reg(r, ctrl); 718 719 write_aux_reg(ARC_REG_SLC_INVALIDATE, 1); 720 721 /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ 722 read_aux_reg(r); 723 724 /* Important to wait for flush to complete */ 725 while (read_aux_reg(r) & SLC_CTRL_BUSY); 726 } 727 728 static inline void arc_slc_disable(void) 729 { 730 const int r = ARC_REG_SLC_CTRL; 731 732 slc_entire_op(OP_FLUSH_N_INV); 733 write_aux_reg(r, read_aux_reg(r) | SLC_CTRL_DIS); 734 } 735 736 static inline void arc_slc_enable(void) 737 { 738 const int r = ARC_REG_SLC_CTRL; 739 740 write_aux_reg(r, read_aux_reg(r) & ~SLC_CTRL_DIS); 741 } 742 743 /*********************************************************** 744 * Exported APIs 745 */ 746 747 /* 748 * Handle cache congruency of kernel and userspace mappings of page when kernel 749 * writes-to/reads-from 750 * 751 * The idea is to defer flushing of kernel mapping after a WRITE, possible if: 752 * -dcache is NOT aliasing, hence any U/K-mappings of page are congruent 753 * -U-mapping doesn't exist yet for page (finalised in update_mmu_cache) 754 * -In SMP, if hardware caches are coherent 755 * 756 * There's a corollary case, where kernel READs from a userspace mapped page. 757 * If the U-mapping is not congruent to to K-mapping, former needs flushing. 758 */ 759 void flush_dcache_page(struct page *page) 760 { 761 struct address_space *mapping; 762 763 if (!cache_is_vipt_aliasing()) { 764 clear_bit(PG_dc_clean, &page->flags); 765 return; 766 } 767 768 /* don't handle anon pages here */ 769 mapping = page_mapping(page); 770 if (!mapping) 771 return; 772 773 /* 774 * pagecache page, file not yet mapped to userspace 775 * Make a note that K-mapping is dirty 776 */ 777 if (!mapping_mapped(mapping)) { 778 clear_bit(PG_dc_clean, &page->flags); 779 } else if (page_mapcount(page)) { 780 781 /* kernel reading from page with U-mapping */ 782 phys_addr_t paddr = (unsigned long)page_address(page); 783 unsigned long vaddr = page->index << PAGE_SHIFT; 784 785 if (addr_not_cache_congruent(paddr, vaddr)) 786 __flush_dcache_page(paddr, vaddr); 787 } 788 } 789 EXPORT_SYMBOL(flush_dcache_page); 790 791 /* 792 * DMA ops for systems with L1 cache only 793 * Make memory coherent with L1 cache by flushing/invalidating L1 lines 794 */ 795 static void __dma_cache_wback_inv_l1(phys_addr_t start, unsigned long sz) 796 { 797 __dc_line_op_k(start, sz, OP_FLUSH_N_INV); 798 } 799 800 static void __dma_cache_inv_l1(phys_addr_t start, unsigned long sz) 801 { 802 __dc_line_op_k(start, sz, OP_INV); 803 } 804 805 static void __dma_cache_wback_l1(phys_addr_t start, unsigned long sz) 806 { 807 __dc_line_op_k(start, sz, OP_FLUSH); 808 } 809 810 /* 811 * DMA ops for systems with both L1 and L2 caches, but without IOC 812 * Both L1 and L2 lines need to be explicitly flushed/invalidated 813 */ 814 static void __dma_cache_wback_inv_slc(phys_addr_t start, unsigned long sz) 815 { 816 __dc_line_op_k(start, sz, OP_FLUSH_N_INV); 817 slc_op(start, sz, OP_FLUSH_N_INV); 818 } 819 820 static void __dma_cache_inv_slc(phys_addr_t start, unsigned long sz) 821 { 822 __dc_line_op_k(start, sz, OP_INV); 823 slc_op(start, sz, OP_INV); 824 } 825 826 static void __dma_cache_wback_slc(phys_addr_t start, unsigned long sz) 827 { 828 __dc_line_op_k(start, sz, OP_FLUSH); 829 slc_op(start, sz, OP_FLUSH); 830 } 831 832 /* 833 * DMA ops for systems with IOC 834 * IOC hardware snoops all DMA traffic keeping the caches consistent with 835 * memory - eliding need for any explicit cache maintenance of DMA buffers 836 */ 837 static void __dma_cache_wback_inv_ioc(phys_addr_t start, unsigned long sz) {} 838 static void __dma_cache_inv_ioc(phys_addr_t start, unsigned long sz) {} 839 static void __dma_cache_wback_ioc(phys_addr_t start, unsigned long sz) {} 840 841 /* 842 * Exported DMA API 843 */ 844 void dma_cache_wback_inv(phys_addr_t start, unsigned long sz) 845 { 846 __dma_cache_wback_inv(start, sz); 847 } 848 EXPORT_SYMBOL(dma_cache_wback_inv); 849 850 void dma_cache_inv(phys_addr_t start, unsigned long sz) 851 { 852 __dma_cache_inv(start, sz); 853 } 854 EXPORT_SYMBOL(dma_cache_inv); 855 856 void dma_cache_wback(phys_addr_t start, unsigned long sz) 857 { 858 __dma_cache_wback(start, sz); 859 } 860 EXPORT_SYMBOL(dma_cache_wback); 861 862 /* 863 * This is API for making I/D Caches consistent when modifying 864 * kernel code (loadable modules, kprobes, kgdb...) 865 * This is called on insmod, with kernel virtual address for CODE of 866 * the module. ARC cache maintenance ops require PHY address thus we 867 * need to convert vmalloc addr to PHY addr 868 */ 869 void flush_icache_range(unsigned long kstart, unsigned long kend) 870 { 871 unsigned int tot_sz; 872 873 WARN(kstart < TASK_SIZE, "%s() can't handle user vaddr", __func__); 874 875 /* Shortcut for bigger flush ranges. 876 * Here we don't care if this was kernel virtual or phy addr 877 */ 878 tot_sz = kend - kstart; 879 if (tot_sz > PAGE_SIZE) { 880 flush_cache_all(); 881 return; 882 } 883 884 /* Case: Kernel Phy addr (0x8000_0000 onwards) */ 885 if (likely(kstart > PAGE_OFFSET)) { 886 /* 887 * The 2nd arg despite being paddr will be used to index icache 888 * This is OK since no alternate virtual mappings will exist 889 * given the callers for this case: kprobe/kgdb in built-in 890 * kernel code only. 891 */ 892 __sync_icache_dcache(kstart, kstart, kend - kstart); 893 return; 894 } 895 896 /* 897 * Case: Kernel Vaddr (0x7000_0000 to 0x7fff_ffff) 898 * (1) ARC Cache Maintenance ops only take Phy addr, hence special 899 * handling of kernel vaddr. 900 * 901 * (2) Despite @tot_sz being < PAGE_SIZE (bigger cases handled already), 902 * it still needs to handle a 2 page scenario, where the range 903 * straddles across 2 virtual pages and hence need for loop 904 */ 905 while (tot_sz > 0) { 906 unsigned int off, sz; 907 unsigned long phy, pfn; 908 909 off = kstart % PAGE_SIZE; 910 pfn = vmalloc_to_pfn((void *)kstart); 911 phy = (pfn << PAGE_SHIFT) + off; 912 sz = min_t(unsigned int, tot_sz, PAGE_SIZE - off); 913 __sync_icache_dcache(phy, kstart, sz); 914 kstart += sz; 915 tot_sz -= sz; 916 } 917 } 918 EXPORT_SYMBOL(flush_icache_range); 919 920 /* 921 * General purpose helper to make I and D cache lines consistent. 922 * @paddr is phy addr of region 923 * @vaddr is typically user vaddr (breakpoint) or kernel vaddr (vmalloc) 924 * However in one instance, when called by kprobe (for a breakpt in 925 * builtin kernel code) @vaddr will be paddr only, meaning CDU operation will 926 * use a paddr to index the cache (despite VIPT). This is fine since since a 927 * builtin kernel page will not have any virtual mappings. 928 * kprobe on loadable module will be kernel vaddr. 929 */ 930 void __sync_icache_dcache(phys_addr_t paddr, unsigned long vaddr, int len) 931 { 932 __dc_line_op(paddr, vaddr, len, OP_FLUSH_N_INV); 933 __ic_line_inv_vaddr(paddr, vaddr, len); 934 } 935 936 /* wrapper to compile time eliminate alignment checks in flush loop */ 937 void __inv_icache_page(phys_addr_t paddr, unsigned long vaddr) 938 { 939 __ic_line_inv_vaddr(paddr, vaddr, PAGE_SIZE); 940 } 941 942 /* 943 * wrapper to clearout kernel or userspace mappings of a page 944 * For kernel mappings @vaddr == @paddr 945 */ 946 void __flush_dcache_page(phys_addr_t paddr, unsigned long vaddr) 947 { 948 __dc_line_op(paddr, vaddr & PAGE_MASK, PAGE_SIZE, OP_FLUSH_N_INV); 949 } 950 951 noinline void flush_cache_all(void) 952 { 953 unsigned long flags; 954 955 local_irq_save(flags); 956 957 __ic_entire_inv(); 958 __dc_entire_op(OP_FLUSH_N_INV); 959 960 local_irq_restore(flags); 961 962 } 963 964 #ifdef CONFIG_ARC_CACHE_VIPT_ALIASING 965 966 void flush_cache_mm(struct mm_struct *mm) 967 { 968 flush_cache_all(); 969 } 970 971 void flush_cache_page(struct vm_area_struct *vma, unsigned long u_vaddr, 972 unsigned long pfn) 973 { 974 unsigned int paddr = pfn << PAGE_SHIFT; 975 976 u_vaddr &= PAGE_MASK; 977 978 __flush_dcache_page(paddr, u_vaddr); 979 980 if (vma->vm_flags & VM_EXEC) 981 __inv_icache_page(paddr, u_vaddr); 982 } 983 984 void flush_cache_range(struct vm_area_struct *vma, unsigned long start, 985 unsigned long end) 986 { 987 flush_cache_all(); 988 } 989 990 void flush_anon_page(struct vm_area_struct *vma, struct page *page, 991 unsigned long u_vaddr) 992 { 993 /* TBD: do we really need to clear the kernel mapping */ 994 __flush_dcache_page(page_address(page), u_vaddr); 995 __flush_dcache_page(page_address(page), page_address(page)); 996 997 } 998 999 #endif 1000 1001 void copy_user_highpage(struct page *to, struct page *from, 1002 unsigned long u_vaddr, struct vm_area_struct *vma) 1003 { 1004 void *kfrom = kmap_atomic(from); 1005 void *kto = kmap_atomic(to); 1006 int clean_src_k_mappings = 0; 1007 1008 /* 1009 * If SRC page was already mapped in userspace AND it's U-mapping is 1010 * not congruent with K-mapping, sync former to physical page so that 1011 * K-mapping in memcpy below, sees the right data 1012 * 1013 * Note that while @u_vaddr refers to DST page's userspace vaddr, it is 1014 * equally valid for SRC page as well 1015 * 1016 * For !VIPT cache, all of this gets compiled out as 1017 * addr_not_cache_congruent() is 0 1018 */ 1019 if (page_mapcount(from) && addr_not_cache_congruent(kfrom, u_vaddr)) { 1020 __flush_dcache_page((unsigned long)kfrom, u_vaddr); 1021 clean_src_k_mappings = 1; 1022 } 1023 1024 copy_page(kto, kfrom); 1025 1026 /* 1027 * Mark DST page K-mapping as dirty for a later finalization by 1028 * update_mmu_cache(). Although the finalization could have been done 1029 * here as well (given that both vaddr/paddr are available). 1030 * But update_mmu_cache() already has code to do that for other 1031 * non copied user pages (e.g. read faults which wire in pagecache page 1032 * directly). 1033 */ 1034 clear_bit(PG_dc_clean, &to->flags); 1035 1036 /* 1037 * if SRC was already usermapped and non-congruent to kernel mapping 1038 * sync the kernel mapping back to physical page 1039 */ 1040 if (clean_src_k_mappings) { 1041 __flush_dcache_page((unsigned long)kfrom, (unsigned long)kfrom); 1042 set_bit(PG_dc_clean, &from->flags); 1043 } else { 1044 clear_bit(PG_dc_clean, &from->flags); 1045 } 1046 1047 kunmap_atomic(kto); 1048 kunmap_atomic(kfrom); 1049 } 1050 1051 void clear_user_page(void *to, unsigned long u_vaddr, struct page *page) 1052 { 1053 clear_page(to); 1054 clear_bit(PG_dc_clean, &page->flags); 1055 } 1056 1057 1058 /********************************************************************** 1059 * Explicit Cache flush request from user space via syscall 1060 * Needed for JITs which generate code on the fly 1061 */ 1062 SYSCALL_DEFINE3(cacheflush, uint32_t, start, uint32_t, sz, uint32_t, flags) 1063 { 1064 /* TBD: optimize this */ 1065 flush_cache_all(); 1066 return 0; 1067 } 1068 1069 /* 1070 * IO-Coherency (IOC) setup rules: 1071 * 1072 * 1. Needs to be at system level, so only once by Master core 1073 * Non-Masters need not be accessing caches at that time 1074 * - They are either HALT_ON_RESET and kick started much later or 1075 * - if run on reset, need to ensure that arc_platform_smp_wait_to_boot() 1076 * doesn't perturb caches or coherency unit 1077 * 1078 * 2. caches (L1 and SLC) need to be purged (flush+inv) before setting up IOC, 1079 * otherwise any straggler data might behave strangely post IOC enabling 1080 * 1081 * 3. All Caches need to be disabled when setting up IOC to elide any in-flight 1082 * Coherency transactions 1083 */ 1084 noinline void __init arc_ioc_setup(void) 1085 { 1086 unsigned int ap_sz; 1087 1088 /* Flush + invalidate + disable L1 dcache */ 1089 __dc_disable(); 1090 1091 /* Flush + invalidate SLC */ 1092 if (read_aux_reg(ARC_REG_SLC_BCR)) 1093 slc_entire_op(OP_FLUSH_N_INV); 1094 1095 /* IOC Aperture start: TDB: handle non default CONFIG_LINUX_LINK_BASE */ 1096 write_aux_reg(ARC_REG_IO_COH_AP0_BASE, 0x80000); 1097 1098 /* 1099 * IOC Aperture size: 1100 * decoded as 2 ^ (SIZE + 2) KB: so setting 0x11 implies 512M 1101 * TBD: fix for PGU + 1GB of low mem 1102 * TBD: fix for PAE 1103 */ 1104 ap_sz = order_base_2(arc_get_mem_sz()/1024) - 2; 1105 write_aux_reg(ARC_REG_IO_COH_AP0_SIZE, ap_sz); 1106 1107 write_aux_reg(ARC_REG_IO_COH_PARTIAL, 1); 1108 write_aux_reg(ARC_REG_IO_COH_ENABLE, 1); 1109 1110 /* Re-enable L1 dcache */ 1111 __dc_enable(); 1112 } 1113 1114 void __init arc_cache_init_master(void) 1115 { 1116 unsigned int __maybe_unused cpu = smp_processor_id(); 1117 1118 if (IS_ENABLED(CONFIG_ARC_HAS_ICACHE)) { 1119 struct cpuinfo_arc_cache *ic = &cpuinfo_arc700[cpu].icache; 1120 1121 if (!ic->line_len) 1122 panic("cache support enabled but non-existent cache\n"); 1123 1124 if (ic->line_len != L1_CACHE_BYTES) 1125 panic("ICache line [%d] != kernel Config [%d]", 1126 ic->line_len, L1_CACHE_BYTES); 1127 1128 /* 1129 * In MMU v4 (HS38x) the aliasing icache config uses IVIL/PTAG 1130 * pair to provide vaddr/paddr respectively, just as in MMU v3 1131 */ 1132 if (is_isa_arcv2() && ic->alias) 1133 _cache_line_loop_ic_fn = __cache_line_loop_v3; 1134 else 1135 _cache_line_loop_ic_fn = __cache_line_loop; 1136 } 1137 1138 if (IS_ENABLED(CONFIG_ARC_HAS_DCACHE)) { 1139 struct cpuinfo_arc_cache *dc = &cpuinfo_arc700[cpu].dcache; 1140 1141 if (!dc->line_len) 1142 panic("cache support enabled but non-existent cache\n"); 1143 1144 if (dc->line_len != L1_CACHE_BYTES) 1145 panic("DCache line [%d] != kernel Config [%d]", 1146 dc->line_len, L1_CACHE_BYTES); 1147 1148 /* check for D-Cache aliasing on ARCompact: ARCv2 has PIPT */ 1149 if (is_isa_arcompact()) { 1150 int handled = IS_ENABLED(CONFIG_ARC_CACHE_VIPT_ALIASING); 1151 int num_colors = dc->sz_k/dc->assoc/TO_KB(PAGE_SIZE); 1152 1153 if (dc->alias) { 1154 if (!handled) 1155 panic("Enable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); 1156 if (CACHE_COLORS_NUM != num_colors) 1157 panic("CACHE_COLORS_NUM not optimized for config\n"); 1158 } else if (!dc->alias && handled) { 1159 panic("Disable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); 1160 } 1161 } 1162 } 1163 1164 /* Note that SLC disable not formally supported till HS 3.0 */ 1165 if (is_isa_arcv2() && l2_line_sz && !slc_enable) 1166 arc_slc_disable(); 1167 1168 if (is_isa_arcv2() && ioc_enable) 1169 arc_ioc_setup(); 1170 1171 if (is_isa_arcv2() && ioc_enable) { 1172 __dma_cache_wback_inv = __dma_cache_wback_inv_ioc; 1173 __dma_cache_inv = __dma_cache_inv_ioc; 1174 __dma_cache_wback = __dma_cache_wback_ioc; 1175 } else if (is_isa_arcv2() && l2_line_sz && slc_enable) { 1176 __dma_cache_wback_inv = __dma_cache_wback_inv_slc; 1177 __dma_cache_inv = __dma_cache_inv_slc; 1178 __dma_cache_wback = __dma_cache_wback_slc; 1179 } else { 1180 __dma_cache_wback_inv = __dma_cache_wback_inv_l1; 1181 __dma_cache_inv = __dma_cache_inv_l1; 1182 __dma_cache_wback = __dma_cache_wback_l1; 1183 } 1184 } 1185 1186 void __ref arc_cache_init(void) 1187 { 1188 unsigned int __maybe_unused cpu = smp_processor_id(); 1189 char str[256]; 1190 1191 printk(arc_cache_mumbojumbo(0, str, sizeof(str))); 1192 1193 /* 1194 * Only master CPU needs to execute rest of function: 1195 * - Assume SMP so all cores will have same cache config so 1196 * any geomtry checks will be same for all 1197 * - IOC setup / dma callbacks only need to be setup once 1198 */ 1199 if (!cpu) 1200 arc_cache_init_master(); 1201 } 1202