1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 1999-2006 Helge Deller <deller@gmx.de> (07-13-1999) 7 * Copyright (C) 1999 SuSE GmbH Nuernberg 8 * Copyright (C) 2000 Philipp Rumpf (prumpf@tux.org) 9 * 10 * Cache and TLB management 11 * 12 */ 13 14 #include <linux/init.h> 15 #include <linux/kernel.h> 16 #include <linux/mm.h> 17 #include <linux/module.h> 18 #include <linux/seq_file.h> 19 #include <linux/pagemap.h> 20 #include <linux/sched.h> 21 #include <linux/sched/mm.h> 22 #include <linux/syscalls.h> 23 #include <linux/vmalloc.h> 24 #include <asm/pdc.h> 25 #include <asm/cache.h> 26 #include <asm/cacheflush.h> 27 #include <asm/tlbflush.h> 28 #include <asm/page.h> 29 #include <asm/processor.h> 30 #include <asm/sections.h> 31 #include <asm/shmparam.h> 32 #include <asm/mmu_context.h> 33 #include <asm/cachectl.h> 34 35 #define PTR_PAGE_ALIGN_DOWN(addr) PTR_ALIGN_DOWN(addr, PAGE_SIZE) 36 37 /* 38 * When nonzero, use _PAGE_ACCESSED bit to try to reduce the number 39 * of page flushes done flush_cache_page_if_present. There are some 40 * pros and cons in using this option. It may increase the risk of 41 * random segmentation faults. 42 */ 43 #define CONFIG_FLUSH_PAGE_ACCESSED 0 44 45 int split_tlb __ro_after_init; 46 int dcache_stride __ro_after_init; 47 int icache_stride __ro_after_init; 48 EXPORT_SYMBOL(dcache_stride); 49 50 /* Internal implementation in arch/parisc/kernel/pacache.S */ 51 void flush_dcache_page_asm(unsigned long phys_addr, unsigned long vaddr); 52 EXPORT_SYMBOL(flush_dcache_page_asm); 53 void purge_dcache_page_asm(unsigned long phys_addr, unsigned long vaddr); 54 void flush_icache_page_asm(unsigned long phys_addr, unsigned long vaddr); 55 void flush_data_cache_local(void *); /* flushes local data-cache only */ 56 void flush_instruction_cache_local(void); /* flushes local code-cache only */ 57 58 static void flush_kernel_dcache_page_addr(const void *addr); 59 60 /* On some machines (i.e., ones with the Merced bus), there can be 61 * only a single PxTLB broadcast at a time; this must be guaranteed 62 * by software. We need a spinlock around all TLB flushes to ensure 63 * this. 64 */ 65 DEFINE_SPINLOCK(pa_tlb_flush_lock); 66 67 #if defined(CONFIG_64BIT) && defined(CONFIG_SMP) 68 int pa_serialize_tlb_flushes __ro_after_init; 69 #endif 70 71 struct pdc_cache_info cache_info __ro_after_init; 72 #ifndef CONFIG_PA20 73 struct pdc_btlb_info btlb_info; 74 #endif 75 76 DEFINE_STATIC_KEY_TRUE(parisc_has_cache); 77 DEFINE_STATIC_KEY_TRUE(parisc_has_dcache); 78 DEFINE_STATIC_KEY_TRUE(parisc_has_icache); 79 80 static void cache_flush_local_cpu(void *dummy) 81 { 82 if (static_branch_likely(&parisc_has_icache)) 83 flush_instruction_cache_local(); 84 if (static_branch_likely(&parisc_has_dcache)) 85 flush_data_cache_local(NULL); 86 } 87 88 void flush_cache_all_local(void) 89 { 90 cache_flush_local_cpu(NULL); 91 } 92 93 void flush_cache_all(void) 94 { 95 if (static_branch_likely(&parisc_has_cache)) 96 on_each_cpu(cache_flush_local_cpu, NULL, 1); 97 } 98 99 static inline void flush_data_cache(void) 100 { 101 if (static_branch_likely(&parisc_has_dcache)) 102 on_each_cpu(flush_data_cache_local, NULL, 1); 103 } 104 105 106 /* Kernel virtual address of pfn. */ 107 #define pfn_va(pfn) __va(PFN_PHYS(pfn)) 108 109 void __update_cache(pte_t pte) 110 { 111 unsigned long pfn = pte_pfn(pte); 112 struct folio *folio; 113 unsigned int nr; 114 115 /* We don't have pte special. As a result, we can be called with 116 an invalid pfn and we don't need to flush the kernel dcache page. 117 This occurs with FireGL card in C8000. */ 118 if (!pfn_valid(pfn)) 119 return; 120 121 folio = page_folio(pfn_to_page(pfn)); 122 pfn = folio_pfn(folio); 123 nr = folio_nr_pages(folio); 124 if (folio_flush_mapping(folio) && 125 test_bit(PG_dcache_dirty, &folio->flags)) { 126 while (nr--) 127 flush_kernel_dcache_page_addr(pfn_va(pfn + nr)); 128 clear_bit(PG_dcache_dirty, &folio->flags); 129 } else if (parisc_requires_coherency()) 130 while (nr--) 131 flush_kernel_dcache_page_addr(pfn_va(pfn + nr)); 132 } 133 134 void 135 show_cache_info(struct seq_file *m) 136 { 137 char buf[32]; 138 139 seq_printf(m, "I-cache\t\t: %ld KB\n", 140 cache_info.ic_size/1024 ); 141 if (cache_info.dc_loop != 1) 142 snprintf(buf, 32, "%lu-way associative", cache_info.dc_loop); 143 seq_printf(m, "D-cache\t\t: %ld KB (%s%s, %s, alias=%d)\n", 144 cache_info.dc_size/1024, 145 (cache_info.dc_conf.cc_wt ? "WT":"WB"), 146 (cache_info.dc_conf.cc_sh ? ", shared I/D":""), 147 ((cache_info.dc_loop == 1) ? "direct mapped" : buf), 148 cache_info.dc_conf.cc_alias 149 ); 150 seq_printf(m, "ITLB entries\t: %ld\n" "DTLB entries\t: %ld%s\n", 151 cache_info.it_size, 152 cache_info.dt_size, 153 cache_info.dt_conf.tc_sh ? " - shared with ITLB":"" 154 ); 155 156 #ifndef CONFIG_PA20 157 /* BTLB - Block TLB */ 158 if (btlb_info.max_size==0) { 159 seq_printf(m, "BTLB\t\t: not supported\n" ); 160 } else { 161 seq_printf(m, 162 "BTLB fixed\t: max. %d pages, pagesize=%d (%dMB)\n" 163 "BTLB fix-entr.\t: %d instruction, %d data (%d combined)\n" 164 "BTLB var-entr.\t: %d instruction, %d data (%d combined)\n", 165 btlb_info.max_size, (int)4096, 166 btlb_info.max_size>>8, 167 btlb_info.fixed_range_info.num_i, 168 btlb_info.fixed_range_info.num_d, 169 btlb_info.fixed_range_info.num_comb, 170 btlb_info.variable_range_info.num_i, 171 btlb_info.variable_range_info.num_d, 172 btlb_info.variable_range_info.num_comb 173 ); 174 } 175 #endif 176 } 177 178 void __init 179 parisc_cache_init(void) 180 { 181 if (pdc_cache_info(&cache_info) < 0) 182 panic("parisc_cache_init: pdc_cache_info failed"); 183 184 #if 0 185 printk("ic_size %lx dc_size %lx it_size %lx\n", 186 cache_info.ic_size, 187 cache_info.dc_size, 188 cache_info.it_size); 189 190 printk("DC base 0x%lx stride 0x%lx count 0x%lx loop 0x%lx\n", 191 cache_info.dc_base, 192 cache_info.dc_stride, 193 cache_info.dc_count, 194 cache_info.dc_loop); 195 196 printk("dc_conf = 0x%lx alias %d blk %d line %d shift %d\n", 197 *(unsigned long *) (&cache_info.dc_conf), 198 cache_info.dc_conf.cc_alias, 199 cache_info.dc_conf.cc_block, 200 cache_info.dc_conf.cc_line, 201 cache_info.dc_conf.cc_shift); 202 printk(" wt %d sh %d cst %d hv %d\n", 203 cache_info.dc_conf.cc_wt, 204 cache_info.dc_conf.cc_sh, 205 cache_info.dc_conf.cc_cst, 206 cache_info.dc_conf.cc_hv); 207 208 printk("IC base 0x%lx stride 0x%lx count 0x%lx loop 0x%lx\n", 209 cache_info.ic_base, 210 cache_info.ic_stride, 211 cache_info.ic_count, 212 cache_info.ic_loop); 213 214 printk("IT base 0x%lx stride 0x%lx count 0x%lx loop 0x%lx off_base 0x%lx off_stride 0x%lx off_count 0x%lx\n", 215 cache_info.it_sp_base, 216 cache_info.it_sp_stride, 217 cache_info.it_sp_count, 218 cache_info.it_loop, 219 cache_info.it_off_base, 220 cache_info.it_off_stride, 221 cache_info.it_off_count); 222 223 printk("DT base 0x%lx stride 0x%lx count 0x%lx loop 0x%lx off_base 0x%lx off_stride 0x%lx off_count 0x%lx\n", 224 cache_info.dt_sp_base, 225 cache_info.dt_sp_stride, 226 cache_info.dt_sp_count, 227 cache_info.dt_loop, 228 cache_info.dt_off_base, 229 cache_info.dt_off_stride, 230 cache_info.dt_off_count); 231 232 printk("ic_conf = 0x%lx alias %d blk %d line %d shift %d\n", 233 *(unsigned long *) (&cache_info.ic_conf), 234 cache_info.ic_conf.cc_alias, 235 cache_info.ic_conf.cc_block, 236 cache_info.ic_conf.cc_line, 237 cache_info.ic_conf.cc_shift); 238 printk(" wt %d sh %d cst %d hv %d\n", 239 cache_info.ic_conf.cc_wt, 240 cache_info.ic_conf.cc_sh, 241 cache_info.ic_conf.cc_cst, 242 cache_info.ic_conf.cc_hv); 243 244 printk("D-TLB conf: sh %d page %d cst %d aid %d sr %d\n", 245 cache_info.dt_conf.tc_sh, 246 cache_info.dt_conf.tc_page, 247 cache_info.dt_conf.tc_cst, 248 cache_info.dt_conf.tc_aid, 249 cache_info.dt_conf.tc_sr); 250 251 printk("I-TLB conf: sh %d page %d cst %d aid %d sr %d\n", 252 cache_info.it_conf.tc_sh, 253 cache_info.it_conf.tc_page, 254 cache_info.it_conf.tc_cst, 255 cache_info.it_conf.tc_aid, 256 cache_info.it_conf.tc_sr); 257 #endif 258 259 split_tlb = 0; 260 if (cache_info.dt_conf.tc_sh == 0 || cache_info.dt_conf.tc_sh == 2) { 261 if (cache_info.dt_conf.tc_sh == 2) 262 printk(KERN_WARNING "Unexpected TLB configuration. " 263 "Will flush I/D separately (could be optimized).\n"); 264 265 split_tlb = 1; 266 } 267 268 /* "New and Improved" version from Jim Hull 269 * (1 << (cc_block-1)) * (cc_line << (4 + cnf.cc_shift)) 270 * The following CAFL_STRIDE is an optimized version, see 271 * http://lists.parisc-linux.org/pipermail/parisc-linux/2004-June/023625.html 272 * http://lists.parisc-linux.org/pipermail/parisc-linux/2004-June/023671.html 273 */ 274 #define CAFL_STRIDE(cnf) (cnf.cc_line << (3 + cnf.cc_block + cnf.cc_shift)) 275 dcache_stride = CAFL_STRIDE(cache_info.dc_conf); 276 icache_stride = CAFL_STRIDE(cache_info.ic_conf); 277 #undef CAFL_STRIDE 278 279 /* stride needs to be non-zero, otherwise cache flushes will not work */ 280 WARN_ON(cache_info.dc_size && dcache_stride == 0); 281 WARN_ON(cache_info.ic_size && icache_stride == 0); 282 283 if ((boot_cpu_data.pdc.capabilities & PDC_MODEL_NVA_MASK) == 284 PDC_MODEL_NVA_UNSUPPORTED) { 285 printk(KERN_WARNING "parisc_cache_init: Only equivalent aliasing supported!\n"); 286 #if 0 287 panic("SMP kernel required to avoid non-equivalent aliasing"); 288 #endif 289 } 290 } 291 292 void disable_sr_hashing(void) 293 { 294 int srhash_type, retval; 295 unsigned long space_bits; 296 297 switch (boot_cpu_data.cpu_type) { 298 case pcx: /* We shouldn't get this far. setup.c should prevent it. */ 299 BUG(); 300 return; 301 302 case pcxs: 303 case pcxt: 304 case pcxt_: 305 srhash_type = SRHASH_PCXST; 306 break; 307 308 case pcxl: 309 srhash_type = SRHASH_PCXL; 310 break; 311 312 case pcxl2: /* pcxl2 doesn't support space register hashing */ 313 return; 314 315 default: /* Currently all PA2.0 machines use the same ins. sequence */ 316 srhash_type = SRHASH_PA20; 317 break; 318 } 319 320 disable_sr_hashing_asm(srhash_type); 321 322 retval = pdc_spaceid_bits(&space_bits); 323 /* If this procedure isn't implemented, don't panic. */ 324 if (retval < 0 && retval != PDC_BAD_OPTION) 325 panic("pdc_spaceid_bits call failed.\n"); 326 if (space_bits != 0) 327 panic("SpaceID hashing is still on!\n"); 328 } 329 330 static inline void 331 __flush_cache_page(struct vm_area_struct *vma, unsigned long vmaddr, 332 unsigned long physaddr) 333 { 334 if (!static_branch_likely(&parisc_has_cache)) 335 return; 336 337 /* 338 * The TLB is the engine of coherence on parisc. The CPU is 339 * entitled to speculate any page with a TLB mapping, so here 340 * we kill the mapping then flush the page along a special flush 341 * only alias mapping. This guarantees that the page is no-longer 342 * in the cache for any process and nor may it be speculatively 343 * read in (until the user or kernel specifically accesses it, 344 * of course). 345 */ 346 flush_tlb_page(vma, vmaddr); 347 348 preempt_disable(); 349 flush_dcache_page_asm(physaddr, vmaddr); 350 if (vma->vm_flags & VM_EXEC) 351 flush_icache_page_asm(physaddr, vmaddr); 352 preempt_enable(); 353 } 354 355 static void flush_kernel_dcache_page_addr(const void *addr) 356 { 357 unsigned long vaddr = (unsigned long)addr; 358 unsigned long flags; 359 360 /* Purge TLB entry to remove translation on all CPUs */ 361 purge_tlb_start(flags); 362 pdtlb(SR_KERNEL, addr); 363 purge_tlb_end(flags); 364 365 /* Use tmpalias flush to prevent data cache move-in */ 366 preempt_disable(); 367 flush_dcache_page_asm(__pa(vaddr), vaddr); 368 preempt_enable(); 369 } 370 371 static void flush_kernel_icache_page_addr(const void *addr) 372 { 373 unsigned long vaddr = (unsigned long)addr; 374 unsigned long flags; 375 376 /* Purge TLB entry to remove translation on all CPUs */ 377 purge_tlb_start(flags); 378 pdtlb(SR_KERNEL, addr); 379 purge_tlb_end(flags); 380 381 /* Use tmpalias flush to prevent instruction cache move-in */ 382 preempt_disable(); 383 flush_icache_page_asm(__pa(vaddr), vaddr); 384 preempt_enable(); 385 } 386 387 void kunmap_flush_on_unmap(const void *addr) 388 { 389 flush_kernel_dcache_page_addr(addr); 390 } 391 EXPORT_SYMBOL(kunmap_flush_on_unmap); 392 393 void flush_icache_pages(struct vm_area_struct *vma, struct page *page, 394 unsigned int nr) 395 { 396 void *kaddr = page_address(page); 397 398 for (;;) { 399 flush_kernel_dcache_page_addr(kaddr); 400 flush_kernel_icache_page_addr(kaddr); 401 if (--nr == 0) 402 break; 403 kaddr += PAGE_SIZE; 404 } 405 } 406 407 /* 408 * Walk page directory for MM to find PTEP pointer for address ADDR. 409 */ 410 static inline pte_t *get_ptep(struct mm_struct *mm, unsigned long addr) 411 { 412 pte_t *ptep = NULL; 413 pgd_t *pgd = mm->pgd; 414 p4d_t *p4d; 415 pud_t *pud; 416 pmd_t *pmd; 417 418 if (!pgd_none(*pgd)) { 419 p4d = p4d_offset(pgd, addr); 420 if (!p4d_none(*p4d)) { 421 pud = pud_offset(p4d, addr); 422 if (!pud_none(*pud)) { 423 pmd = pmd_offset(pud, addr); 424 if (!pmd_none(*pmd)) 425 ptep = pte_offset_map(pmd, addr); 426 } 427 } 428 } 429 return ptep; 430 } 431 432 static inline bool pte_needs_flush(pte_t pte) 433 { 434 return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_NO_CACHE)) 435 == (_PAGE_PRESENT | _PAGE_ACCESSED); 436 } 437 438 /* 439 * Return user physical address. Returns 0 if page is not present. 440 */ 441 static inline unsigned long get_upa(struct mm_struct *mm, unsigned long addr) 442 { 443 unsigned long flags, space, pgd, prot, pa; 444 #ifdef CONFIG_TLB_PTLOCK 445 unsigned long pgd_lock; 446 #endif 447 448 /* Save context */ 449 local_irq_save(flags); 450 prot = mfctl(8); 451 space = mfsp(SR_USER); 452 pgd = mfctl(25); 453 #ifdef CONFIG_TLB_PTLOCK 454 pgd_lock = mfctl(28); 455 #endif 456 457 /* Set context for lpa_user */ 458 switch_mm_irqs_off(NULL, mm, NULL); 459 pa = lpa_user(addr); 460 461 /* Restore previous context */ 462 #ifdef CONFIG_TLB_PTLOCK 463 mtctl(pgd_lock, 28); 464 #endif 465 mtctl(pgd, 25); 466 mtsp(space, SR_USER); 467 mtctl(prot, 8); 468 local_irq_restore(flags); 469 470 return pa; 471 } 472 473 void flush_dcache_folio(struct folio *folio) 474 { 475 struct address_space *mapping = folio_flush_mapping(folio); 476 struct vm_area_struct *vma; 477 unsigned long addr, old_addr = 0; 478 void *kaddr; 479 unsigned long count = 0; 480 unsigned long i, nr, flags; 481 pgoff_t pgoff; 482 483 if (mapping && !mapping_mapped(mapping)) { 484 set_bit(PG_dcache_dirty, &folio->flags); 485 return; 486 } 487 488 nr = folio_nr_pages(folio); 489 kaddr = folio_address(folio); 490 for (i = 0; i < nr; i++) 491 flush_kernel_dcache_page_addr(kaddr + i * PAGE_SIZE); 492 493 if (!mapping) 494 return; 495 496 pgoff = folio->index; 497 498 /* 499 * We have carefully arranged in arch_get_unmapped_area() that 500 * *any* mappings of a file are always congruently mapped (whether 501 * declared as MAP_PRIVATE or MAP_SHARED), so we only need 502 * to flush one address here for them all to become coherent 503 * on machines that support equivalent aliasing 504 */ 505 flush_dcache_mmap_lock_irqsave(mapping, flags); 506 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff + nr - 1) { 507 unsigned long offset = pgoff - vma->vm_pgoff; 508 unsigned long pfn = folio_pfn(folio); 509 510 addr = vma->vm_start; 511 nr = folio_nr_pages(folio); 512 if (offset > -nr) { 513 pfn -= offset; 514 nr += offset; 515 } else { 516 addr += offset * PAGE_SIZE; 517 } 518 if (addr + nr * PAGE_SIZE > vma->vm_end) 519 nr = (vma->vm_end - addr) / PAGE_SIZE; 520 521 if (old_addr == 0 || (old_addr & (SHM_COLOUR - 1)) 522 != (addr & (SHM_COLOUR - 1))) { 523 for (i = 0; i < nr; i++) 524 __flush_cache_page(vma, 525 addr + i * PAGE_SIZE, 526 (pfn + i) * PAGE_SIZE); 527 /* 528 * Software is allowed to have any number 529 * of private mappings to a page. 530 */ 531 if (!(vma->vm_flags & VM_SHARED)) 532 continue; 533 if (old_addr) 534 pr_err("INEQUIVALENT ALIASES 0x%lx and 0x%lx in file %pD\n", 535 old_addr, addr, vma->vm_file); 536 if (nr == folio_nr_pages(folio)) 537 old_addr = addr; 538 } 539 WARN_ON(++count == 4096); 540 } 541 flush_dcache_mmap_unlock_irqrestore(mapping, flags); 542 } 543 EXPORT_SYMBOL(flush_dcache_folio); 544 545 /* Defined in arch/parisc/kernel/pacache.S */ 546 EXPORT_SYMBOL(flush_kernel_dcache_range_asm); 547 EXPORT_SYMBOL(flush_kernel_icache_range_asm); 548 549 #define FLUSH_THRESHOLD 0x80000 /* 0.5MB */ 550 static unsigned long parisc_cache_flush_threshold __ro_after_init = FLUSH_THRESHOLD; 551 552 #define FLUSH_TLB_THRESHOLD (16*1024) /* 16 KiB minimum TLB threshold */ 553 static unsigned long parisc_tlb_flush_threshold __ro_after_init = ~0UL; 554 555 void __init parisc_setup_cache_timing(void) 556 { 557 unsigned long rangetime, alltime; 558 unsigned long size; 559 unsigned long threshold, threshold2; 560 561 alltime = mfctl(16); 562 flush_data_cache(); 563 alltime = mfctl(16) - alltime; 564 565 size = (unsigned long)(_end - _text); 566 rangetime = mfctl(16); 567 flush_kernel_dcache_range((unsigned long)_text, size); 568 rangetime = mfctl(16) - rangetime; 569 570 printk(KERN_DEBUG "Whole cache flush %lu cycles, flushing %lu bytes %lu cycles\n", 571 alltime, size, rangetime); 572 573 threshold = L1_CACHE_ALIGN((unsigned long)((uint64_t)size * alltime / rangetime)); 574 pr_info("Calculated flush threshold is %lu KiB\n", 575 threshold/1024); 576 577 /* 578 * The threshold computed above isn't very reliable. The following 579 * heuristic works reasonably well on c8000/rp3440. 580 */ 581 threshold2 = cache_info.dc_size * num_online_cpus(); 582 parisc_cache_flush_threshold = threshold2; 583 printk(KERN_INFO "Cache flush threshold set to %lu KiB\n", 584 parisc_cache_flush_threshold/1024); 585 586 /* calculate TLB flush threshold */ 587 588 /* On SMP machines, skip the TLB measure of kernel text which 589 * has been mapped as huge pages. */ 590 if (num_online_cpus() > 1 && !parisc_requires_coherency()) { 591 threshold = max(cache_info.it_size, cache_info.dt_size); 592 threshold *= PAGE_SIZE; 593 threshold /= num_online_cpus(); 594 goto set_tlb_threshold; 595 } 596 597 size = (unsigned long)_end - (unsigned long)_text; 598 rangetime = mfctl(16); 599 flush_tlb_kernel_range((unsigned long)_text, (unsigned long)_end); 600 rangetime = mfctl(16) - rangetime; 601 602 alltime = mfctl(16); 603 flush_tlb_all(); 604 alltime = mfctl(16) - alltime; 605 606 printk(KERN_INFO "Whole TLB flush %lu cycles, Range flush %lu bytes %lu cycles\n", 607 alltime, size, rangetime); 608 609 threshold = PAGE_ALIGN((num_online_cpus() * size * alltime) / rangetime); 610 printk(KERN_INFO "Calculated TLB flush threshold %lu KiB\n", 611 threshold/1024); 612 613 set_tlb_threshold: 614 if (threshold > FLUSH_TLB_THRESHOLD) 615 parisc_tlb_flush_threshold = threshold; 616 else 617 parisc_tlb_flush_threshold = FLUSH_TLB_THRESHOLD; 618 619 printk(KERN_INFO "TLB flush threshold set to %lu KiB\n", 620 parisc_tlb_flush_threshold/1024); 621 } 622 623 extern void purge_kernel_dcache_page_asm(unsigned long); 624 extern void clear_user_page_asm(void *, unsigned long); 625 extern void copy_user_page_asm(void *, void *, unsigned long); 626 627 static void flush_cache_page_if_present(struct vm_area_struct *vma, 628 unsigned long vmaddr) 629 { 630 #if CONFIG_FLUSH_PAGE_ACCESSED 631 bool needs_flush = false; 632 pte_t *ptep, pte; 633 634 ptep = get_ptep(vma->vm_mm, vmaddr); 635 if (ptep) { 636 pte = ptep_get(ptep); 637 needs_flush = pte_needs_flush(pte); 638 pte_unmap(ptep); 639 } 640 if (needs_flush) 641 __flush_cache_page(vma, vmaddr, PFN_PHYS(pte_pfn(pte))); 642 #else 643 struct mm_struct *mm = vma->vm_mm; 644 unsigned long physaddr = get_upa(mm, vmaddr); 645 646 if (physaddr) 647 __flush_cache_page(vma, vmaddr, PAGE_ALIGN_DOWN(physaddr)); 648 #endif 649 } 650 651 void copy_user_highpage(struct page *to, struct page *from, 652 unsigned long vaddr, struct vm_area_struct *vma) 653 { 654 void *kto, *kfrom; 655 656 kfrom = kmap_local_page(from); 657 kto = kmap_local_page(to); 658 __flush_cache_page(vma, vaddr, PFN_PHYS(page_to_pfn(from))); 659 copy_page_asm(kto, kfrom); 660 kunmap_local(kto); 661 kunmap_local(kfrom); 662 } 663 664 void copy_to_user_page(struct vm_area_struct *vma, struct page *page, 665 unsigned long user_vaddr, void *dst, void *src, int len) 666 { 667 __flush_cache_page(vma, user_vaddr, PFN_PHYS(page_to_pfn(page))); 668 memcpy(dst, src, len); 669 flush_kernel_dcache_page_addr(PTR_PAGE_ALIGN_DOWN(dst)); 670 } 671 672 void copy_from_user_page(struct vm_area_struct *vma, struct page *page, 673 unsigned long user_vaddr, void *dst, void *src, int len) 674 { 675 __flush_cache_page(vma, user_vaddr, PFN_PHYS(page_to_pfn(page))); 676 memcpy(dst, src, len); 677 flush_kernel_dcache_page_addr(PTR_PAGE_ALIGN_DOWN(src)); 678 } 679 680 /* __flush_tlb_range() 681 * 682 * returns 1 if all TLBs were flushed. 683 */ 684 int __flush_tlb_range(unsigned long sid, unsigned long start, 685 unsigned long end) 686 { 687 unsigned long flags; 688 689 if ((!IS_ENABLED(CONFIG_SMP) || !arch_irqs_disabled()) && 690 end - start >= parisc_tlb_flush_threshold) { 691 flush_tlb_all(); 692 return 1; 693 } 694 695 /* Purge TLB entries for small ranges using the pdtlb and 696 pitlb instructions. These instructions execute locally 697 but cause a purge request to be broadcast to other TLBs. */ 698 while (start < end) { 699 purge_tlb_start(flags); 700 mtsp(sid, SR_TEMP1); 701 pdtlb(SR_TEMP1, start); 702 pitlb(SR_TEMP1, start); 703 purge_tlb_end(flags); 704 start += PAGE_SIZE; 705 } 706 return 0; 707 } 708 709 static void flush_cache_pages(struct vm_area_struct *vma, unsigned long start, unsigned long end) 710 { 711 unsigned long addr; 712 713 for (addr = start; addr < end; addr += PAGE_SIZE) 714 flush_cache_page_if_present(vma, addr); 715 } 716 717 static inline unsigned long mm_total_size(struct mm_struct *mm) 718 { 719 struct vm_area_struct *vma; 720 unsigned long usize = 0; 721 VMA_ITERATOR(vmi, mm, 0); 722 723 for_each_vma(vmi, vma) { 724 if (usize >= parisc_cache_flush_threshold) 725 break; 726 usize += vma->vm_end - vma->vm_start; 727 } 728 return usize; 729 } 730 731 void flush_cache_mm(struct mm_struct *mm) 732 { 733 struct vm_area_struct *vma; 734 VMA_ITERATOR(vmi, mm, 0); 735 736 /* 737 * Flushing the whole cache on each cpu takes forever on 738 * rp3440, etc. So, avoid it if the mm isn't too big. 739 * 740 * Note that we must flush the entire cache on machines 741 * with aliasing caches to prevent random segmentation 742 * faults. 743 */ 744 if (!parisc_requires_coherency() 745 || mm_total_size(mm) >= parisc_cache_flush_threshold) { 746 if (WARN_ON(IS_ENABLED(CONFIG_SMP) && arch_irqs_disabled())) 747 return; 748 flush_tlb_all(); 749 flush_cache_all(); 750 return; 751 } 752 753 /* Flush mm */ 754 for_each_vma(vmi, vma) 755 flush_cache_pages(vma, vma->vm_start, vma->vm_end); 756 } 757 758 void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) 759 { 760 if (!parisc_requires_coherency() 761 || end - start >= parisc_cache_flush_threshold) { 762 if (WARN_ON(IS_ENABLED(CONFIG_SMP) && arch_irqs_disabled())) 763 return; 764 flush_tlb_range(vma, start, end); 765 if (vma->vm_flags & VM_EXEC) 766 flush_cache_all(); 767 else 768 flush_data_cache(); 769 return; 770 } 771 772 flush_cache_pages(vma, start & PAGE_MASK, end); 773 } 774 775 void flush_cache_page(struct vm_area_struct *vma, unsigned long vmaddr, unsigned long pfn) 776 { 777 __flush_cache_page(vma, vmaddr, PFN_PHYS(pfn)); 778 } 779 780 void flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr) 781 { 782 if (!PageAnon(page)) 783 return; 784 785 __flush_cache_page(vma, vmaddr, PFN_PHYS(page_to_pfn(page))); 786 } 787 788 int ptep_clear_flush_young(struct vm_area_struct *vma, unsigned long addr, 789 pte_t *ptep) 790 { 791 pte_t pte = ptep_get(ptep); 792 793 if (!pte_young(pte)) 794 return 0; 795 set_pte(ptep, pte_mkold(pte)); 796 #if CONFIG_FLUSH_PAGE_ACCESSED 797 __flush_cache_page(vma, addr, PFN_PHYS(pte_pfn(pte))); 798 #endif 799 return 1; 800 } 801 802 /* 803 * After a PTE is cleared, we have no way to flush the cache for 804 * the physical page. On PA8800 and PA8900 processors, these lines 805 * can cause random cache corruption. Thus, we must flush the cache 806 * as well as the TLB when clearing a PTE that's valid. 807 */ 808 pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, 809 pte_t *ptep) 810 { 811 struct mm_struct *mm = (vma)->vm_mm; 812 pte_t pte = ptep_get_and_clear(mm, addr, ptep); 813 unsigned long pfn = pte_pfn(pte); 814 815 if (pfn_valid(pfn)) 816 __flush_cache_page(vma, addr, PFN_PHYS(pfn)); 817 else if (pte_accessible(mm, pte)) 818 flush_tlb_page(vma, addr); 819 820 return pte; 821 } 822 823 /* 824 * The physical address for pages in the ioremap case can be obtained 825 * from the vm_struct struct. I wasn't able to successfully handle the 826 * vmalloc and vmap cases. We have an array of struct page pointers in 827 * the uninitialized vmalloc case but the flush failed using page_to_pfn. 828 */ 829 void flush_cache_vmap(unsigned long start, unsigned long end) 830 { 831 unsigned long addr, physaddr; 832 struct vm_struct *vm; 833 834 /* Prevent cache move-in */ 835 flush_tlb_kernel_range(start, end); 836 837 if (end - start >= parisc_cache_flush_threshold) { 838 flush_cache_all(); 839 return; 840 } 841 842 if (WARN_ON_ONCE(!is_vmalloc_addr((void *)start))) { 843 flush_cache_all(); 844 return; 845 } 846 847 vm = find_vm_area((void *)start); 848 if (WARN_ON_ONCE(!vm)) { 849 flush_cache_all(); 850 return; 851 } 852 853 /* The physical addresses of IOREMAP regions are contiguous */ 854 if (vm->flags & VM_IOREMAP) { 855 physaddr = vm->phys_addr; 856 for (addr = start; addr < end; addr += PAGE_SIZE) { 857 preempt_disable(); 858 flush_dcache_page_asm(physaddr, start); 859 flush_icache_page_asm(physaddr, start); 860 preempt_enable(); 861 physaddr += PAGE_SIZE; 862 } 863 return; 864 } 865 866 flush_cache_all(); 867 } 868 EXPORT_SYMBOL(flush_cache_vmap); 869 870 /* 871 * The vm_struct has been retired and the page table is set up. The 872 * last page in the range is a guard page. Its physical address can't 873 * be determined using lpa, so there is no way to flush the range 874 * using flush_dcache_page_asm. 875 */ 876 void flush_cache_vunmap(unsigned long start, unsigned long end) 877 { 878 /* Prevent cache move-in */ 879 flush_tlb_kernel_range(start, end); 880 flush_data_cache(); 881 } 882 EXPORT_SYMBOL(flush_cache_vunmap); 883 884 /* 885 * On systems with PA8800/PA8900 processors, there is no way to flush 886 * a vmap range other than using the architected loop to flush the 887 * entire cache. The page directory is not set up, so we can't use 888 * fdc, etc. FDCE/FICE don't work to flush a portion of the cache. 889 * L2 is physically indexed but FDCE/FICE instructions in virtual 890 * mode output their virtual address on the core bus, not their 891 * real address. As a result, the L2 cache index formed from the 892 * virtual address will most likely not be the same as the L2 index 893 * formed from the real address. 894 */ 895 void flush_kernel_vmap_range(void *vaddr, int size) 896 { 897 unsigned long start = (unsigned long)vaddr; 898 unsigned long end = start + size; 899 900 flush_tlb_kernel_range(start, end); 901 902 if (!static_branch_likely(&parisc_has_dcache)) 903 return; 904 905 /* If interrupts are disabled, we can only do local flush */ 906 if (WARN_ON(IS_ENABLED(CONFIG_SMP) && arch_irqs_disabled())) { 907 flush_data_cache_local(NULL); 908 return; 909 } 910 911 flush_data_cache(); 912 } 913 EXPORT_SYMBOL(flush_kernel_vmap_range); 914 915 void invalidate_kernel_vmap_range(void *vaddr, int size) 916 { 917 unsigned long start = (unsigned long)vaddr; 918 unsigned long end = start + size; 919 920 /* Ensure DMA is complete */ 921 asm_syncdma(); 922 923 flush_tlb_kernel_range(start, end); 924 925 if (!static_branch_likely(&parisc_has_dcache)) 926 return; 927 928 /* If interrupts are disabled, we can only do local flush */ 929 if (WARN_ON(IS_ENABLED(CONFIG_SMP) && arch_irqs_disabled())) { 930 flush_data_cache_local(NULL); 931 return; 932 } 933 934 flush_data_cache(); 935 } 936 EXPORT_SYMBOL(invalidate_kernel_vmap_range); 937 938 939 SYSCALL_DEFINE3(cacheflush, unsigned long, addr, unsigned long, bytes, 940 unsigned int, cache) 941 { 942 unsigned long start, end; 943 ASM_EXCEPTIONTABLE_VAR(error); 944 945 if (bytes == 0) 946 return 0; 947 if (!access_ok((void __user *) addr, bytes)) 948 return -EFAULT; 949 950 end = addr + bytes; 951 952 if (cache & DCACHE) { 953 start = addr; 954 __asm__ __volatile__ ( 955 #ifdef CONFIG_64BIT 956 "1: cmpb,*<<,n %0,%2,1b\n" 957 #else 958 "1: cmpb,<<,n %0,%2,1b\n" 959 #endif 960 " fic,m %3(%4,%0)\n" 961 "2: sync\n" 962 ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 2b, "%1") 963 : "+r" (start), "+r" (error) 964 : "r" (end), "r" (dcache_stride), "i" (SR_USER)); 965 } 966 967 if (cache & ICACHE && error == 0) { 968 start = addr; 969 __asm__ __volatile__ ( 970 #ifdef CONFIG_64BIT 971 "1: cmpb,*<<,n %0,%2,1b\n" 972 #else 973 "1: cmpb,<<,n %0,%2,1b\n" 974 #endif 975 " fdc,m %3(%4,%0)\n" 976 "2: sync\n" 977 ASM_EXCEPTIONTABLE_ENTRY_EFAULT(1b, 2b, "%1") 978 : "+r" (start), "+r" (error) 979 : "r" (end), "r" (icache_stride), "i" (SR_USER)); 980 } 981 982 return error; 983 } 984