1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 #include <linux/highmem.h> 4 #include <linux/kprobes.h> 5 6 /** 7 * flush_coherent_icache() - if a CPU has a coherent icache, flush it 8 * Return true if the cache was flushed, false otherwise 9 */ 10 static inline bool flush_coherent_icache(void) 11 { 12 /* 13 * For a snooping icache, we still need a dummy icbi to purge all the 14 * prefetched instructions from the ifetch buffers. We also need a sync 15 * before the icbi to order the actual stores to memory that might 16 * have modified instructions with the icbi. 17 */ 18 if (cpu_has_feature(CPU_FTR_COHERENT_ICACHE)) { 19 mb(); /* sync */ 20 icbi((void *)PAGE_OFFSET); 21 mb(); /* sync */ 22 isync(); 23 return true; 24 } 25 26 return false; 27 } 28 29 /** 30 * invalidate_icache_range() - Flush the icache by issuing icbi across an address range 31 * @start: the start address 32 * @stop: the stop address (exclusive) 33 */ 34 static void invalidate_icache_range(unsigned long start, unsigned long stop) 35 { 36 unsigned long shift = l1_icache_shift(); 37 unsigned long bytes = l1_icache_bytes(); 38 char *addr = (char *)(start & ~(bytes - 1)); 39 unsigned long size = stop - (unsigned long)addr + (bytes - 1); 40 unsigned long i; 41 42 for (i = 0; i < size >> shift; i++, addr += bytes) 43 icbi(addr); 44 45 mb(); /* sync */ 46 isync(); 47 } 48 49 /** 50 * flush_icache_range: Write any modified data cache blocks out to memory 51 * and invalidate the corresponding blocks in the instruction cache 52 * 53 * Generic code will call this after writing memory, before executing from it. 54 * 55 * @start: the start address 56 * @stop: the stop address (exclusive) 57 */ 58 void flush_icache_range(unsigned long start, unsigned long stop) 59 { 60 if (flush_coherent_icache()) 61 return; 62 63 clean_dcache_range(start, stop); 64 65 if (IS_ENABLED(CONFIG_44x)) { 66 /* 67 * Flash invalidate on 44x because we are passed kmapped 68 * addresses and this doesn't work for userspace pages due to 69 * the virtually tagged icache. 70 */ 71 iccci((void *)start); 72 mb(); /* sync */ 73 isync(); 74 } else 75 invalidate_icache_range(start, stop); 76 } 77 EXPORT_SYMBOL(flush_icache_range); 78 79 #ifdef CONFIG_HIGHMEM 80 /** 81 * flush_dcache_icache_phys() - Flush a page by its physical address 82 * @physaddr: the physical address of the page 83 */ 84 static void flush_dcache_icache_phys(unsigned long physaddr) 85 { 86 unsigned long bytes = l1_dcache_bytes(); 87 unsigned long nb = PAGE_SIZE / bytes; 88 unsigned long addr = physaddr & PAGE_MASK; 89 unsigned long msr, msr0; 90 unsigned long loop1 = addr, loop2 = addr; 91 92 msr0 = mfmsr(); 93 msr = msr0 & ~MSR_DR; 94 /* 95 * This must remain as ASM to prevent potential memory accesses 96 * while the data MMU is disabled 97 */ 98 asm volatile( 99 " mtctr %2;\n" 100 " mtmsr %3;\n" 101 " isync;\n" 102 "0: dcbst 0, %0;\n" 103 " addi %0, %0, %4;\n" 104 " bdnz 0b;\n" 105 " sync;\n" 106 " mtctr %2;\n" 107 "1: icbi 0, %1;\n" 108 " addi %1, %1, %4;\n" 109 " bdnz 1b;\n" 110 " sync;\n" 111 " mtmsr %5;\n" 112 " isync;\n" 113 : "+&r" (loop1), "+&r" (loop2) 114 : "r" (nb), "r" (msr), "i" (bytes), "r" (msr0) 115 : "ctr", "memory"); 116 } 117 NOKPROBE_SYMBOL(flush_dcache_icache_phys) 118 #else 119 static void flush_dcache_icache_phys(unsigned long physaddr) 120 { 121 } 122 #endif 123 124 /** 125 * __flush_dcache_icache(): Flush a particular page from the data cache to RAM. 126 * Note: this is necessary because the instruction cache does *not* 127 * snoop from the data cache. 128 * 129 * @p: the address of the page to flush 130 */ 131 static void __flush_dcache_icache(void *p) 132 { 133 unsigned long addr = (unsigned long)p & PAGE_MASK; 134 135 clean_dcache_range(addr, addr + PAGE_SIZE); 136 137 /* 138 * We don't flush the icache on 44x. Those have a virtual icache and we 139 * don't have access to the virtual address here (it's not the page 140 * vaddr but where it's mapped in user space). The flushing of the 141 * icache on these is handled elsewhere, when a change in the address 142 * space occurs, before returning to user space. 143 */ 144 145 if (mmu_has_feature(MMU_FTR_TYPE_44x)) 146 return; 147 148 invalidate_icache_range(addr, addr + PAGE_SIZE); 149 } 150 151 void flush_dcache_icache_folio(struct folio *folio) 152 { 153 unsigned int i, nr = folio_nr_pages(folio); 154 155 if (flush_coherent_icache()) 156 return; 157 158 if (!folio_test_highmem(folio)) { 159 void *addr = folio_address(folio); 160 for (i = 0; i < nr; i++) 161 __flush_dcache_icache(addr + i * PAGE_SIZE); 162 } else if (IS_ENABLED(CONFIG_BOOKE) || sizeof(phys_addr_t) > sizeof(void *)) { 163 for (i = 0; i < nr; i++) { 164 void *start = kmap_local_folio(folio, i * PAGE_SIZE); 165 166 __flush_dcache_icache(start); 167 kunmap_local(start); 168 } 169 } else { 170 unsigned long pfn = folio_pfn(folio); 171 for (i = 0; i < nr; i++) 172 flush_dcache_icache_phys((pfn + i) * PAGE_SIZE); 173 } 174 } 175 EXPORT_SYMBOL(flush_dcache_icache_folio); 176 177 void clear_user_page(void *page, unsigned long vaddr, struct page *pg) 178 { 179 clear_page(page); 180 181 /* 182 * We shouldn't have to do this, but some versions of glibc 183 * require it (ld.so assumes zero filled pages are icache clean) 184 * - Anton 185 */ 186 flush_dcache_page(pg); 187 } 188 EXPORT_SYMBOL(clear_user_page); 189 190 void copy_user_page(void *vto, void *vfrom, unsigned long vaddr, 191 struct page *pg) 192 { 193 copy_page(vto, vfrom); 194 195 /* 196 * We should be able to use the following optimisation, however 197 * there are two problems. 198 * Firstly a bug in some versions of binutils meant PLT sections 199 * were not marked executable. 200 * Secondly the first word in the GOT section is blrl, used 201 * to establish the GOT address. Until recently the GOT was 202 * not marked executable. 203 * - Anton 204 */ 205 #if 0 206 if (!vma->vm_file && ((vma->vm_flags & VM_EXEC) == 0)) 207 return; 208 #endif 209 210 flush_dcache_page(pg); 211 } 212 213 void flush_icache_user_page(struct vm_area_struct *vma, struct page *page, 214 unsigned long addr, int len) 215 { 216 void *maddr; 217 218 maddr = kmap_local_page(page) + (addr & ~PAGE_MASK); 219 flush_icache_range((unsigned long)maddr, (unsigned long)maddr + len); 220 kunmap_local(maddr); 221 } 222