1 /* 2 * Copyright (C) 2001 Ben. Herrenschmidt (benh@kernel.crashing.org) 3 * 4 * Modifications for ppc64: 5 * Copyright (C) 2003 Dave Engebretsen <engebret@us.ibm.com> 6 * 7 * Copyright 2008 Michael Ellerman, IBM Corporation. 8 * 9 * This program is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU General Public License 11 * as published by the Free Software Foundation; either version 12 * 2 of the License, or (at your option) any later version. 13 */ 14 15 #include <linux/types.h> 16 #include <linux/jump_label.h> 17 #include <linux/kernel.h> 18 #include <linux/string.h> 19 #include <linux/init.h> 20 #include <asm/cputable.h> 21 #include <asm/code-patching.h> 22 #include <asm/page.h> 23 #include <asm/sections.h> 24 #include <asm/setup.h> 25 #include <asm/firmware.h> 26 27 struct fixup_entry { 28 unsigned long mask; 29 unsigned long value; 30 long start_off; 31 long end_off; 32 long alt_start_off; 33 long alt_end_off; 34 }; 35 36 static unsigned int *calc_addr(struct fixup_entry *fcur, long offset) 37 { 38 /* 39 * We store the offset to the code as a negative offset from 40 * the start of the alt_entry, to support the VDSO. This 41 * routine converts that back into an actual address. 42 */ 43 return (unsigned int *)((unsigned long)fcur + offset); 44 } 45 46 static int patch_alt_instruction(unsigned int *src, unsigned int *dest, 47 unsigned int *alt_start, unsigned int *alt_end) 48 { 49 unsigned int instr; 50 51 instr = *src; 52 53 if (instr_is_relative_branch(*src)) { 54 unsigned int *target = (unsigned int *)branch_target(src); 55 56 /* Branch within the section doesn't need translating */ 57 if (target < alt_start || target >= alt_end) { 58 instr = translate_branch(dest, src); 59 if (!instr) 60 return 1; 61 } 62 } 63 64 patch_instruction(dest, instr); 65 66 return 0; 67 } 68 69 static int patch_feature_section(unsigned long value, struct fixup_entry *fcur) 70 { 71 unsigned int *start, *end, *alt_start, *alt_end, *src, *dest; 72 73 start = calc_addr(fcur, fcur->start_off); 74 end = calc_addr(fcur, fcur->end_off); 75 alt_start = calc_addr(fcur, fcur->alt_start_off); 76 alt_end = calc_addr(fcur, fcur->alt_end_off); 77 78 if ((alt_end - alt_start) > (end - start)) 79 return 1; 80 81 if ((value & fcur->mask) == fcur->value) 82 return 0; 83 84 src = alt_start; 85 dest = start; 86 87 for (; src < alt_end; src++, dest++) { 88 if (patch_alt_instruction(src, dest, alt_start, alt_end)) 89 return 1; 90 } 91 92 for (; dest < end; dest++) 93 patch_instruction(dest, PPC_INST_NOP); 94 95 return 0; 96 } 97 98 void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end) 99 { 100 struct fixup_entry *fcur, *fend; 101 102 fcur = fixup_start; 103 fend = fixup_end; 104 105 for (; fcur < fend; fcur++) { 106 if (patch_feature_section(value, fcur)) { 107 WARN_ON(1); 108 printk("Unable to patch feature section at %p - %p" \ 109 " with %p - %p\n", 110 calc_addr(fcur, fcur->start_off), 111 calc_addr(fcur, fcur->end_off), 112 calc_addr(fcur, fcur->alt_start_off), 113 calc_addr(fcur, fcur->alt_end_off)); 114 } 115 } 116 } 117 118 void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end) 119 { 120 long *start, *end; 121 unsigned int *dest; 122 123 if (!(value & CPU_FTR_LWSYNC)) 124 return ; 125 126 start = fixup_start; 127 end = fixup_end; 128 129 for (; start < end; start++) { 130 dest = (void *)start + *start; 131 patch_instruction(dest, PPC_INST_LWSYNC); 132 } 133 } 134 135 static void do_final_fixups(void) 136 { 137 #if defined(CONFIG_PPC64) && defined(CONFIG_RELOCATABLE) 138 int *src, *dest; 139 unsigned long length; 140 141 if (PHYSICAL_START == 0) 142 return; 143 144 src = (int *)(KERNELBASE + PHYSICAL_START); 145 dest = (int *)KERNELBASE; 146 length = (__end_interrupts - _stext) / sizeof(int); 147 148 while (length--) { 149 patch_instruction(dest, *src); 150 src++; 151 dest++; 152 } 153 #endif 154 } 155 156 static unsigned long __initdata saved_cpu_features; 157 static unsigned int __initdata saved_mmu_features; 158 #ifdef CONFIG_PPC64 159 static unsigned long __initdata saved_firmware_features; 160 #endif 161 162 void __init apply_feature_fixups(void) 163 { 164 struct cpu_spec *spec = PTRRELOC(*PTRRELOC(&cur_cpu_spec)); 165 166 *PTRRELOC(&saved_cpu_features) = spec->cpu_features; 167 *PTRRELOC(&saved_mmu_features) = spec->mmu_features; 168 169 /* 170 * Apply the CPU-specific and firmware specific fixups to kernel text 171 * (nop out sections not relevant to this CPU or this firmware). 172 */ 173 do_feature_fixups(spec->cpu_features, 174 PTRRELOC(&__start___ftr_fixup), 175 PTRRELOC(&__stop___ftr_fixup)); 176 177 do_feature_fixups(spec->mmu_features, 178 PTRRELOC(&__start___mmu_ftr_fixup), 179 PTRRELOC(&__stop___mmu_ftr_fixup)); 180 181 do_lwsync_fixups(spec->cpu_features, 182 PTRRELOC(&__start___lwsync_fixup), 183 PTRRELOC(&__stop___lwsync_fixup)); 184 185 #ifdef CONFIG_PPC64 186 saved_firmware_features = powerpc_firmware_features; 187 do_feature_fixups(powerpc_firmware_features, 188 &__start___fw_ftr_fixup, &__stop___fw_ftr_fixup); 189 #endif 190 do_final_fixups(); 191 192 /* 193 * Initialise jump label. This causes all the cpu/mmu_has_feature() 194 * checks to take on their correct polarity based on the current set of 195 * CPU/MMU features. 196 */ 197 jump_label_init(); 198 cpu_feature_keys_init(); 199 mmu_feature_keys_init(); 200 } 201 202 static int __init check_features(void) 203 { 204 WARN(saved_cpu_features != cur_cpu_spec->cpu_features, 205 "CPU features changed after feature patching!\n"); 206 WARN(saved_mmu_features != cur_cpu_spec->mmu_features, 207 "MMU features changed after feature patching!\n"); 208 #ifdef CONFIG_PPC64 209 WARN(saved_firmware_features != powerpc_firmware_features, 210 "Firmware features changed after feature patching!\n"); 211 #endif 212 213 return 0; 214 } 215 late_initcall(check_features); 216 217 #ifdef CONFIG_FTR_FIXUP_SELFTEST 218 219 #define check(x) \ 220 if (!(x)) printk("feature-fixups: test failed at line %d\n", __LINE__); 221 222 /* This must be after the text it fixes up, vmlinux.lds.S enforces that atm */ 223 static struct fixup_entry fixup; 224 225 static long calc_offset(struct fixup_entry *entry, unsigned int *p) 226 { 227 return (unsigned long)p - (unsigned long)entry; 228 } 229 230 static void test_basic_patching(void) 231 { 232 extern unsigned int ftr_fixup_test1; 233 extern unsigned int end_ftr_fixup_test1; 234 extern unsigned int ftr_fixup_test1_orig; 235 extern unsigned int ftr_fixup_test1_expected; 236 int size = &end_ftr_fixup_test1 - &ftr_fixup_test1; 237 238 fixup.value = fixup.mask = 8; 239 fixup.start_off = calc_offset(&fixup, &ftr_fixup_test1 + 1); 240 fixup.end_off = calc_offset(&fixup, &ftr_fixup_test1 + 2); 241 fixup.alt_start_off = fixup.alt_end_off = 0; 242 243 /* Sanity check */ 244 check(memcmp(&ftr_fixup_test1, &ftr_fixup_test1_orig, size) == 0); 245 246 /* Check we don't patch if the value matches */ 247 patch_feature_section(8, &fixup); 248 check(memcmp(&ftr_fixup_test1, &ftr_fixup_test1_orig, size) == 0); 249 250 /* Check we do patch if the value doesn't match */ 251 patch_feature_section(0, &fixup); 252 check(memcmp(&ftr_fixup_test1, &ftr_fixup_test1_expected, size) == 0); 253 254 /* Check we do patch if the mask doesn't match */ 255 memcpy(&ftr_fixup_test1, &ftr_fixup_test1_orig, size); 256 check(memcmp(&ftr_fixup_test1, &ftr_fixup_test1_orig, size) == 0); 257 patch_feature_section(~8, &fixup); 258 check(memcmp(&ftr_fixup_test1, &ftr_fixup_test1_expected, size) == 0); 259 } 260 261 static void test_alternative_patching(void) 262 { 263 extern unsigned int ftr_fixup_test2; 264 extern unsigned int end_ftr_fixup_test2; 265 extern unsigned int ftr_fixup_test2_orig; 266 extern unsigned int ftr_fixup_test2_alt; 267 extern unsigned int ftr_fixup_test2_expected; 268 int size = &end_ftr_fixup_test2 - &ftr_fixup_test2; 269 270 fixup.value = fixup.mask = 0xF; 271 fixup.start_off = calc_offset(&fixup, &ftr_fixup_test2 + 1); 272 fixup.end_off = calc_offset(&fixup, &ftr_fixup_test2 + 2); 273 fixup.alt_start_off = calc_offset(&fixup, &ftr_fixup_test2_alt); 274 fixup.alt_end_off = calc_offset(&fixup, &ftr_fixup_test2_alt + 1); 275 276 /* Sanity check */ 277 check(memcmp(&ftr_fixup_test2, &ftr_fixup_test2_orig, size) == 0); 278 279 /* Check we don't patch if the value matches */ 280 patch_feature_section(0xF, &fixup); 281 check(memcmp(&ftr_fixup_test2, &ftr_fixup_test2_orig, size) == 0); 282 283 /* Check we do patch if the value doesn't match */ 284 patch_feature_section(0, &fixup); 285 check(memcmp(&ftr_fixup_test2, &ftr_fixup_test2_expected, size) == 0); 286 287 /* Check we do patch if the mask doesn't match */ 288 memcpy(&ftr_fixup_test2, &ftr_fixup_test2_orig, size); 289 check(memcmp(&ftr_fixup_test2, &ftr_fixup_test2_orig, size) == 0); 290 patch_feature_section(~0xF, &fixup); 291 check(memcmp(&ftr_fixup_test2, &ftr_fixup_test2_expected, size) == 0); 292 } 293 294 static void test_alternative_case_too_big(void) 295 { 296 extern unsigned int ftr_fixup_test3; 297 extern unsigned int end_ftr_fixup_test3; 298 extern unsigned int ftr_fixup_test3_orig; 299 extern unsigned int ftr_fixup_test3_alt; 300 int size = &end_ftr_fixup_test3 - &ftr_fixup_test3; 301 302 fixup.value = fixup.mask = 0xC; 303 fixup.start_off = calc_offset(&fixup, &ftr_fixup_test3 + 1); 304 fixup.end_off = calc_offset(&fixup, &ftr_fixup_test3 + 2); 305 fixup.alt_start_off = calc_offset(&fixup, &ftr_fixup_test3_alt); 306 fixup.alt_end_off = calc_offset(&fixup, &ftr_fixup_test3_alt + 2); 307 308 /* Sanity check */ 309 check(memcmp(&ftr_fixup_test3, &ftr_fixup_test3_orig, size) == 0); 310 311 /* Expect nothing to be patched, and the error returned to us */ 312 check(patch_feature_section(0xF, &fixup) == 1); 313 check(memcmp(&ftr_fixup_test3, &ftr_fixup_test3_orig, size) == 0); 314 check(patch_feature_section(0, &fixup) == 1); 315 check(memcmp(&ftr_fixup_test3, &ftr_fixup_test3_orig, size) == 0); 316 check(patch_feature_section(~0xF, &fixup) == 1); 317 check(memcmp(&ftr_fixup_test3, &ftr_fixup_test3_orig, size) == 0); 318 } 319 320 static void test_alternative_case_too_small(void) 321 { 322 extern unsigned int ftr_fixup_test4; 323 extern unsigned int end_ftr_fixup_test4; 324 extern unsigned int ftr_fixup_test4_orig; 325 extern unsigned int ftr_fixup_test4_alt; 326 extern unsigned int ftr_fixup_test4_expected; 327 int size = &end_ftr_fixup_test4 - &ftr_fixup_test4; 328 unsigned long flag; 329 330 /* Check a high-bit flag */ 331 flag = 1UL << ((sizeof(unsigned long) - 1) * 8); 332 fixup.value = fixup.mask = flag; 333 fixup.start_off = calc_offset(&fixup, &ftr_fixup_test4 + 1); 334 fixup.end_off = calc_offset(&fixup, &ftr_fixup_test4 + 5); 335 fixup.alt_start_off = calc_offset(&fixup, &ftr_fixup_test4_alt); 336 fixup.alt_end_off = calc_offset(&fixup, &ftr_fixup_test4_alt + 2); 337 338 /* Sanity check */ 339 check(memcmp(&ftr_fixup_test4, &ftr_fixup_test4_orig, size) == 0); 340 341 /* Check we don't patch if the value matches */ 342 patch_feature_section(flag, &fixup); 343 check(memcmp(&ftr_fixup_test4, &ftr_fixup_test4_orig, size) == 0); 344 345 /* Check we do patch if the value doesn't match */ 346 patch_feature_section(0, &fixup); 347 check(memcmp(&ftr_fixup_test4, &ftr_fixup_test4_expected, size) == 0); 348 349 /* Check we do patch if the mask doesn't match */ 350 memcpy(&ftr_fixup_test4, &ftr_fixup_test4_orig, size); 351 check(memcmp(&ftr_fixup_test4, &ftr_fixup_test4_orig, size) == 0); 352 patch_feature_section(~flag, &fixup); 353 check(memcmp(&ftr_fixup_test4, &ftr_fixup_test4_expected, size) == 0); 354 } 355 356 static void test_alternative_case_with_branch(void) 357 { 358 extern unsigned int ftr_fixup_test5; 359 extern unsigned int end_ftr_fixup_test5; 360 extern unsigned int ftr_fixup_test5_expected; 361 int size = &end_ftr_fixup_test5 - &ftr_fixup_test5; 362 363 check(memcmp(&ftr_fixup_test5, &ftr_fixup_test5_expected, size) == 0); 364 } 365 366 static void test_alternative_case_with_external_branch(void) 367 { 368 extern unsigned int ftr_fixup_test6; 369 extern unsigned int end_ftr_fixup_test6; 370 extern unsigned int ftr_fixup_test6_expected; 371 int size = &end_ftr_fixup_test6 - &ftr_fixup_test6; 372 373 check(memcmp(&ftr_fixup_test6, &ftr_fixup_test6_expected, size) == 0); 374 } 375 376 static void test_cpu_macros(void) 377 { 378 extern u8 ftr_fixup_test_FTR_macros; 379 extern u8 ftr_fixup_test_FTR_macros_expected; 380 unsigned long size = &ftr_fixup_test_FTR_macros_expected - 381 &ftr_fixup_test_FTR_macros; 382 383 /* The fixups have already been done for us during boot */ 384 check(memcmp(&ftr_fixup_test_FTR_macros, 385 &ftr_fixup_test_FTR_macros_expected, size) == 0); 386 } 387 388 static void test_fw_macros(void) 389 { 390 #ifdef CONFIG_PPC64 391 extern u8 ftr_fixup_test_FW_FTR_macros; 392 extern u8 ftr_fixup_test_FW_FTR_macros_expected; 393 unsigned long size = &ftr_fixup_test_FW_FTR_macros_expected - 394 &ftr_fixup_test_FW_FTR_macros; 395 396 /* The fixups have already been done for us during boot */ 397 check(memcmp(&ftr_fixup_test_FW_FTR_macros, 398 &ftr_fixup_test_FW_FTR_macros_expected, size) == 0); 399 #endif 400 } 401 402 static void test_lwsync_macros(void) 403 { 404 extern u8 lwsync_fixup_test; 405 extern u8 end_lwsync_fixup_test; 406 extern u8 lwsync_fixup_test_expected_LWSYNC; 407 extern u8 lwsync_fixup_test_expected_SYNC; 408 unsigned long size = &end_lwsync_fixup_test - 409 &lwsync_fixup_test; 410 411 /* The fixups have already been done for us during boot */ 412 if (cur_cpu_spec->cpu_features & CPU_FTR_LWSYNC) { 413 check(memcmp(&lwsync_fixup_test, 414 &lwsync_fixup_test_expected_LWSYNC, size) == 0); 415 } else { 416 check(memcmp(&lwsync_fixup_test, 417 &lwsync_fixup_test_expected_SYNC, size) == 0); 418 } 419 } 420 421 static int __init test_feature_fixups(void) 422 { 423 printk(KERN_DEBUG "Running feature fixup self-tests ...\n"); 424 425 test_basic_patching(); 426 test_alternative_patching(); 427 test_alternative_case_too_big(); 428 test_alternative_case_too_small(); 429 test_alternative_case_with_branch(); 430 test_alternative_case_with_external_branch(); 431 test_cpu_macros(); 432 test_fw_macros(); 433 test_lwsync_macros(); 434 435 return 0; 436 } 437 late_initcall(test_feature_fixups); 438 439 #endif /* CONFIG_FTR_FIXUP_SELFTEST */ 440