1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 2011 by Kevin Cernekee (cernekee@gmail.com) 7 * 8 * SMP support for BMIPS 9 */ 10 11 #include <linux/version.h> 12 #include <linux/init.h> 13 #include <linux/sched.h> 14 #include <linux/mm.h> 15 #include <linux/delay.h> 16 #include <linux/smp.h> 17 #include <linux/interrupt.h> 18 #include <linux/spinlock.h> 19 #include <linux/init.h> 20 #include <linux/cpu.h> 21 #include <linux/cpumask.h> 22 #include <linux/reboot.h> 23 #include <linux/io.h> 24 #include <linux/compiler.h> 25 #include <linux/linkage.h> 26 #include <linux/bug.h> 27 #include <linux/kernel.h> 28 29 #include <asm/time.h> 30 #include <asm/pgtable.h> 31 #include <asm/processor.h> 32 #include <asm/system.h> 33 #include <asm/bootinfo.h> 34 #include <asm/pmon.h> 35 #include <asm/cacheflush.h> 36 #include <asm/tlbflush.h> 37 #include <asm/mipsregs.h> 38 #include <asm/bmips.h> 39 #include <asm/traps.h> 40 #include <asm/barrier.h> 41 42 static int __maybe_unused max_cpus = 1; 43 44 /* these may be configured by the platform code */ 45 int bmips_smp_enabled = 1; 46 int bmips_cpu_offset; 47 cpumask_t bmips_booted_mask; 48 49 #ifdef CONFIG_SMP 50 51 /* initial $sp, $gp - used by arch/mips/kernel/bmips_vec.S */ 52 unsigned long bmips_smp_boot_sp; 53 unsigned long bmips_smp_boot_gp; 54 55 static void bmips_send_ipi_single(int cpu, unsigned int action); 56 static irqreturn_t bmips_ipi_interrupt(int irq, void *dev_id); 57 58 /* SW interrupts 0,1 are used for interprocessor signaling */ 59 #define IPI0_IRQ (MIPS_CPU_IRQ_BASE + 0) 60 #define IPI1_IRQ (MIPS_CPU_IRQ_BASE + 1) 61 62 #define CPUNUM(cpu, shift) (((cpu) + bmips_cpu_offset) << (shift)) 63 #define ACTION_CLR_IPI(cpu, ipi) (0x2000 | CPUNUM(cpu, 9) | ((ipi) << 8)) 64 #define ACTION_SET_IPI(cpu, ipi) (0x3000 | CPUNUM(cpu, 9) | ((ipi) << 8)) 65 #define ACTION_BOOT_THREAD(cpu) (0x08 | CPUNUM(cpu, 0)) 66 67 static void __init bmips_smp_setup(void) 68 { 69 int i; 70 71 #if defined(CONFIG_CPU_BMIPS4350) || defined(CONFIG_CPU_BMIPS4380) 72 /* arbitration priority */ 73 clear_c0_brcm_cmt_ctrl(0x30); 74 75 /* NBK and weak order flags */ 76 set_c0_brcm_config_0(0x30000); 77 78 /* 79 * MIPS interrupts 0,1 (SW INT 0,1) cross over to the other thread 80 * MIPS interrupt 2 (HW INT 0) is the CPU0 L1 controller output 81 * MIPS interrupt 3 (HW INT 1) is the CPU1 L1 controller output 82 */ 83 change_c0_brcm_cmt_intr(0xf8018000, 84 (0x02 << 27) | (0x03 << 15)); 85 86 /* single core, 2 threads (2 pipelines) */ 87 max_cpus = 2; 88 #elif defined(CONFIG_CPU_BMIPS5000) 89 /* enable raceless SW interrupts */ 90 set_c0_brcm_config(0x03 << 22); 91 92 /* route HW interrupt 0 to CPU0, HW interrupt 1 to CPU1 */ 93 change_c0_brcm_mode(0x1f << 27, 0x02 << 27); 94 95 /* N cores, 2 threads per core */ 96 max_cpus = (((read_c0_brcm_config() >> 6) & 0x03) + 1) << 1; 97 98 /* clear any pending SW interrupts */ 99 for (i = 0; i < max_cpus; i++) { 100 write_c0_brcm_action(ACTION_CLR_IPI(i, 0)); 101 write_c0_brcm_action(ACTION_CLR_IPI(i, 1)); 102 } 103 #endif 104 105 if (!bmips_smp_enabled) 106 max_cpus = 1; 107 108 /* this can be overridden by the BSP */ 109 if (!board_ebase_setup) 110 board_ebase_setup = &bmips_ebase_setup; 111 112 for (i = 0; i < max_cpus; i++) { 113 __cpu_number_map[i] = 1; 114 __cpu_logical_map[i] = 1; 115 set_cpu_possible(i, 1); 116 set_cpu_present(i, 1); 117 } 118 } 119 120 /* 121 * IPI IRQ setup - runs on CPU0 122 */ 123 static void bmips_prepare_cpus(unsigned int max_cpus) 124 { 125 if (request_irq(IPI0_IRQ, bmips_ipi_interrupt, IRQF_PERCPU, 126 "smp_ipi0", NULL)) 127 panic("Can't request IPI0 interrupt\n"); 128 if (request_irq(IPI1_IRQ, bmips_ipi_interrupt, IRQF_PERCPU, 129 "smp_ipi1", NULL)) 130 panic("Can't request IPI1 interrupt\n"); 131 } 132 133 /* 134 * Tell the hardware to boot CPUx - runs on CPU0 135 */ 136 static void bmips_boot_secondary(int cpu, struct task_struct *idle) 137 { 138 bmips_smp_boot_sp = __KSTK_TOS(idle); 139 bmips_smp_boot_gp = (unsigned long)task_thread_info(idle); 140 mb(); 141 142 /* 143 * Initial boot sequence for secondary CPU: 144 * bmips_reset_nmi_vec @ a000_0000 -> 145 * bmips_smp_entry -> 146 * plat_wired_tlb_setup (cached function call; optional) -> 147 * start_secondary (cached jump) 148 * 149 * Warm restart sequence: 150 * play_dead WAIT loop -> 151 * bmips_smp_int_vec @ BMIPS_WARM_RESTART_VEC -> 152 * eret to play_dead -> 153 * bmips_secondary_reentry -> 154 * start_secondary 155 */ 156 157 pr_info("SMP: Booting CPU%d...\n", cpu); 158 159 if (cpumask_test_cpu(cpu, &bmips_booted_mask)) 160 bmips_send_ipi_single(cpu, 0); 161 else { 162 #if defined(CONFIG_CPU_BMIPS4350) || defined(CONFIG_CPU_BMIPS4380) 163 set_c0_brcm_cmt_ctrl(0x01); 164 #elif defined(CONFIG_CPU_BMIPS5000) 165 if (cpu & 0x01) 166 write_c0_brcm_action(ACTION_BOOT_THREAD(cpu)); 167 else { 168 /* 169 * core N thread 0 was already booted; just 170 * pulse the NMI line 171 */ 172 bmips_write_zscm_reg(0x210, 0xc0000000); 173 udelay(10); 174 bmips_write_zscm_reg(0x210, 0x00); 175 } 176 #endif 177 cpumask_set_cpu(cpu, &bmips_booted_mask); 178 } 179 } 180 181 /* 182 * Early setup - runs on secondary CPU after cache probe 183 */ 184 static void bmips_init_secondary(void) 185 { 186 /* move NMI vector to kseg0, in case XKS01 is enabled */ 187 188 #if defined(CONFIG_CPU_BMIPS4350) || defined(CONFIG_CPU_BMIPS4380) 189 void __iomem *cbr = BMIPS_GET_CBR(); 190 unsigned long old_vec; 191 192 old_vec = __raw_readl(cbr + BMIPS_RELO_VECTOR_CONTROL_1); 193 __raw_writel(old_vec & ~0x20000000, cbr + BMIPS_RELO_VECTOR_CONTROL_1); 194 195 clear_c0_cause(smp_processor_id() ? C_SW1 : C_SW0); 196 #elif defined(CONFIG_CPU_BMIPS5000) 197 write_c0_brcm_bootvec(read_c0_brcm_bootvec() & 198 (smp_processor_id() & 0x01 ? ~0x20000000 : ~0x2000)); 199 200 write_c0_brcm_action(ACTION_CLR_IPI(smp_processor_id(), 0)); 201 #endif 202 203 /* make sure there won't be a timer interrupt for a little while */ 204 write_c0_compare(read_c0_count() + mips_hpt_frequency / HZ); 205 206 irq_enable_hazard(); 207 set_c0_status(IE_SW0 | IE_SW1 | IE_IRQ1 | IE_IRQ5 | ST0_IE); 208 irq_enable_hazard(); 209 } 210 211 /* 212 * Late setup - runs on secondary CPU before entering the idle loop 213 */ 214 static void bmips_smp_finish(void) 215 { 216 pr_info("SMP: CPU%d is running\n", smp_processor_id()); 217 } 218 219 /* 220 * Runs on CPU0 after all CPUs have been booted 221 */ 222 static void bmips_cpus_done(void) 223 { 224 } 225 226 #if defined(CONFIG_CPU_BMIPS5000) 227 228 /* 229 * BMIPS5000 raceless IPIs 230 * 231 * Each CPU has two inbound SW IRQs which are independent of all other CPUs. 232 * IPI0 is used for SMP_RESCHEDULE_YOURSELF 233 * IPI1 is used for SMP_CALL_FUNCTION 234 */ 235 236 static void bmips_send_ipi_single(int cpu, unsigned int action) 237 { 238 write_c0_brcm_action(ACTION_SET_IPI(cpu, action == SMP_CALL_FUNCTION)); 239 } 240 241 static irqreturn_t bmips_ipi_interrupt(int irq, void *dev_id) 242 { 243 int action = irq - IPI0_IRQ; 244 245 write_c0_brcm_action(ACTION_CLR_IPI(smp_processor_id(), action)); 246 247 if (action == 0) 248 scheduler_ipi(); 249 else 250 smp_call_function_interrupt(); 251 252 return IRQ_HANDLED; 253 } 254 255 #else 256 257 /* 258 * BMIPS43xx racey IPIs 259 * 260 * We use one inbound SW IRQ for each CPU. 261 * 262 * A spinlock must be held in order to keep CPUx from accidentally clearing 263 * an incoming IPI when it writes CP0 CAUSE to raise an IPI on CPUy. The 264 * same spinlock is used to protect the action masks. 265 */ 266 267 static DEFINE_SPINLOCK(ipi_lock); 268 static DEFINE_PER_CPU(int, ipi_action_mask); 269 270 static void bmips_send_ipi_single(int cpu, unsigned int action) 271 { 272 unsigned long flags; 273 274 spin_lock_irqsave(&ipi_lock, flags); 275 set_c0_cause(cpu ? C_SW1 : C_SW0); 276 per_cpu(ipi_action_mask, cpu) |= action; 277 irq_enable_hazard(); 278 spin_unlock_irqrestore(&ipi_lock, flags); 279 } 280 281 static irqreturn_t bmips_ipi_interrupt(int irq, void *dev_id) 282 { 283 unsigned long flags; 284 int action, cpu = irq - IPI0_IRQ; 285 286 spin_lock_irqsave(&ipi_lock, flags); 287 action = __get_cpu_var(ipi_action_mask); 288 per_cpu(ipi_action_mask, cpu) = 0; 289 clear_c0_cause(cpu ? C_SW1 : C_SW0); 290 spin_unlock_irqrestore(&ipi_lock, flags); 291 292 if (action & SMP_RESCHEDULE_YOURSELF) 293 scheduler_ipi(); 294 if (action & SMP_CALL_FUNCTION) 295 smp_call_function_interrupt(); 296 297 return IRQ_HANDLED; 298 } 299 300 #endif /* BMIPS type */ 301 302 static void bmips_send_ipi_mask(const struct cpumask *mask, 303 unsigned int action) 304 { 305 unsigned int i; 306 307 for_each_cpu(i, mask) 308 bmips_send_ipi_single(i, action); 309 } 310 311 #ifdef CONFIG_HOTPLUG_CPU 312 313 static int bmips_cpu_disable(void) 314 { 315 unsigned int cpu = smp_processor_id(); 316 317 if (cpu == 0) 318 return -EBUSY; 319 320 pr_info("SMP: CPU%d is offline\n", cpu); 321 322 cpu_clear(cpu, cpu_online_map); 323 cpu_clear(cpu, cpu_callin_map); 324 325 local_flush_tlb_all(); 326 local_flush_icache_range(0, ~0); 327 328 return 0; 329 } 330 331 static void bmips_cpu_die(unsigned int cpu) 332 { 333 } 334 335 void __ref play_dead(void) 336 { 337 idle_task_exit(); 338 339 /* flush data cache */ 340 _dma_cache_wback_inv(0, ~0); 341 342 /* 343 * Wakeup is on SW0 or SW1; disable everything else 344 * Use BEV !IV (BMIPS_WARM_RESTART_VEC) to avoid the regular Linux 345 * IRQ handlers; this clears ST0_IE and returns immediately. 346 */ 347 clear_c0_cause(CAUSEF_IV | C_SW0 | C_SW1); 348 change_c0_status(IE_IRQ5 | IE_IRQ1 | IE_SW0 | IE_SW1 | ST0_IE | ST0_BEV, 349 IE_SW0 | IE_SW1 | ST0_IE | ST0_BEV); 350 irq_disable_hazard(); 351 352 /* 353 * wait for SW interrupt from bmips_boot_secondary(), then jump 354 * back to start_secondary() 355 */ 356 __asm__ __volatile__( 357 " wait\n" 358 " j bmips_secondary_reentry\n" 359 : : : "memory"); 360 } 361 362 #endif /* CONFIG_HOTPLUG_CPU */ 363 364 struct plat_smp_ops bmips_smp_ops = { 365 .smp_setup = bmips_smp_setup, 366 .prepare_cpus = bmips_prepare_cpus, 367 .boot_secondary = bmips_boot_secondary, 368 .smp_finish = bmips_smp_finish, 369 .init_secondary = bmips_init_secondary, 370 .cpus_done = bmips_cpus_done, 371 .send_ipi_single = bmips_send_ipi_single, 372 .send_ipi_mask = bmips_send_ipi_mask, 373 #ifdef CONFIG_HOTPLUG_CPU 374 .cpu_disable = bmips_cpu_disable, 375 .cpu_die = bmips_cpu_die, 376 #endif 377 }; 378 379 #endif /* CONFIG_SMP */ 380 381 /*********************************************************************** 382 * BMIPS vector relocation 383 * This is primarily used for SMP boot, but it is applicable to some 384 * UP BMIPS systems as well. 385 ***********************************************************************/ 386 387 static void __cpuinit bmips_wr_vec(unsigned long dst, char *start, char *end) 388 { 389 memcpy((void *)dst, start, end - start); 390 dma_cache_wback((unsigned long)start, end - start); 391 local_flush_icache_range(dst, dst + (end - start)); 392 instruction_hazard(); 393 } 394 395 static inline void __cpuinit bmips_nmi_handler_setup(void) 396 { 397 bmips_wr_vec(BMIPS_NMI_RESET_VEC, &bmips_reset_nmi_vec, 398 &bmips_reset_nmi_vec_end); 399 bmips_wr_vec(BMIPS_WARM_RESTART_VEC, &bmips_smp_int_vec, 400 &bmips_smp_int_vec_end); 401 } 402 403 void __cpuinit bmips_ebase_setup(void) 404 { 405 unsigned long new_ebase = ebase; 406 void __iomem __maybe_unused *cbr; 407 408 BUG_ON(ebase != CKSEG0); 409 410 #if defined(CONFIG_CPU_BMIPS4350) 411 /* 412 * BMIPS4350 cannot relocate the normal vectors, but it 413 * can relocate the BEV=1 vectors. So CPU1 starts up at 414 * the relocated BEV=1, IV=0 general exception vector @ 415 * 0xa000_0380. 416 * 417 * set_uncached_handler() is used here because: 418 * - CPU1 will run this from uncached space 419 * - None of the cacheflush functions are set up yet 420 */ 421 set_uncached_handler(BMIPS_WARM_RESTART_VEC - CKSEG0, 422 &bmips_smp_int_vec, 0x80); 423 __sync(); 424 return; 425 #elif defined(CONFIG_CPU_BMIPS4380) 426 /* 427 * 0x8000_0000: reset/NMI (initially in kseg1) 428 * 0x8000_0400: normal vectors 429 */ 430 new_ebase = 0x80000400; 431 cbr = BMIPS_GET_CBR(); 432 __raw_writel(0x80080800, cbr + BMIPS_RELO_VECTOR_CONTROL_0); 433 __raw_writel(0xa0080800, cbr + BMIPS_RELO_VECTOR_CONTROL_1); 434 #elif defined(CONFIG_CPU_BMIPS5000) 435 /* 436 * 0x8000_0000: reset/NMI (initially in kseg1) 437 * 0x8000_1000: normal vectors 438 */ 439 new_ebase = 0x80001000; 440 write_c0_brcm_bootvec(0xa0088008); 441 write_c0_ebase(new_ebase); 442 if (max_cpus > 2) 443 bmips_write_zscm_reg(0xa0, 0xa008a008); 444 #else 445 return; 446 #endif 447 board_nmi_handler_setup = &bmips_nmi_handler_setup; 448 ebase = new_ebase; 449 } 450 451 asmlinkage void __weak plat_wired_tlb_setup(void) 452 { 453 /* 454 * Called when starting/restarting a secondary CPU. 455 * Kernel stacks and other important data might only be accessible 456 * once the wired entries are present. 457 */ 458 } 459