1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 2011 by Kevin Cernekee (cernekee@gmail.com) 7 * 8 * SMP support for BMIPS 9 */ 10 11 #include <linux/init.h> 12 #include <linux/sched.h> 13 #include <linux/mm.h> 14 #include <linux/delay.h> 15 #include <linux/smp.h> 16 #include <linux/interrupt.h> 17 #include <linux/spinlock.h> 18 #include <linux/init.h> 19 #include <linux/cpu.h> 20 #include <linux/cpumask.h> 21 #include <linux/reboot.h> 22 #include <linux/io.h> 23 #include <linux/compiler.h> 24 #include <linux/linkage.h> 25 #include <linux/bug.h> 26 #include <linux/kernel.h> 27 28 #include <asm/time.h> 29 #include <asm/pgtable.h> 30 #include <asm/processor.h> 31 #include <asm/system.h> 32 #include <asm/bootinfo.h> 33 #include <asm/pmon.h> 34 #include <asm/cacheflush.h> 35 #include <asm/tlbflush.h> 36 #include <asm/mipsregs.h> 37 #include <asm/bmips.h> 38 #include <asm/traps.h> 39 #include <asm/barrier.h> 40 41 static int __maybe_unused max_cpus = 1; 42 43 /* these may be configured by the platform code */ 44 int bmips_smp_enabled = 1; 45 int bmips_cpu_offset; 46 cpumask_t bmips_booted_mask; 47 48 #ifdef CONFIG_SMP 49 50 /* initial $sp, $gp - used by arch/mips/kernel/bmips_vec.S */ 51 unsigned long bmips_smp_boot_sp; 52 unsigned long bmips_smp_boot_gp; 53 54 static void bmips_send_ipi_single(int cpu, unsigned int action); 55 static irqreturn_t bmips_ipi_interrupt(int irq, void *dev_id); 56 57 /* SW interrupts 0,1 are used for interprocessor signaling */ 58 #define IPI0_IRQ (MIPS_CPU_IRQ_BASE + 0) 59 #define IPI1_IRQ (MIPS_CPU_IRQ_BASE + 1) 60 61 #define CPUNUM(cpu, shift) (((cpu) + bmips_cpu_offset) << (shift)) 62 #define ACTION_CLR_IPI(cpu, ipi) (0x2000 | CPUNUM(cpu, 9) | ((ipi) << 8)) 63 #define ACTION_SET_IPI(cpu, ipi) (0x3000 | CPUNUM(cpu, 9) | ((ipi) << 8)) 64 #define ACTION_BOOT_THREAD(cpu) (0x08 | CPUNUM(cpu, 0)) 65 66 static void __init bmips_smp_setup(void) 67 { 68 int i; 69 70 #if defined(CONFIG_CPU_BMIPS4350) || defined(CONFIG_CPU_BMIPS4380) 71 /* arbitration priority */ 72 clear_c0_brcm_cmt_ctrl(0x30); 73 74 /* NBK and weak order flags */ 75 set_c0_brcm_config_0(0x30000); 76 77 /* 78 * MIPS interrupts 0,1 (SW INT 0,1) cross over to the other thread 79 * MIPS interrupt 2 (HW INT 0) is the CPU0 L1 controller output 80 * MIPS interrupt 3 (HW INT 1) is the CPU1 L1 controller output 81 */ 82 change_c0_brcm_cmt_intr(0xf8018000, 83 (0x02 << 27) | (0x03 << 15)); 84 85 /* single core, 2 threads (2 pipelines) */ 86 max_cpus = 2; 87 #elif defined(CONFIG_CPU_BMIPS5000) 88 /* enable raceless SW interrupts */ 89 set_c0_brcm_config(0x03 << 22); 90 91 /* route HW interrupt 0 to CPU0, HW interrupt 1 to CPU1 */ 92 change_c0_brcm_mode(0x1f << 27, 0x02 << 27); 93 94 /* N cores, 2 threads per core */ 95 max_cpus = (((read_c0_brcm_config() >> 6) & 0x03) + 1) << 1; 96 97 /* clear any pending SW interrupts */ 98 for (i = 0; i < max_cpus; i++) { 99 write_c0_brcm_action(ACTION_CLR_IPI(i, 0)); 100 write_c0_brcm_action(ACTION_CLR_IPI(i, 1)); 101 } 102 #endif 103 104 if (!bmips_smp_enabled) 105 max_cpus = 1; 106 107 /* this can be overridden by the BSP */ 108 if (!board_ebase_setup) 109 board_ebase_setup = &bmips_ebase_setup; 110 111 for (i = 0; i < max_cpus; i++) { 112 __cpu_number_map[i] = 1; 113 __cpu_logical_map[i] = 1; 114 set_cpu_possible(i, 1); 115 set_cpu_present(i, 1); 116 } 117 } 118 119 /* 120 * IPI IRQ setup - runs on CPU0 121 */ 122 static void bmips_prepare_cpus(unsigned int max_cpus) 123 { 124 if (request_irq(IPI0_IRQ, bmips_ipi_interrupt, IRQF_PERCPU, 125 "smp_ipi0", NULL)) 126 panic("Can't request IPI0 interrupt\n"); 127 if (request_irq(IPI1_IRQ, bmips_ipi_interrupt, IRQF_PERCPU, 128 "smp_ipi1", NULL)) 129 panic("Can't request IPI1 interrupt\n"); 130 } 131 132 /* 133 * Tell the hardware to boot CPUx - runs on CPU0 134 */ 135 static void bmips_boot_secondary(int cpu, struct task_struct *idle) 136 { 137 bmips_smp_boot_sp = __KSTK_TOS(idle); 138 bmips_smp_boot_gp = (unsigned long)task_thread_info(idle); 139 mb(); 140 141 /* 142 * Initial boot sequence for secondary CPU: 143 * bmips_reset_nmi_vec @ a000_0000 -> 144 * bmips_smp_entry -> 145 * plat_wired_tlb_setup (cached function call; optional) -> 146 * start_secondary (cached jump) 147 * 148 * Warm restart sequence: 149 * play_dead WAIT loop -> 150 * bmips_smp_int_vec @ BMIPS_WARM_RESTART_VEC -> 151 * eret to play_dead -> 152 * bmips_secondary_reentry -> 153 * start_secondary 154 */ 155 156 pr_info("SMP: Booting CPU%d...\n", cpu); 157 158 if (cpumask_test_cpu(cpu, &bmips_booted_mask)) 159 bmips_send_ipi_single(cpu, 0); 160 else { 161 #if defined(CONFIG_CPU_BMIPS4350) || defined(CONFIG_CPU_BMIPS4380) 162 set_c0_brcm_cmt_ctrl(0x01); 163 #elif defined(CONFIG_CPU_BMIPS5000) 164 if (cpu & 0x01) 165 write_c0_brcm_action(ACTION_BOOT_THREAD(cpu)); 166 else { 167 /* 168 * core N thread 0 was already booted; just 169 * pulse the NMI line 170 */ 171 bmips_write_zscm_reg(0x210, 0xc0000000); 172 udelay(10); 173 bmips_write_zscm_reg(0x210, 0x00); 174 } 175 #endif 176 cpumask_set_cpu(cpu, &bmips_booted_mask); 177 } 178 } 179 180 /* 181 * Early setup - runs on secondary CPU after cache probe 182 */ 183 static void bmips_init_secondary(void) 184 { 185 /* move NMI vector to kseg0, in case XKS01 is enabled */ 186 187 #if defined(CONFIG_CPU_BMIPS4350) || defined(CONFIG_CPU_BMIPS4380) 188 void __iomem *cbr = BMIPS_GET_CBR(); 189 unsigned long old_vec; 190 191 old_vec = __raw_readl(cbr + BMIPS_RELO_VECTOR_CONTROL_1); 192 __raw_writel(old_vec & ~0x20000000, cbr + BMIPS_RELO_VECTOR_CONTROL_1); 193 194 clear_c0_cause(smp_processor_id() ? C_SW1 : C_SW0); 195 #elif defined(CONFIG_CPU_BMIPS5000) 196 write_c0_brcm_bootvec(read_c0_brcm_bootvec() & 197 (smp_processor_id() & 0x01 ? ~0x20000000 : ~0x2000)); 198 199 write_c0_brcm_action(ACTION_CLR_IPI(smp_processor_id(), 0)); 200 #endif 201 202 /* make sure there won't be a timer interrupt for a little while */ 203 write_c0_compare(read_c0_count() + mips_hpt_frequency / HZ); 204 205 irq_enable_hazard(); 206 set_c0_status(IE_SW0 | IE_SW1 | IE_IRQ1 | IE_IRQ5 | ST0_IE); 207 irq_enable_hazard(); 208 } 209 210 /* 211 * Late setup - runs on secondary CPU before entering the idle loop 212 */ 213 static void bmips_smp_finish(void) 214 { 215 pr_info("SMP: CPU%d is running\n", smp_processor_id()); 216 } 217 218 /* 219 * Runs on CPU0 after all CPUs have been booted 220 */ 221 static void bmips_cpus_done(void) 222 { 223 } 224 225 #if defined(CONFIG_CPU_BMIPS5000) 226 227 /* 228 * BMIPS5000 raceless IPIs 229 * 230 * Each CPU has two inbound SW IRQs which are independent of all other CPUs. 231 * IPI0 is used for SMP_RESCHEDULE_YOURSELF 232 * IPI1 is used for SMP_CALL_FUNCTION 233 */ 234 235 static void bmips_send_ipi_single(int cpu, unsigned int action) 236 { 237 write_c0_brcm_action(ACTION_SET_IPI(cpu, action == SMP_CALL_FUNCTION)); 238 } 239 240 static irqreturn_t bmips_ipi_interrupt(int irq, void *dev_id) 241 { 242 int action = irq - IPI0_IRQ; 243 244 write_c0_brcm_action(ACTION_CLR_IPI(smp_processor_id(), action)); 245 246 if (action == 0) 247 scheduler_ipi(); 248 else 249 smp_call_function_interrupt(); 250 251 return IRQ_HANDLED; 252 } 253 254 #else 255 256 /* 257 * BMIPS43xx racey IPIs 258 * 259 * We use one inbound SW IRQ for each CPU. 260 * 261 * A spinlock must be held in order to keep CPUx from accidentally clearing 262 * an incoming IPI when it writes CP0 CAUSE to raise an IPI on CPUy. The 263 * same spinlock is used to protect the action masks. 264 */ 265 266 static DEFINE_SPINLOCK(ipi_lock); 267 static DEFINE_PER_CPU(int, ipi_action_mask); 268 269 static void bmips_send_ipi_single(int cpu, unsigned int action) 270 { 271 unsigned long flags; 272 273 spin_lock_irqsave(&ipi_lock, flags); 274 set_c0_cause(cpu ? C_SW1 : C_SW0); 275 per_cpu(ipi_action_mask, cpu) |= action; 276 irq_enable_hazard(); 277 spin_unlock_irqrestore(&ipi_lock, flags); 278 } 279 280 static irqreturn_t bmips_ipi_interrupt(int irq, void *dev_id) 281 { 282 unsigned long flags; 283 int action, cpu = irq - IPI0_IRQ; 284 285 spin_lock_irqsave(&ipi_lock, flags); 286 action = __get_cpu_var(ipi_action_mask); 287 per_cpu(ipi_action_mask, cpu) = 0; 288 clear_c0_cause(cpu ? C_SW1 : C_SW0); 289 spin_unlock_irqrestore(&ipi_lock, flags); 290 291 if (action & SMP_RESCHEDULE_YOURSELF) 292 scheduler_ipi(); 293 if (action & SMP_CALL_FUNCTION) 294 smp_call_function_interrupt(); 295 296 return IRQ_HANDLED; 297 } 298 299 #endif /* BMIPS type */ 300 301 static void bmips_send_ipi_mask(const struct cpumask *mask, 302 unsigned int action) 303 { 304 unsigned int i; 305 306 for_each_cpu(i, mask) 307 bmips_send_ipi_single(i, action); 308 } 309 310 #ifdef CONFIG_HOTPLUG_CPU 311 312 static int bmips_cpu_disable(void) 313 { 314 unsigned int cpu = smp_processor_id(); 315 316 if (cpu == 0) 317 return -EBUSY; 318 319 pr_info("SMP: CPU%d is offline\n", cpu); 320 321 cpu_clear(cpu, cpu_online_map); 322 cpu_clear(cpu, cpu_callin_map); 323 324 local_flush_tlb_all(); 325 local_flush_icache_range(0, ~0); 326 327 return 0; 328 } 329 330 static void bmips_cpu_die(unsigned int cpu) 331 { 332 } 333 334 void __ref play_dead(void) 335 { 336 idle_task_exit(); 337 338 /* flush data cache */ 339 _dma_cache_wback_inv(0, ~0); 340 341 /* 342 * Wakeup is on SW0 or SW1; disable everything else 343 * Use BEV !IV (BMIPS_WARM_RESTART_VEC) to avoid the regular Linux 344 * IRQ handlers; this clears ST0_IE and returns immediately. 345 */ 346 clear_c0_cause(CAUSEF_IV | C_SW0 | C_SW1); 347 change_c0_status(IE_IRQ5 | IE_IRQ1 | IE_SW0 | IE_SW1 | ST0_IE | ST0_BEV, 348 IE_SW0 | IE_SW1 | ST0_IE | ST0_BEV); 349 irq_disable_hazard(); 350 351 /* 352 * wait for SW interrupt from bmips_boot_secondary(), then jump 353 * back to start_secondary() 354 */ 355 __asm__ __volatile__( 356 " wait\n" 357 " j bmips_secondary_reentry\n" 358 : : : "memory"); 359 } 360 361 #endif /* CONFIG_HOTPLUG_CPU */ 362 363 struct plat_smp_ops bmips_smp_ops = { 364 .smp_setup = bmips_smp_setup, 365 .prepare_cpus = bmips_prepare_cpus, 366 .boot_secondary = bmips_boot_secondary, 367 .smp_finish = bmips_smp_finish, 368 .init_secondary = bmips_init_secondary, 369 .cpus_done = bmips_cpus_done, 370 .send_ipi_single = bmips_send_ipi_single, 371 .send_ipi_mask = bmips_send_ipi_mask, 372 #ifdef CONFIG_HOTPLUG_CPU 373 .cpu_disable = bmips_cpu_disable, 374 .cpu_die = bmips_cpu_die, 375 #endif 376 }; 377 378 #endif /* CONFIG_SMP */ 379 380 /*********************************************************************** 381 * BMIPS vector relocation 382 * This is primarily used for SMP boot, but it is applicable to some 383 * UP BMIPS systems as well. 384 ***********************************************************************/ 385 386 static void __cpuinit bmips_wr_vec(unsigned long dst, char *start, char *end) 387 { 388 memcpy((void *)dst, start, end - start); 389 dma_cache_wback((unsigned long)start, end - start); 390 local_flush_icache_range(dst, dst + (end - start)); 391 instruction_hazard(); 392 } 393 394 static inline void __cpuinit bmips_nmi_handler_setup(void) 395 { 396 bmips_wr_vec(BMIPS_NMI_RESET_VEC, &bmips_reset_nmi_vec, 397 &bmips_reset_nmi_vec_end); 398 bmips_wr_vec(BMIPS_WARM_RESTART_VEC, &bmips_smp_int_vec, 399 &bmips_smp_int_vec_end); 400 } 401 402 void __cpuinit bmips_ebase_setup(void) 403 { 404 unsigned long new_ebase = ebase; 405 void __iomem __maybe_unused *cbr; 406 407 BUG_ON(ebase != CKSEG0); 408 409 #if defined(CONFIG_CPU_BMIPS4350) 410 /* 411 * BMIPS4350 cannot relocate the normal vectors, but it 412 * can relocate the BEV=1 vectors. So CPU1 starts up at 413 * the relocated BEV=1, IV=0 general exception vector @ 414 * 0xa000_0380. 415 * 416 * set_uncached_handler() is used here because: 417 * - CPU1 will run this from uncached space 418 * - None of the cacheflush functions are set up yet 419 */ 420 set_uncached_handler(BMIPS_WARM_RESTART_VEC - CKSEG0, 421 &bmips_smp_int_vec, 0x80); 422 __sync(); 423 return; 424 #elif defined(CONFIG_CPU_BMIPS4380) 425 /* 426 * 0x8000_0000: reset/NMI (initially in kseg1) 427 * 0x8000_0400: normal vectors 428 */ 429 new_ebase = 0x80000400; 430 cbr = BMIPS_GET_CBR(); 431 __raw_writel(0x80080800, cbr + BMIPS_RELO_VECTOR_CONTROL_0); 432 __raw_writel(0xa0080800, cbr + BMIPS_RELO_VECTOR_CONTROL_1); 433 #elif defined(CONFIG_CPU_BMIPS5000) 434 /* 435 * 0x8000_0000: reset/NMI (initially in kseg1) 436 * 0x8000_1000: normal vectors 437 */ 438 new_ebase = 0x80001000; 439 write_c0_brcm_bootvec(0xa0088008); 440 write_c0_ebase(new_ebase); 441 if (max_cpus > 2) 442 bmips_write_zscm_reg(0xa0, 0xa008a008); 443 #else 444 return; 445 #endif 446 board_nmi_handler_setup = &bmips_nmi_handler_setup; 447 ebase = new_ebase; 448 } 449 450 asmlinkage void __weak plat_wired_tlb_setup(void) 451 { 452 /* 453 * Called when starting/restarting a secondary CPU. 454 * Kernel stacks and other important data might only be accessible 455 * once the wired entries are present. 456 */ 457 } 458