1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2/* 3 * Copyright (C) 2013 Imagination Technologies 4 * Author: Paul Burton <paul.burton@mips.com> 5 */ 6 7#include <linux/init.h> 8#include <asm/addrspace.h> 9#include <asm/asm.h> 10#include <asm/asm-offsets.h> 11#include <asm/asmmacro.h> 12#include <asm/cacheops.h> 13#include <asm/eva.h> 14#include <asm/mipsregs.h> 15#include <asm/mipsmtregs.h> 16#include <asm/pm.h> 17#include <asm/smp-cps.h> 18 19#define GCR_CPC_BASE_OFS 0x0088 20#define GCR_CL_COHERENCE_OFS 0x2008 21#define GCR_CL_ID_OFS 0x2028 22#define CM3_GCR_Cx_ID_CLUSTER_SHF 8 23#define CM3_GCR_Cx_ID_CLUSTER_MSK (0xff << 8) 24#define CM3_GCR_Cx_ID_CORENUM_SHF 0 25#define CM3_GCR_Cx_ID_CORENUM_MSK (0xff << 0) 26 27#define CPC_CL_VC_STOP_OFS 0x2020 28#define CPC_CL_VC_RUN_OFS 0x2028 29 30.extern mips_cm_base 31 32.set noreorder 33 34#ifdef CONFIG_64BIT 35# define STATUS_BITDEPS ST0_KX 36#else 37# define STATUS_BITDEPS 0 38#endif 39 40#ifdef CONFIG_MIPS_CPS_NS16550 41 42#define DUMP_EXCEP(name) \ 43 PTR_LA a0, 8f; \ 44 jal mips_cps_bev_dump; \ 45 nop; \ 46 TEXT(name) 47 48#else /* !CONFIG_MIPS_CPS_NS16550 */ 49 50#define DUMP_EXCEP(name) 51 52#endif /* !CONFIG_MIPS_CPS_NS16550 */ 53 54 /* 55 * Set dest to non-zero if the core supports the MT ASE, else zero. If 56 * MT is not supported then branch to nomt. 57 */ 58 .macro has_mt dest, nomt 59 mfc0 \dest, CP0_CONFIG, 1 60 bgez \dest, \nomt 61 mfc0 \dest, CP0_CONFIG, 2 62 bgez \dest, \nomt 63 mfc0 \dest, CP0_CONFIG, 3 64 andi \dest, \dest, MIPS_CONF3_MT 65 beqz \dest, \nomt 66 nop 67 .endm 68 69 /* 70 * Set dest to non-zero if the core supports MIPSr6 multithreading 71 * (ie. VPs), else zero. If MIPSr6 multithreading is not supported then 72 * branch to nomt. 73 */ 74 .macro has_vp dest, nomt 75 mfc0 \dest, CP0_CONFIG, 1 76 bgez \dest, \nomt 77 mfc0 \dest, CP0_CONFIG, 2 78 bgez \dest, \nomt 79 mfc0 \dest, CP0_CONFIG, 3 80 bgez \dest, \nomt 81 mfc0 \dest, CP0_CONFIG, 4 82 bgez \dest, \nomt 83 mfc0 \dest, CP0_CONFIG, 5 84 andi \dest, \dest, MIPS_CONF5_VP 85 beqz \dest, \nomt 86 nop 87 .endm 88 89 90LEAF(mips_cps_core_boot) 91 /* Save CCA and GCR base */ 92 move s0, a0 93 move s1, a1 94 95 /* We don't know how to do coherence setup on earlier ISA */ 96#if MIPS_ISA_REV > 0 97 /* Skip cache & coherence setup if we're already coherent */ 98 lw s7, GCR_CL_COHERENCE_OFS(s1) 99 bnez s7, 1f 100 nop 101 102 /* Initialize the L1 caches */ 103 jal mips_cps_cache_init 104 nop 105 106 /* Enter the coherent domain */ 107 li t0, 0xff 108 sw t0, GCR_CL_COHERENCE_OFS(s1) 109 ehb 110#endif /* MIPS_ISA_REV > 0 */ 111 112 /* Set Kseg0 CCA to that in s0 */ 1131: mfc0 t0, CP0_CONFIG 114 ori t0, 0x7 115 xori t0, 0x7 116 or t0, t0, s0 117 mtc0 t0, CP0_CONFIG 118 ehb 119 120 /* Jump to kseg0 */ 121 PTR_LA t0, 1f 122 jr t0 123 nop 124 125 /* 126 * We're up, cached & coherent. Perform any EVA initialization necessary 127 * before we access memory. 128 */ 1291: eva_init 130 131 /* Retrieve boot configuration pointers */ 132 jal mips_cps_get_bootcfg 133 nop 134 135 /* Skip core-level init if we started up coherent */ 136 bnez s7, 1f 137 nop 138 139 /* Perform any further required core-level initialisation */ 140 jal mips_cps_core_init 141 nop 142 143 /* 144 * Boot any other VPEs within this core that should be online, and 145 * deactivate this VPE if it should be offline. 146 */ 147 move a1, t9 148 jal mips_cps_boot_vpes 149 move a0, v0 150 151 /* Off we go! */ 1521: PTR_L t1, VPEBOOTCFG_PC(v1) 153 PTR_L gp, VPEBOOTCFG_GP(v1) 154 PTR_L sp, VPEBOOTCFG_SP(v1) 155 jr t1 156 nop 157 END(mips_cps_core_boot) 158 159 __INIT 160LEAF(excep_tlbfill) 161 DUMP_EXCEP("TLB Fill") 162 b . 163 nop 164 END(excep_tlbfill) 165 166LEAF(excep_xtlbfill) 167 DUMP_EXCEP("XTLB Fill") 168 b . 169 nop 170 END(excep_xtlbfill) 171 172LEAF(excep_cache) 173 DUMP_EXCEP("Cache") 174 b . 175 nop 176 END(excep_cache) 177 178LEAF(excep_genex) 179 DUMP_EXCEP("General") 180 b . 181 nop 182 END(excep_genex) 183 184LEAF(excep_intex) 185 DUMP_EXCEP("Interrupt") 186 b . 187 nop 188 END(excep_intex) 189 190LEAF(excep_ejtag) 191 PTR_LA k0, ejtag_debug_handler 192 jr k0 193 nop 194 END(excep_ejtag) 195 __FINIT 196 197LEAF(mips_cps_core_init) 198#ifdef CONFIG_MIPS_MT_SMP 199 /* Check that the core implements the MT ASE */ 200 has_mt t0, 3f 201 202 .set push 203 .set MIPS_ISA_LEVEL_RAW 204 .set mt 205 206 /* Only allow 1 TC per VPE to execute... */ 207 dmt 208 209 /* ...and for the moment only 1 VPE */ 210 dvpe 211 PTR_LA t1, 1f 212 jr.hb t1 213 nop 214 215 /* Enter VPE configuration state */ 2161: mfc0 t0, CP0_MVPCONTROL 217 ori t0, t0, MVPCONTROL_VPC 218 mtc0 t0, CP0_MVPCONTROL 219 220 /* Retrieve the number of VPEs within the core */ 221 mfc0 t0, CP0_MVPCONF0 222 srl t0, t0, MVPCONF0_PVPE_SHIFT 223 andi t0, t0, (MVPCONF0_PVPE >> MVPCONF0_PVPE_SHIFT) 224 addiu ta3, t0, 1 225 226 /* If there's only 1, we're done */ 227 beqz t0, 2f 228 nop 229 230 /* Loop through each VPE within this core */ 231 li ta1, 1 232 2331: /* Operate on the appropriate TC */ 234 mtc0 ta1, CP0_VPECONTROL 235 ehb 236 237 /* Bind TC to VPE (1:1 TC:VPE mapping) */ 238 mttc0 ta1, CP0_TCBIND 239 240 /* Set exclusive TC, non-active, master */ 241 li t0, VPECONF0_MVP 242 sll t1, ta1, VPECONF0_XTC_SHIFT 243 or t0, t0, t1 244 mttc0 t0, CP0_VPECONF0 245 246 /* Set TC non-active, non-allocatable */ 247 mttc0 zero, CP0_TCSTATUS 248 249 /* Set TC halted */ 250 li t0, TCHALT_H 251 mttc0 t0, CP0_TCHALT 252 253 /* Next VPE */ 254 addiu ta1, ta1, 1 255 slt t0, ta1, ta3 256 bnez t0, 1b 257 nop 258 259 /* Leave VPE configuration state */ 2602: mfc0 t0, CP0_MVPCONTROL 261 xori t0, t0, MVPCONTROL_VPC 262 mtc0 t0, CP0_MVPCONTROL 263 2643: .set pop 265#endif 266 jr ra 267 nop 268 END(mips_cps_core_init) 269 270/** 271 * mips_cps_get_bootcfg() - retrieve boot configuration pointers 272 * 273 * Returns: pointer to struct core_boot_config in v0, pointer to 274 * struct vpe_boot_config in v1, VPE ID in t9 275 */ 276LEAF(mips_cps_get_bootcfg) 277 /* Calculate a pointer to this cores struct core_boot_config */ 278 PTR_LA v0, mips_cps_cluster_bootcfg 279 PTR_L v0, 0(v0) 280 lw t0, GCR_CL_ID_OFS(s1) 281#ifdef CONFIG_CPU_MIPSR6 282 ext t1, t0, CM3_GCR_Cx_ID_CLUSTER_SHF, 8 283 li t2, CLUSTERBOOTCFG_SIZE 284 mul t1, t1, t2 285 PTR_ADDU \ 286 v0, v0, t1 287#endif 288 PTR_L v0, CLUSTERBOOTCFG_CORECONFIG(v0) 289 andi t0, t0, CM3_GCR_Cx_ID_CORENUM_MSK 290 li t1, COREBOOTCFG_SIZE 291 mul t0, t0, t1 292 PTR_ADDU v0, v0, t0 293 294 /* Calculate this VPEs ID. If the core doesn't support MT use 0 */ 295 li t9, 0 296#if defined(CONFIG_CPU_MIPSR6) 297 has_vp ta2, 1f 298 299 /* 300 * Assume non-contiguous numbering. Perhaps some day we'll need 301 * to handle contiguous VP numbering, but no such systems yet 302 * exist. 303 */ 304 mfc0 t9, CP0_GLOBALNUMBER 305 andi t9, t9, MIPS_GLOBALNUMBER_VP 306#elif defined(CONFIG_MIPS_MT_SMP) 307 has_mt ta2, 1f 308 309 /* Find the number of VPEs present in the core */ 310 mfc0 t1, CP0_MVPCONF0 311 srl t1, t1, MVPCONF0_PVPE_SHIFT 312 andi t1, t1, MVPCONF0_PVPE >> MVPCONF0_PVPE_SHIFT 313 addiu t1, t1, 1 314 315 /* Calculate a mask for the VPE ID from EBase.CPUNum */ 316 clz t1, t1 317 li t2, 31 318 subu t1, t2, t1 319 li t2, 1 320 sll t1, t2, t1 321 addiu t1, t1, -1 322 323 /* Retrieve the VPE ID from EBase.CPUNum */ 324 mfc0 t9, $15, 1 325 and t9, t9, t1 326#endif 327 3281: /* Calculate a pointer to this VPEs struct vpe_boot_config */ 329 li t1, VPEBOOTCFG_SIZE 330 mul v1, t9, t1 331 PTR_L ta3, COREBOOTCFG_VPECONFIG(v0) 332 PTR_ADDU v1, v1, ta3 333 334 jr ra 335 nop 336 END(mips_cps_get_bootcfg) 337 338LEAF(mips_cps_boot_vpes) 339 lw ta2, COREBOOTCFG_VPEMASK(a0) 340 PTR_L ta3, COREBOOTCFG_VPECONFIG(a0) 341 342#if defined(CONFIG_CPU_MIPSR6) 343 344 has_vp t0, 5f 345 346 /* Find base address of CPC */ 347 PTR_LA t1, mips_gcr_base 348 PTR_L t1, 0(t1) 349 PTR_L t1, GCR_CPC_BASE_OFS(t1) 350 PTR_LI t2, ~0x7fff 351 and t1, t1, t2 352 PTR_LI t2, UNCAC_BASE 353 PTR_ADD t1, t1, t2 354 355 /* Start any other VPs that ought to be running */ 356 PTR_S ta2, CPC_CL_VC_RUN_OFS(t1) 357 358 /* Ensure this VP stops running if it shouldn't be */ 359 not ta2 360 PTR_S ta2, CPC_CL_VC_STOP_OFS(t1) 361 ehb 362 363#elif defined(CONFIG_MIPS_MT) 364 365 /* If the core doesn't support MT then return */ 366 has_mt t0, 5f 367 368 /* Enter VPE configuration state */ 369 .set push 370 .set MIPS_ISA_LEVEL_RAW 371 .set mt 372 dvpe 373 .set pop 374 375 PTR_LA t1, 1f 376 .set push 377 .set MIPS_ISA_LEVEL_RAW 378 jr.hb t1 379 nop 380 .set pop 3811: mfc0 t1, CP0_MVPCONTROL 382 ori t1, t1, MVPCONTROL_VPC 383 mtc0 t1, CP0_MVPCONTROL 384 ehb 385 386 /* Loop through each VPE */ 387 move t8, ta2 388 li ta1, 0 389 390 /* Check whether the VPE should be running. If not, skip it */ 3911: andi t0, ta2, 1 392 beqz t0, 2f 393 nop 394 395 /* Operate on the appropriate TC */ 396 mfc0 t0, CP0_VPECONTROL 397 ori t0, t0, VPECONTROL_TARGTC 398 xori t0, t0, VPECONTROL_TARGTC 399 or t0, t0, ta1 400 mtc0 t0, CP0_VPECONTROL 401 ehb 402 403 .set push 404 .set MIPS_ISA_LEVEL_RAW 405 .set mt 406 407 /* Skip the VPE if its TC is not halted */ 408 mftc0 t0, CP0_TCHALT 409 beqz t0, 2f 410 nop 411 412 /* Calculate a pointer to the VPEs struct vpe_boot_config */ 413 li t0, VPEBOOTCFG_SIZE 414 mul t0, t0, ta1 415 PTR_ADDU t0, t0, ta3 416 417 /* Set the TC restart PC */ 418 lw t1, VPEBOOTCFG_PC(t0) 419 mttc0 t1, CP0_TCRESTART 420 421 /* Set the TC stack pointer */ 422 lw t1, VPEBOOTCFG_SP(t0) 423 mttgpr t1, sp 424 425 /* Set the TC global pointer */ 426 lw t1, VPEBOOTCFG_GP(t0) 427 mttgpr t1, gp 428 429 /* Copy config from this VPE */ 430 mfc0 t0, CP0_CONFIG 431 mttc0 t0, CP0_CONFIG 432 433 /* 434 * Copy the EVA config from this VPE if the CPU supports it. 435 * CONFIG3 must exist to be running MT startup - just read it. 436 */ 437 mfc0 t0, CP0_CONFIG, 3 438 and t0, t0, MIPS_CONF3_SC 439 beqz t0, 3f 440 nop 441 mfc0 t0, CP0_SEGCTL0 442 mttc0 t0, CP0_SEGCTL0 443 mfc0 t0, CP0_SEGCTL1 444 mttc0 t0, CP0_SEGCTL1 445 mfc0 t0, CP0_SEGCTL2 446 mttc0 t0, CP0_SEGCTL2 4473: 448 /* Ensure no software interrupts are pending */ 449 mttc0 zero, CP0_CAUSE 450 mttc0 zero, CP0_STATUS 451 452 /* Set TC active, not interrupt exempt */ 453 mftc0 t0, CP0_TCSTATUS 454 li t1, ~TCSTATUS_IXMT 455 and t0, t0, t1 456 ori t0, t0, TCSTATUS_A 457 mttc0 t0, CP0_TCSTATUS 458 459 /* Clear the TC halt bit */ 460 mttc0 zero, CP0_TCHALT 461 462 /* Set VPE active */ 463 mftc0 t0, CP0_VPECONF0 464 ori t0, t0, VPECONF0_VPA 465 mttc0 t0, CP0_VPECONF0 466 467 /* Next VPE */ 4682: srl ta2, ta2, 1 469 addiu ta1, ta1, 1 470 bnez ta2, 1b 471 nop 472 473 /* Leave VPE configuration state */ 474 mfc0 t1, CP0_MVPCONTROL 475 xori t1, t1, MVPCONTROL_VPC 476 mtc0 t1, CP0_MVPCONTROL 477 ehb 478 evpe 479 480 .set pop 481 482 /* Check whether this VPE is meant to be running */ 483 li t0, 1 484 sll t0, t0, a1 485 and t0, t0, t8 486 bnez t0, 2f 487 nop 488 489 /* This VPE should be offline, halt the TC */ 490 li t0, TCHALT_H 491 mtc0 t0, CP0_TCHALT 492 PTR_LA t0, 1f 493 .set push 494 .set MIPS_ISA_LEVEL_RAW 4951: jr.hb t0 496 nop 497 .set pop 498 4992: 500 501#endif /* CONFIG_MIPS_MT_SMP */ 502 503 /* Return */ 5045: jr ra 505 nop 506 END(mips_cps_boot_vpes) 507 508#if MIPS_ISA_REV > 0 509LEAF(mips_cps_cache_init) 510 /* 511 * Clear the bits used to index the caches. Note that the architecture 512 * dictates that writing to any of TagLo or TagHi selects 0 or 2 should 513 * be valid for all MIPS32 CPUs, even those for which said writes are 514 * unnecessary. 515 */ 516 mtc0 zero, CP0_TAGLO, 0 517 mtc0 zero, CP0_TAGHI, 0 518 mtc0 zero, CP0_TAGLO, 2 519 mtc0 zero, CP0_TAGHI, 2 520 ehb 521 522 /* Primary cache configuration is indicated by Config1 */ 523 mfc0 v0, CP0_CONFIG, 1 524 525 /* Detect I-cache line size */ 526 _EXT t0, v0, MIPS_CONF1_IL_SHF, MIPS_CONF1_IL_SZ 527 beqz t0, icache_done 528 li t1, 2 529 sllv t0, t1, t0 530 531 /* Detect I-cache size */ 532 _EXT t1, v0, MIPS_CONF1_IS_SHF, MIPS_CONF1_IS_SZ 533 xori t2, t1, 0x7 534 beqz t2, 1f 535 li t3, 32 536 addiu t1, t1, 1 537 sllv t1, t3, t1 5381: /* At this point t1 == I-cache sets per way */ 539 _EXT t2, v0, MIPS_CONF1_IA_SHF, MIPS_CONF1_IA_SZ 540 addiu t2, t2, 1 541 mul t1, t1, t0 542 mul t1, t1, t2 543 544 li a0, CKSEG0 545 PTR_ADD a1, a0, t1 5461: cache Index_Store_Tag_I, 0(a0) 547 PTR_ADD a0, a0, t0 548 bne a0, a1, 1b 549 nop 550icache_done: 551 552 /* Detect D-cache line size */ 553 _EXT t0, v0, MIPS_CONF1_DL_SHF, MIPS_CONF1_DL_SZ 554 beqz t0, dcache_done 555 li t1, 2 556 sllv t0, t1, t0 557 558 /* Detect D-cache size */ 559 _EXT t1, v0, MIPS_CONF1_DS_SHF, MIPS_CONF1_DS_SZ 560 xori t2, t1, 0x7 561 beqz t2, 1f 562 li t3, 32 563 addiu t1, t1, 1 564 sllv t1, t3, t1 5651: /* At this point t1 == D-cache sets per way */ 566 _EXT t2, v0, MIPS_CONF1_DA_SHF, MIPS_CONF1_DA_SZ 567 addiu t2, t2, 1 568 mul t1, t1, t0 569 mul t1, t1, t2 570 571 li a0, CKSEG0 572 PTR_ADDU a1, a0, t1 573 PTR_SUBU a1, a1, t0 5741: cache Index_Store_Tag_D, 0(a0) 575 bne a0, a1, 1b 576 PTR_ADD a0, a0, t0 577dcache_done: 578 579 jr ra 580 nop 581 END(mips_cps_cache_init) 582#endif /* MIPS_ISA_REV > 0 */ 583 584#if defined(CONFIG_MIPS_CPS_PM) && defined(CONFIG_CPU_PM) 585 586 /* Calculate a pointer to this CPUs struct mips_static_suspend_state */ 587 .macro psstate dest 588 .set push 589 .set noat 590 lw $1, TI_CPU(gp) 591 sll $1, $1, LONGLOG 592 PTR_LA \dest, __per_cpu_offset 593 PTR_ADDU $1, $1, \dest 594 lw $1, 0($1) 595 PTR_LA \dest, cps_cpu_state 596 PTR_ADDU \dest, \dest, $1 597 .set pop 598 .endm 599 600LEAF(mips_cps_pm_save) 601 /* Save CPU state */ 602 SUSPEND_SAVE_REGS 603 psstate t1 604 SUSPEND_SAVE_STATIC 605 jr v0 606 nop 607 END(mips_cps_pm_save) 608 609LEAF(mips_cps_pm_restore) 610 /* Restore CPU state */ 611 psstate t1 612 RESUME_RESTORE_STATIC 613 RESUME_RESTORE_REGS_RETURN 614 END(mips_cps_pm_restore) 615 616#endif /* CONFIG_MIPS_CPS_PM && CONFIG_CPU_PM */ 617