1 /* 2 * spu_switch.c 3 * 4 * (C) Copyright IBM Corp. 2005 5 * 6 * Author: Mark Nutter <mnutter@us.ibm.com> 7 * 8 * Host-side part of SPU context switch sequence outlined in 9 * Synergistic Processor Element, Book IV. 10 * 11 * A fully premptive switch of an SPE is very expensive in terms 12 * of time and system resources. SPE Book IV indicates that SPE 13 * allocation should follow a "serially reusable device" model, 14 * in which the SPE is assigned a task until it completes. When 15 * this is not possible, this sequence may be used to premptively 16 * save, and then later (optionally) restore the context of a 17 * program executing on an SPE. 18 * 19 * 20 * This program is free software; you can redistribute it and/or modify 21 * it under the terms of the GNU General Public License as published by 22 * the Free Software Foundation; either version 2, or (at your option) 23 * any later version. 24 * 25 * This program is distributed in the hope that it will be useful, 26 * but WITHOUT ANY WARRANTY; without even the implied warranty of 27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 28 * GNU General Public License for more details. 29 * 30 * You should have received a copy of the GNU General Public License 31 * along with this program; if not, write to the Free Software 32 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 33 */ 34 35 #include <linux/config.h> 36 #include <linux/module.h> 37 #include <linux/errno.h> 38 #include <linux/sched.h> 39 #include <linux/kernel.h> 40 #include <linux/mm.h> 41 #include <linux/vmalloc.h> 42 #include <linux/smp.h> 43 #include <linux/smp_lock.h> 44 #include <linux/stddef.h> 45 #include <linux/unistd.h> 46 47 #include <asm/io.h> 48 #include <asm/spu.h> 49 #include <asm/spu_csa.h> 50 #include <asm/mmu_context.h> 51 52 #include "spu_save_dump.h" 53 #include "spu_restore_dump.h" 54 55 #if 0 56 #define POLL_WHILE_TRUE(_c) { \ 57 do { \ 58 } while (_c); \ 59 } 60 #else 61 #define RELAX_SPIN_COUNT 1000 62 #define POLL_WHILE_TRUE(_c) { \ 63 do { \ 64 int _i; \ 65 for (_i=0; _i<RELAX_SPIN_COUNT && (_c); _i++) { \ 66 cpu_relax(); \ 67 } \ 68 if (unlikely(_c)) yield(); \ 69 else break; \ 70 } while (_c); \ 71 } 72 #endif /* debug */ 73 74 #define POLL_WHILE_FALSE(_c) POLL_WHILE_TRUE(!(_c)) 75 76 static inline void acquire_spu_lock(struct spu *spu) 77 { 78 /* Save, Step 1: 79 * Restore, Step 1: 80 * Acquire SPU-specific mutual exclusion lock. 81 * TBD. 82 */ 83 } 84 85 static inline void release_spu_lock(struct spu *spu) 86 { 87 /* Restore, Step 76: 88 * Release SPU-specific mutual exclusion lock. 89 * TBD. 90 */ 91 } 92 93 static inline int check_spu_isolate(struct spu_state *csa, struct spu *spu) 94 { 95 struct spu_problem __iomem *prob = spu->problem; 96 u32 isolate_state; 97 98 /* Save, Step 2: 99 * Save, Step 6: 100 * If SPU_Status[E,L,IS] any field is '1', this 101 * SPU is in isolate state and cannot be context 102 * saved at this time. 103 */ 104 isolate_state = SPU_STATUS_ISOLATED_STATE | 105 SPU_STATUS_ISOLATED_LOAD_STAUTUS | SPU_STATUS_ISOLATED_EXIT_STAUTUS; 106 return (in_be32(&prob->spu_status_R) & isolate_state) ? 1 : 0; 107 } 108 109 static inline void disable_interrupts(struct spu_state *csa, struct spu *spu) 110 { 111 /* Save, Step 3: 112 * Restore, Step 2: 113 * Save INT_Mask_class0 in CSA. 114 * Write INT_MASK_class0 with value of 0. 115 * Save INT_Mask_class1 in CSA. 116 * Write INT_MASK_class1 with value of 0. 117 * Save INT_Mask_class2 in CSA. 118 * Write INT_MASK_class2 with value of 0. 119 */ 120 spin_lock_irq(&spu->register_lock); 121 if (csa) { 122 csa->priv1.int_mask_class0_RW = spu_int_mask_get(spu, 0); 123 csa->priv1.int_mask_class1_RW = spu_int_mask_get(spu, 1); 124 csa->priv1.int_mask_class2_RW = spu_int_mask_get(spu, 2); 125 } 126 spu_int_mask_set(spu, 0, 0ul); 127 spu_int_mask_set(spu, 1, 0ul); 128 spu_int_mask_set(spu, 2, 0ul); 129 eieio(); 130 spin_unlock_irq(&spu->register_lock); 131 } 132 133 static inline void set_watchdog_timer(struct spu_state *csa, struct spu *spu) 134 { 135 /* Save, Step 4: 136 * Restore, Step 25. 137 * Set a software watchdog timer, which specifies the 138 * maximum allowable time for a context save sequence. 139 * 140 * For present, this implementation will not set a global 141 * watchdog timer, as virtualization & variable system load 142 * may cause unpredictable execution times. 143 */ 144 } 145 146 static inline void inhibit_user_access(struct spu_state *csa, struct spu *spu) 147 { 148 /* Save, Step 5: 149 * Restore, Step 3: 150 * Inhibit user-space access (if provided) to this 151 * SPU by unmapping the virtual pages assigned to 152 * the SPU memory-mapped I/O (MMIO) for problem 153 * state. TBD. 154 */ 155 } 156 157 static inline void set_switch_pending(struct spu_state *csa, struct spu *spu) 158 { 159 /* Save, Step 7: 160 * Restore, Step 5: 161 * Set a software context switch pending flag. 162 */ 163 set_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags); 164 mb(); 165 } 166 167 static inline void save_mfc_cntl(struct spu_state *csa, struct spu *spu) 168 { 169 struct spu_priv2 __iomem *priv2 = spu->priv2; 170 171 /* Save, Step 8: 172 * Suspend DMA and save MFC_CNTL. 173 */ 174 switch (in_be64(&priv2->mfc_control_RW) & 175 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) { 176 case MFC_CNTL_SUSPEND_IN_PROGRESS: 177 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 178 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) == 179 MFC_CNTL_SUSPEND_COMPLETE); 180 /* fall through */ 181 case MFC_CNTL_SUSPEND_COMPLETE: 182 if (csa) { 183 csa->priv2.mfc_control_RW = 184 in_be64(&priv2->mfc_control_RW) | 185 MFC_CNTL_SUSPEND_DMA_QUEUE; 186 } 187 break; 188 case MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION: 189 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE); 190 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 191 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) == 192 MFC_CNTL_SUSPEND_COMPLETE); 193 if (csa) { 194 csa->priv2.mfc_control_RW = 195 in_be64(&priv2->mfc_control_RW) & 196 ~MFC_CNTL_SUSPEND_DMA_QUEUE; 197 } 198 break; 199 } 200 } 201 202 static inline void save_spu_runcntl(struct spu_state *csa, struct spu *spu) 203 { 204 struct spu_problem __iomem *prob = spu->problem; 205 206 /* Save, Step 9: 207 * Save SPU_Runcntl in the CSA. This value contains 208 * the "Application Desired State". 209 */ 210 csa->prob.spu_runcntl_RW = in_be32(&prob->spu_runcntl_RW); 211 } 212 213 static inline void save_mfc_sr1(struct spu_state *csa, struct spu *spu) 214 { 215 /* Save, Step 10: 216 * Save MFC_SR1 in the CSA. 217 */ 218 csa->priv1.mfc_sr1_RW = spu_mfc_sr1_get(spu); 219 } 220 221 static inline void save_spu_status(struct spu_state *csa, struct spu *spu) 222 { 223 struct spu_problem __iomem *prob = spu->problem; 224 225 /* Save, Step 11: 226 * Read SPU_Status[R], and save to CSA. 227 */ 228 if ((in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) == 0) { 229 csa->prob.spu_status_R = in_be32(&prob->spu_status_R); 230 } else { 231 u32 stopped; 232 233 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 234 eieio(); 235 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 236 SPU_STATUS_RUNNING); 237 stopped = 238 SPU_STATUS_INVALID_INSTR | SPU_STATUS_SINGLE_STEP | 239 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP; 240 if ((in_be32(&prob->spu_status_R) & stopped) == 0) 241 csa->prob.spu_status_R = SPU_STATUS_RUNNING; 242 else 243 csa->prob.spu_status_R = in_be32(&prob->spu_status_R); 244 } 245 } 246 247 static inline void save_mfc_decr(struct spu_state *csa, struct spu *spu) 248 { 249 struct spu_priv2 __iomem *priv2 = spu->priv2; 250 251 /* Save, Step 12: 252 * Read MFC_CNTL[Ds]. Update saved copy of 253 * CSA.MFC_CNTL[Ds]. 254 */ 255 if (in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DECREMENTER_RUNNING) { 256 csa->priv2.mfc_control_RW |= MFC_CNTL_DECREMENTER_RUNNING; 257 csa->suspend_time = get_cycles(); 258 out_be64(&priv2->spu_chnlcntptr_RW, 7ULL); 259 eieio(); 260 csa->spu_chnldata_RW[7] = in_be64(&priv2->spu_chnldata_RW); 261 eieio(); 262 } else { 263 csa->priv2.mfc_control_RW &= ~MFC_CNTL_DECREMENTER_RUNNING; 264 } 265 } 266 267 static inline void halt_mfc_decr(struct spu_state *csa, struct spu *spu) 268 { 269 struct spu_priv2 __iomem *priv2 = spu->priv2; 270 271 /* Save, Step 13: 272 * Write MFC_CNTL[Dh] set to a '1' to halt 273 * the decrementer. 274 */ 275 out_be64(&priv2->mfc_control_RW, MFC_CNTL_DECREMENTER_HALTED); 276 eieio(); 277 } 278 279 static inline void save_timebase(struct spu_state *csa, struct spu *spu) 280 { 281 /* Save, Step 14: 282 * Read PPE Timebase High and Timebase low registers 283 * and save in CSA. TBD. 284 */ 285 csa->suspend_time = get_cycles(); 286 } 287 288 static inline void remove_other_spu_access(struct spu_state *csa, 289 struct spu *spu) 290 { 291 /* Save, Step 15: 292 * Remove other SPU access to this SPU by unmapping 293 * this SPU's pages from their address space. TBD. 294 */ 295 } 296 297 static inline void do_mfc_mssync(struct spu_state *csa, struct spu *spu) 298 { 299 struct spu_problem __iomem *prob = spu->problem; 300 301 /* Save, Step 16: 302 * Restore, Step 11. 303 * Write SPU_MSSync register. Poll SPU_MSSync[P] 304 * for a value of 0. 305 */ 306 out_be64(&prob->spc_mssync_RW, 1UL); 307 POLL_WHILE_TRUE(in_be64(&prob->spc_mssync_RW) & MS_SYNC_PENDING); 308 } 309 310 static inline void issue_mfc_tlbie(struct spu_state *csa, struct spu *spu) 311 { 312 /* Save, Step 17: 313 * Restore, Step 12. 314 * Restore, Step 48. 315 * Write TLB_Invalidate_Entry[IS,VPN,L,Lp]=0 register. 316 * Then issue a PPE sync instruction. 317 */ 318 spu_tlb_invalidate(spu); 319 mb(); 320 } 321 322 static inline void handle_pending_interrupts(struct spu_state *csa, 323 struct spu *spu) 324 { 325 /* Save, Step 18: 326 * Handle any pending interrupts from this SPU 327 * here. This is OS or hypervisor specific. One 328 * option is to re-enable interrupts to handle any 329 * pending interrupts, with the interrupt handlers 330 * recognizing the software Context Switch Pending 331 * flag, to ensure the SPU execution or MFC command 332 * queue is not restarted. TBD. 333 */ 334 } 335 336 static inline void save_mfc_queues(struct spu_state *csa, struct spu *spu) 337 { 338 struct spu_priv2 __iomem *priv2 = spu->priv2; 339 int i; 340 341 /* Save, Step 19: 342 * If MFC_Cntl[Se]=0 then save 343 * MFC command queues. 344 */ 345 if ((in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DMA_QUEUES_EMPTY) == 0) { 346 for (i = 0; i < 8; i++) { 347 csa->priv2.puq[i].mfc_cq_data0_RW = 348 in_be64(&priv2->puq[i].mfc_cq_data0_RW); 349 csa->priv2.puq[i].mfc_cq_data1_RW = 350 in_be64(&priv2->puq[i].mfc_cq_data1_RW); 351 csa->priv2.puq[i].mfc_cq_data2_RW = 352 in_be64(&priv2->puq[i].mfc_cq_data2_RW); 353 csa->priv2.puq[i].mfc_cq_data3_RW = 354 in_be64(&priv2->puq[i].mfc_cq_data3_RW); 355 } 356 for (i = 0; i < 16; i++) { 357 csa->priv2.spuq[i].mfc_cq_data0_RW = 358 in_be64(&priv2->spuq[i].mfc_cq_data0_RW); 359 csa->priv2.spuq[i].mfc_cq_data1_RW = 360 in_be64(&priv2->spuq[i].mfc_cq_data1_RW); 361 csa->priv2.spuq[i].mfc_cq_data2_RW = 362 in_be64(&priv2->spuq[i].mfc_cq_data2_RW); 363 csa->priv2.spuq[i].mfc_cq_data3_RW = 364 in_be64(&priv2->spuq[i].mfc_cq_data3_RW); 365 } 366 } 367 } 368 369 static inline void save_ppu_querymask(struct spu_state *csa, struct spu *spu) 370 { 371 struct spu_problem __iomem *prob = spu->problem; 372 373 /* Save, Step 20: 374 * Save the PPU_QueryMask register 375 * in the CSA. 376 */ 377 csa->prob.dma_querymask_RW = in_be32(&prob->dma_querymask_RW); 378 } 379 380 static inline void save_ppu_querytype(struct spu_state *csa, struct spu *spu) 381 { 382 struct spu_problem __iomem *prob = spu->problem; 383 384 /* Save, Step 21: 385 * Save the PPU_QueryType register 386 * in the CSA. 387 */ 388 csa->prob.dma_querytype_RW = in_be32(&prob->dma_querytype_RW); 389 } 390 391 static inline void save_mfc_csr_tsq(struct spu_state *csa, struct spu *spu) 392 { 393 struct spu_priv2 __iomem *priv2 = spu->priv2; 394 395 /* Save, Step 22: 396 * Save the MFC_CSR_TSQ register 397 * in the LSCSA. 398 */ 399 csa->priv2.spu_tag_status_query_RW = 400 in_be64(&priv2->spu_tag_status_query_RW); 401 } 402 403 static inline void save_mfc_csr_cmd(struct spu_state *csa, struct spu *spu) 404 { 405 struct spu_priv2 __iomem *priv2 = spu->priv2; 406 407 /* Save, Step 23: 408 * Save the MFC_CSR_CMD1 and MFC_CSR_CMD2 409 * registers in the CSA. 410 */ 411 csa->priv2.spu_cmd_buf1_RW = in_be64(&priv2->spu_cmd_buf1_RW); 412 csa->priv2.spu_cmd_buf2_RW = in_be64(&priv2->spu_cmd_buf2_RW); 413 } 414 415 static inline void save_mfc_csr_ato(struct spu_state *csa, struct spu *spu) 416 { 417 struct spu_priv2 __iomem *priv2 = spu->priv2; 418 419 /* Save, Step 24: 420 * Save the MFC_CSR_ATO register in 421 * the CSA. 422 */ 423 csa->priv2.spu_atomic_status_RW = in_be64(&priv2->spu_atomic_status_RW); 424 } 425 426 static inline void save_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 427 { 428 /* Save, Step 25: 429 * Save the MFC_TCLASS_ID register in 430 * the CSA. 431 */ 432 csa->priv1.mfc_tclass_id_RW = spu_mfc_tclass_id_get(spu); 433 } 434 435 static inline void set_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 436 { 437 /* Save, Step 26: 438 * Restore, Step 23. 439 * Write the MFC_TCLASS_ID register with 440 * the value 0x10000000. 441 */ 442 spu_mfc_tclass_id_set(spu, 0x10000000); 443 eieio(); 444 } 445 446 static inline void purge_mfc_queue(struct spu_state *csa, struct spu *spu) 447 { 448 struct spu_priv2 __iomem *priv2 = spu->priv2; 449 450 /* Save, Step 27: 451 * Restore, Step 14. 452 * Write MFC_CNTL[Pc]=1 (purge queue). 453 */ 454 out_be64(&priv2->mfc_control_RW, MFC_CNTL_PURGE_DMA_REQUEST); 455 eieio(); 456 } 457 458 static inline void wait_purge_complete(struct spu_state *csa, struct spu *spu) 459 { 460 struct spu_priv2 __iomem *priv2 = spu->priv2; 461 462 /* Save, Step 28: 463 * Poll MFC_CNTL[Ps] until value '11' is read 464 * (purge complete). 465 */ 466 POLL_WHILE_FALSE(in_be64(&priv2->mfc_control_RW) & 467 MFC_CNTL_PURGE_DMA_COMPLETE); 468 } 469 470 static inline void save_mfc_slbs(struct spu_state *csa, struct spu *spu) 471 { 472 struct spu_priv2 __iomem *priv2 = spu->priv2; 473 int i; 474 475 /* Save, Step 29: 476 * If MFC_SR1[R]='1', save SLBs in CSA. 477 */ 478 if (spu_mfc_sr1_get(spu) & MFC_STATE1_RELOCATE_MASK) { 479 csa->priv2.slb_index_W = in_be64(&priv2->slb_index_W); 480 for (i = 0; i < 8; i++) { 481 out_be64(&priv2->slb_index_W, i); 482 eieio(); 483 csa->slb_esid_RW[i] = in_be64(&priv2->slb_esid_RW); 484 csa->slb_vsid_RW[i] = in_be64(&priv2->slb_vsid_RW); 485 eieio(); 486 } 487 } 488 } 489 490 static inline void setup_mfc_sr1(struct spu_state *csa, struct spu *spu) 491 { 492 /* Save, Step 30: 493 * Restore, Step 18: 494 * Write MFC_SR1 with MFC_SR1[D=0,S=1] and 495 * MFC_SR1[TL,R,Pr,T] set correctly for the 496 * OS specific environment. 497 * 498 * Implementation note: The SPU-side code 499 * for save/restore is privileged, so the 500 * MFC_SR1[Pr] bit is not set. 501 * 502 */ 503 spu_mfc_sr1_set(spu, (MFC_STATE1_MASTER_RUN_CONTROL_MASK | 504 MFC_STATE1_RELOCATE_MASK | 505 MFC_STATE1_BUS_TLBIE_MASK)); 506 } 507 508 static inline void save_spu_npc(struct spu_state *csa, struct spu *spu) 509 { 510 struct spu_problem __iomem *prob = spu->problem; 511 512 /* Save, Step 31: 513 * Save SPU_NPC in the CSA. 514 */ 515 csa->prob.spu_npc_RW = in_be32(&prob->spu_npc_RW); 516 } 517 518 static inline void save_spu_privcntl(struct spu_state *csa, struct spu *spu) 519 { 520 struct spu_priv2 __iomem *priv2 = spu->priv2; 521 522 /* Save, Step 32: 523 * Save SPU_PrivCntl in the CSA. 524 */ 525 csa->priv2.spu_privcntl_RW = in_be64(&priv2->spu_privcntl_RW); 526 } 527 528 static inline void reset_spu_privcntl(struct spu_state *csa, struct spu *spu) 529 { 530 struct spu_priv2 __iomem *priv2 = spu->priv2; 531 532 /* Save, Step 33: 533 * Restore, Step 16: 534 * Write SPU_PrivCntl[S,Le,A] fields reset to 0. 535 */ 536 out_be64(&priv2->spu_privcntl_RW, 0UL); 537 eieio(); 538 } 539 540 static inline void save_spu_lslr(struct spu_state *csa, struct spu *spu) 541 { 542 struct spu_priv2 __iomem *priv2 = spu->priv2; 543 544 /* Save, Step 34: 545 * Save SPU_LSLR in the CSA. 546 */ 547 csa->priv2.spu_lslr_RW = in_be64(&priv2->spu_lslr_RW); 548 } 549 550 static inline void reset_spu_lslr(struct spu_state *csa, struct spu *spu) 551 { 552 struct spu_priv2 __iomem *priv2 = spu->priv2; 553 554 /* Save, Step 35: 555 * Restore, Step 17. 556 * Reset SPU_LSLR. 557 */ 558 out_be64(&priv2->spu_lslr_RW, LS_ADDR_MASK); 559 eieio(); 560 } 561 562 static inline void save_spu_cfg(struct spu_state *csa, struct spu *spu) 563 { 564 struct spu_priv2 __iomem *priv2 = spu->priv2; 565 566 /* Save, Step 36: 567 * Save SPU_Cfg in the CSA. 568 */ 569 csa->priv2.spu_cfg_RW = in_be64(&priv2->spu_cfg_RW); 570 } 571 572 static inline void save_pm_trace(struct spu_state *csa, struct spu *spu) 573 { 574 /* Save, Step 37: 575 * Save PM_Trace_Tag_Wait_Mask in the CSA. 576 * Not performed by this implementation. 577 */ 578 } 579 580 static inline void save_mfc_rag(struct spu_state *csa, struct spu *spu) 581 { 582 /* Save, Step 38: 583 * Save RA_GROUP_ID register and the 584 * RA_ENABLE reigster in the CSA. 585 */ 586 csa->priv1.resource_allocation_groupID_RW = 587 spu_resource_allocation_groupID_get(spu); 588 csa->priv1.resource_allocation_enable_RW = 589 spu_resource_allocation_enable_get(spu); 590 } 591 592 static inline void save_ppu_mb_stat(struct spu_state *csa, struct spu *spu) 593 { 594 struct spu_problem __iomem *prob = spu->problem; 595 596 /* Save, Step 39: 597 * Save MB_Stat register in the CSA. 598 */ 599 csa->prob.mb_stat_R = in_be32(&prob->mb_stat_R); 600 } 601 602 static inline void save_ppu_mb(struct spu_state *csa, struct spu *spu) 603 { 604 struct spu_problem __iomem *prob = spu->problem; 605 606 /* Save, Step 40: 607 * Save the PPU_MB register in the CSA. 608 */ 609 csa->prob.pu_mb_R = in_be32(&prob->pu_mb_R); 610 } 611 612 static inline void save_ppuint_mb(struct spu_state *csa, struct spu *spu) 613 { 614 struct spu_priv2 __iomem *priv2 = spu->priv2; 615 616 /* Save, Step 41: 617 * Save the PPUINT_MB register in the CSA. 618 */ 619 csa->priv2.puint_mb_R = in_be64(&priv2->puint_mb_R); 620 } 621 622 static inline void save_ch_part1(struct spu_state *csa, struct spu *spu) 623 { 624 struct spu_priv2 __iomem *priv2 = spu->priv2; 625 u64 idx, ch_indices[7] = { 0UL, 1UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 626 int i; 627 628 /* Save, Step 42: 629 * Save the following CH: [0,1,3,4,24,25,27] 630 */ 631 for (i = 0; i < 7; i++) { 632 idx = ch_indices[i]; 633 out_be64(&priv2->spu_chnlcntptr_RW, idx); 634 eieio(); 635 csa->spu_chnldata_RW[idx] = in_be64(&priv2->spu_chnldata_RW); 636 csa->spu_chnlcnt_RW[idx] = in_be64(&priv2->spu_chnlcnt_RW); 637 out_be64(&priv2->spu_chnldata_RW, 0UL); 638 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 639 eieio(); 640 } 641 } 642 643 static inline void save_spu_mb(struct spu_state *csa, struct spu *spu) 644 { 645 struct spu_priv2 __iomem *priv2 = spu->priv2; 646 int i; 647 648 /* Save, Step 43: 649 * Save SPU Read Mailbox Channel. 650 */ 651 out_be64(&priv2->spu_chnlcntptr_RW, 29UL); 652 eieio(); 653 csa->spu_chnlcnt_RW[29] = in_be64(&priv2->spu_chnlcnt_RW); 654 for (i = 0; i < 4; i++) { 655 csa->spu_mailbox_data[i] = in_be64(&priv2->spu_chnldata_RW); 656 } 657 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 658 eieio(); 659 } 660 661 static inline void save_mfc_cmd(struct spu_state *csa, struct spu *spu) 662 { 663 struct spu_priv2 __iomem *priv2 = spu->priv2; 664 665 /* Save, Step 44: 666 * Save MFC_CMD Channel. 667 */ 668 out_be64(&priv2->spu_chnlcntptr_RW, 21UL); 669 eieio(); 670 csa->spu_chnlcnt_RW[21] = in_be64(&priv2->spu_chnlcnt_RW); 671 eieio(); 672 } 673 674 static inline void reset_ch(struct spu_state *csa, struct spu *spu) 675 { 676 struct spu_priv2 __iomem *priv2 = spu->priv2; 677 u64 ch_indices[4] = { 21UL, 23UL, 28UL, 30UL }; 678 u64 ch_counts[4] = { 16UL, 1UL, 1UL, 1UL }; 679 u64 idx; 680 int i; 681 682 /* Save, Step 45: 683 * Reset the following CH: [21, 23, 28, 30] 684 */ 685 for (i = 0; i < 4; i++) { 686 idx = ch_indices[i]; 687 out_be64(&priv2->spu_chnlcntptr_RW, idx); 688 eieio(); 689 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 690 eieio(); 691 } 692 } 693 694 static inline void resume_mfc_queue(struct spu_state *csa, struct spu *spu) 695 { 696 struct spu_priv2 __iomem *priv2 = spu->priv2; 697 698 /* Save, Step 46: 699 * Restore, Step 25. 700 * Write MFC_CNTL[Sc]=0 (resume queue processing). 701 */ 702 out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESUME_DMA_QUEUE); 703 } 704 705 static inline void invalidate_slbs(struct spu_state *csa, struct spu *spu) 706 { 707 struct spu_priv2 __iomem *priv2 = spu->priv2; 708 709 /* Save, Step 45: 710 * Restore, Step 19: 711 * If MFC_SR1[R]=1, write 0 to SLB_Invalidate_All. 712 */ 713 if (spu_mfc_sr1_get(spu) & MFC_STATE1_RELOCATE_MASK) { 714 out_be64(&priv2->slb_invalidate_all_W, 0UL); 715 eieio(); 716 } 717 } 718 719 static inline void get_kernel_slb(u64 ea, u64 slb[2]) 720 { 721 slb[0] = (get_kernel_vsid(ea) << SLB_VSID_SHIFT) | SLB_VSID_KERNEL; 722 slb[1] = (ea & ESID_MASK) | SLB_ESID_V; 723 724 /* Large pages are used for kernel text/data, but not vmalloc. */ 725 if (cpu_has_feature(CPU_FTR_16M_PAGE) 726 && REGION_ID(ea) == KERNEL_REGION_ID) 727 slb[0] |= SLB_VSID_L; 728 } 729 730 static inline void load_mfc_slb(struct spu *spu, u64 slb[2], int slbe) 731 { 732 struct spu_priv2 __iomem *priv2 = spu->priv2; 733 734 out_be64(&priv2->slb_index_W, slbe); 735 eieio(); 736 out_be64(&priv2->slb_vsid_RW, slb[0]); 737 out_be64(&priv2->slb_esid_RW, slb[1]); 738 eieio(); 739 } 740 741 static inline void setup_mfc_slbs(struct spu_state *csa, struct spu *spu) 742 { 743 u64 code_slb[2]; 744 u64 lscsa_slb[2]; 745 746 /* Save, Step 47: 747 * Restore, Step 30. 748 * If MFC_SR1[R]=1, write 0 to SLB_Invalidate_All 749 * register, then initialize SLB_VSID and SLB_ESID 750 * to provide access to SPU context save code and 751 * LSCSA. 752 * 753 * This implementation places both the context 754 * switch code and LSCSA in kernel address space. 755 * 756 * Further this implementation assumes that the 757 * MFC_SR1[R]=1 (in other words, assume that 758 * translation is desired by OS environment). 759 */ 760 invalidate_slbs(csa, spu); 761 get_kernel_slb((unsigned long)&spu_save_code[0], code_slb); 762 get_kernel_slb((unsigned long)csa->lscsa, lscsa_slb); 763 load_mfc_slb(spu, code_slb, 0); 764 if ((lscsa_slb[0] != code_slb[0]) || (lscsa_slb[1] != code_slb[1])) 765 load_mfc_slb(spu, lscsa_slb, 1); 766 } 767 768 static inline void set_switch_active(struct spu_state *csa, struct spu *spu) 769 { 770 /* Save, Step 48: 771 * Restore, Step 23. 772 * Change the software context switch pending flag 773 * to context switch active. 774 */ 775 set_bit(SPU_CONTEXT_SWITCH_ACTIVE, &spu->flags); 776 clear_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags); 777 mb(); 778 } 779 780 static inline void enable_interrupts(struct spu_state *csa, struct spu *spu) 781 { 782 unsigned long class1_mask = CLASS1_ENABLE_SEGMENT_FAULT_INTR | 783 CLASS1_ENABLE_STORAGE_FAULT_INTR; 784 785 /* Save, Step 49: 786 * Restore, Step 22: 787 * Reset and then enable interrupts, as 788 * needed by OS. 789 * 790 * This implementation enables only class1 791 * (translation) interrupts. 792 */ 793 spin_lock_irq(&spu->register_lock); 794 spu_int_stat_clear(spu, 0, ~0ul); 795 spu_int_stat_clear(spu, 1, ~0ul); 796 spu_int_stat_clear(spu, 2, ~0ul); 797 spu_int_mask_set(spu, 0, 0ul); 798 spu_int_mask_set(spu, 1, class1_mask); 799 spu_int_mask_set(spu, 2, 0ul); 800 spin_unlock_irq(&spu->register_lock); 801 } 802 803 static inline int send_mfc_dma(struct spu *spu, unsigned long ea, 804 unsigned int ls_offset, unsigned int size, 805 unsigned int tag, unsigned int rclass, 806 unsigned int cmd) 807 { 808 struct spu_problem __iomem *prob = spu->problem; 809 union mfc_tag_size_class_cmd command; 810 unsigned int transfer_size; 811 volatile unsigned int status = 0x0; 812 813 while (size > 0) { 814 transfer_size = 815 (size > MFC_MAX_DMA_SIZE) ? MFC_MAX_DMA_SIZE : size; 816 command.u.mfc_size = transfer_size; 817 command.u.mfc_tag = tag; 818 command.u.mfc_rclassid = rclass; 819 command.u.mfc_cmd = cmd; 820 do { 821 out_be32(&prob->mfc_lsa_W, ls_offset); 822 out_be64(&prob->mfc_ea_W, ea); 823 out_be64(&prob->mfc_union_W.all64, command.all64); 824 status = 825 in_be32(&prob->mfc_union_W.by32.mfc_class_cmd32); 826 if (unlikely(status & 0x2)) { 827 cpu_relax(); 828 } 829 } while (status & 0x3); 830 size -= transfer_size; 831 ea += transfer_size; 832 ls_offset += transfer_size; 833 } 834 return 0; 835 } 836 837 static inline void save_ls_16kb(struct spu_state *csa, struct spu *spu) 838 { 839 unsigned long addr = (unsigned long)&csa->lscsa->ls[0]; 840 unsigned int ls_offset = 0x0; 841 unsigned int size = 16384; 842 unsigned int tag = 0; 843 unsigned int rclass = 0; 844 unsigned int cmd = MFC_PUT_CMD; 845 846 /* Save, Step 50: 847 * Issue a DMA command to copy the first 16K bytes 848 * of local storage to the CSA. 849 */ 850 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 851 } 852 853 static inline void set_spu_npc(struct spu_state *csa, struct spu *spu) 854 { 855 struct spu_problem __iomem *prob = spu->problem; 856 857 /* Save, Step 51: 858 * Restore, Step 31. 859 * Write SPU_NPC[IE]=0 and SPU_NPC[LSA] to entry 860 * point address of context save code in local 861 * storage. 862 * 863 * This implementation uses SPU-side save/restore 864 * programs with entry points at LSA of 0. 865 */ 866 out_be32(&prob->spu_npc_RW, 0); 867 eieio(); 868 } 869 870 static inline void set_signot1(struct spu_state *csa, struct spu *spu) 871 { 872 struct spu_problem __iomem *prob = spu->problem; 873 union { 874 u64 ull; 875 u32 ui[2]; 876 } addr64; 877 878 /* Save, Step 52: 879 * Restore, Step 32: 880 * Write SPU_Sig_Notify_1 register with upper 32-bits 881 * of the CSA.LSCSA effective address. 882 */ 883 addr64.ull = (u64) csa->lscsa; 884 out_be32(&prob->signal_notify1, addr64.ui[0]); 885 eieio(); 886 } 887 888 static inline void set_signot2(struct spu_state *csa, struct spu *spu) 889 { 890 struct spu_problem __iomem *prob = spu->problem; 891 union { 892 u64 ull; 893 u32 ui[2]; 894 } addr64; 895 896 /* Save, Step 53: 897 * Restore, Step 33: 898 * Write SPU_Sig_Notify_2 register with lower 32-bits 899 * of the CSA.LSCSA effective address. 900 */ 901 addr64.ull = (u64) csa->lscsa; 902 out_be32(&prob->signal_notify2, addr64.ui[1]); 903 eieio(); 904 } 905 906 static inline void send_save_code(struct spu_state *csa, struct spu *spu) 907 { 908 unsigned long addr = (unsigned long)&spu_save_code[0]; 909 unsigned int ls_offset = 0x0; 910 unsigned int size = sizeof(spu_save_code); 911 unsigned int tag = 0; 912 unsigned int rclass = 0; 913 unsigned int cmd = MFC_GETFS_CMD; 914 915 /* Save, Step 54: 916 * Issue a DMA command to copy context save code 917 * to local storage and start SPU. 918 */ 919 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 920 } 921 922 static inline void set_ppu_querymask(struct spu_state *csa, struct spu *spu) 923 { 924 struct spu_problem __iomem *prob = spu->problem; 925 926 /* Save, Step 55: 927 * Restore, Step 38. 928 * Write PPU_QueryMask=1 (enable Tag Group 0) 929 * and issue eieio instruction. 930 */ 931 out_be32(&prob->dma_querymask_RW, MFC_TAGID_TO_TAGMASK(0)); 932 eieio(); 933 } 934 935 static inline void wait_tag_complete(struct spu_state *csa, struct spu *spu) 936 { 937 struct spu_problem __iomem *prob = spu->problem; 938 u32 mask = MFC_TAGID_TO_TAGMASK(0); 939 unsigned long flags; 940 941 /* Save, Step 56: 942 * Restore, Step 39. 943 * Restore, Step 39. 944 * Restore, Step 46. 945 * Poll PPU_TagStatus[gn] until 01 (Tag group 0 complete) 946 * or write PPU_QueryType[TS]=01 and wait for Tag Group 947 * Complete Interrupt. Write INT_Stat_Class0 or 948 * INT_Stat_Class2 with value of 'handled'. 949 */ 950 POLL_WHILE_FALSE(in_be32(&prob->dma_tagstatus_R) & mask); 951 952 local_irq_save(flags); 953 spu_int_stat_clear(spu, 0, ~(0ul)); 954 spu_int_stat_clear(spu, 2, ~(0ul)); 955 local_irq_restore(flags); 956 } 957 958 static inline void wait_spu_stopped(struct spu_state *csa, struct spu *spu) 959 { 960 struct spu_problem __iomem *prob = spu->problem; 961 unsigned long flags; 962 963 /* Save, Step 57: 964 * Restore, Step 40. 965 * Poll until SPU_Status[R]=0 or wait for SPU Class 0 966 * or SPU Class 2 interrupt. Write INT_Stat_class0 967 * or INT_Stat_class2 with value of handled. 968 */ 969 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING); 970 971 local_irq_save(flags); 972 spu_int_stat_clear(spu, 0, ~(0ul)); 973 spu_int_stat_clear(spu, 2, ~(0ul)); 974 local_irq_restore(flags); 975 } 976 977 static inline int check_save_status(struct spu_state *csa, struct spu *spu) 978 { 979 struct spu_problem __iomem *prob = spu->problem; 980 u32 complete; 981 982 /* Save, Step 54: 983 * If SPU_Status[P]=1 and SPU_Status[SC] = "success", 984 * context save succeeded, otherwise context save 985 * failed. 986 */ 987 complete = ((SPU_SAVE_COMPLETE << SPU_STOP_STATUS_SHIFT) | 988 SPU_STATUS_STOPPED_BY_STOP); 989 return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0; 990 } 991 992 static inline void terminate_spu_app(struct spu_state *csa, struct spu *spu) 993 { 994 /* Restore, Step 4: 995 * If required, notify the "using application" that 996 * the SPU task has been terminated. TBD. 997 */ 998 } 999 1000 static inline void suspend_mfc(struct spu_state *csa, struct spu *spu) 1001 { 1002 struct spu_priv2 __iomem *priv2 = spu->priv2; 1003 1004 /* Restore, Step 7: 1005 * Restore, Step 47. 1006 * Write MFC_Cntl[Dh,Sc]='1','1' to suspend 1007 * the queue and halt the decrementer. 1008 */ 1009 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE | 1010 MFC_CNTL_DECREMENTER_HALTED); 1011 eieio(); 1012 } 1013 1014 static inline void wait_suspend_mfc_complete(struct spu_state *csa, 1015 struct spu *spu) 1016 { 1017 struct spu_priv2 __iomem *priv2 = spu->priv2; 1018 1019 /* Restore, Step 8: 1020 * Restore, Step 47. 1021 * Poll MFC_CNTL[Ss] until 11 is returned. 1022 */ 1023 POLL_WHILE_FALSE(in_be64(&priv2->mfc_control_RW) & 1024 MFC_CNTL_SUSPEND_COMPLETE); 1025 } 1026 1027 static inline int suspend_spe(struct spu_state *csa, struct spu *spu) 1028 { 1029 struct spu_problem __iomem *prob = spu->problem; 1030 1031 /* Restore, Step 9: 1032 * If SPU_Status[R]=1, stop SPU execution 1033 * and wait for stop to complete. 1034 * 1035 * Returns 1 if SPU_Status[R]=1 on entry. 1036 * 0 otherwise 1037 */ 1038 if (in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) { 1039 if (in_be32(&prob->spu_status_R) & 1040 SPU_STATUS_ISOLATED_EXIT_STAUTUS) { 1041 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1042 SPU_STATUS_RUNNING); 1043 } 1044 if ((in_be32(&prob->spu_status_R) & 1045 SPU_STATUS_ISOLATED_LOAD_STAUTUS) 1046 || (in_be32(&prob->spu_status_R) & 1047 SPU_STATUS_ISOLATED_STATE)) { 1048 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1049 eieio(); 1050 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1051 SPU_STATUS_RUNNING); 1052 out_be32(&prob->spu_runcntl_RW, 0x2); 1053 eieio(); 1054 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1055 SPU_STATUS_RUNNING); 1056 } 1057 if (in_be32(&prob->spu_status_R) & 1058 SPU_STATUS_WAITING_FOR_CHANNEL) { 1059 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1060 eieio(); 1061 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1062 SPU_STATUS_RUNNING); 1063 } 1064 return 1; 1065 } 1066 return 0; 1067 } 1068 1069 static inline void clear_spu_status(struct spu_state *csa, struct spu *spu) 1070 { 1071 struct spu_problem __iomem *prob = spu->problem; 1072 1073 /* Restore, Step 10: 1074 * If SPU_Status[R]=0 and SPU_Status[E,L,IS]=1, 1075 * release SPU from isolate state. 1076 */ 1077 if (!(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING)) { 1078 if (in_be32(&prob->spu_status_R) & 1079 SPU_STATUS_ISOLATED_EXIT_STAUTUS) { 1080 spu_mfc_sr1_set(spu, 1081 MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1082 eieio(); 1083 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1084 eieio(); 1085 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1086 SPU_STATUS_RUNNING); 1087 } 1088 if ((in_be32(&prob->spu_status_R) & 1089 SPU_STATUS_ISOLATED_LOAD_STAUTUS) 1090 || (in_be32(&prob->spu_status_R) & 1091 SPU_STATUS_ISOLATED_STATE)) { 1092 spu_mfc_sr1_set(spu, 1093 MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1094 eieio(); 1095 out_be32(&prob->spu_runcntl_RW, 0x2); 1096 eieio(); 1097 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1098 SPU_STATUS_RUNNING); 1099 } 1100 } 1101 } 1102 1103 static inline void reset_ch_part1(struct spu_state *csa, struct spu *spu) 1104 { 1105 struct spu_priv2 __iomem *priv2 = spu->priv2; 1106 u64 ch_indices[7] = { 0UL, 1UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 1107 u64 idx; 1108 int i; 1109 1110 /* Restore, Step 20: 1111 * Reset the following CH: [0,1,3,4,24,25,27] 1112 */ 1113 for (i = 0; i < 7; i++) { 1114 idx = ch_indices[i]; 1115 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1116 eieio(); 1117 out_be64(&priv2->spu_chnldata_RW, 0UL); 1118 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 1119 eieio(); 1120 } 1121 } 1122 1123 static inline void reset_ch_part2(struct spu_state *csa, struct spu *spu) 1124 { 1125 struct spu_priv2 __iomem *priv2 = spu->priv2; 1126 u64 ch_indices[5] = { 21UL, 23UL, 28UL, 29UL, 30UL }; 1127 u64 ch_counts[5] = { 16UL, 1UL, 1UL, 0UL, 1UL }; 1128 u64 idx; 1129 int i; 1130 1131 /* Restore, Step 21: 1132 * Reset the following CH: [21, 23, 28, 29, 30] 1133 */ 1134 for (i = 0; i < 5; i++) { 1135 idx = ch_indices[i]; 1136 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1137 eieio(); 1138 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 1139 eieio(); 1140 } 1141 } 1142 1143 static inline void setup_spu_status_part1(struct spu_state *csa, 1144 struct spu *spu) 1145 { 1146 u32 status_P = SPU_STATUS_STOPPED_BY_STOP; 1147 u32 status_I = SPU_STATUS_INVALID_INSTR; 1148 u32 status_H = SPU_STATUS_STOPPED_BY_HALT; 1149 u32 status_S = SPU_STATUS_SINGLE_STEP; 1150 u32 status_S_I = SPU_STATUS_SINGLE_STEP | SPU_STATUS_INVALID_INSTR; 1151 u32 status_S_P = SPU_STATUS_SINGLE_STEP | SPU_STATUS_STOPPED_BY_STOP; 1152 u32 status_P_H = SPU_STATUS_STOPPED_BY_HALT |SPU_STATUS_STOPPED_BY_STOP; 1153 u32 status_P_I = SPU_STATUS_STOPPED_BY_STOP |SPU_STATUS_INVALID_INSTR; 1154 u32 status_code; 1155 1156 /* Restore, Step 27: 1157 * If the CSA.SPU_Status[I,S,H,P]=1 then add the correct 1158 * instruction sequence to the end of the SPU based restore 1159 * code (after the "context restored" stop and signal) to 1160 * restore the correct SPU status. 1161 * 1162 * NOTE: Rather than modifying the SPU executable, we 1163 * instead add a new 'stopped_status' field to the 1164 * LSCSA. The SPU-side restore reads this field and 1165 * takes the appropriate action when exiting. 1166 */ 1167 1168 status_code = 1169 (csa->prob.spu_status_R >> SPU_STOP_STATUS_SHIFT) & 0xFFFF; 1170 if ((csa->prob.spu_status_R & status_P_I) == status_P_I) { 1171 1172 /* SPU_Status[P,I]=1 - Illegal Instruction followed 1173 * by Stop and Signal instruction, followed by 'br -4'. 1174 * 1175 */ 1176 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_I; 1177 csa->lscsa->stopped_status.slot[1] = status_code; 1178 1179 } else if ((csa->prob.spu_status_R & status_P_H) == status_P_H) { 1180 1181 /* SPU_Status[P,H]=1 - Halt Conditional, followed 1182 * by Stop and Signal instruction, followed by 1183 * 'br -4'. 1184 */ 1185 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_H; 1186 csa->lscsa->stopped_status.slot[1] = status_code; 1187 1188 } else if ((csa->prob.spu_status_R & status_S_P) == status_S_P) { 1189 1190 /* SPU_Status[S,P]=1 - Stop and Signal instruction 1191 * followed by 'br -4'. 1192 */ 1193 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_P; 1194 csa->lscsa->stopped_status.slot[1] = status_code; 1195 1196 } else if ((csa->prob.spu_status_R & status_S_I) == status_S_I) { 1197 1198 /* SPU_Status[S,I]=1 - Illegal instruction followed 1199 * by 'br -4'. 1200 */ 1201 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_I; 1202 csa->lscsa->stopped_status.slot[1] = status_code; 1203 1204 } else if ((csa->prob.spu_status_R & status_P) == status_P) { 1205 1206 /* SPU_Status[P]=1 - Stop and Signal instruction 1207 * followed by 'br -4'. 1208 */ 1209 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P; 1210 csa->lscsa->stopped_status.slot[1] = status_code; 1211 1212 } else if ((csa->prob.spu_status_R & status_H) == status_H) { 1213 1214 /* SPU_Status[H]=1 - Halt Conditional, followed 1215 * by 'br -4'. 1216 */ 1217 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_H; 1218 1219 } else if ((csa->prob.spu_status_R & status_S) == status_S) { 1220 1221 /* SPU_Status[S]=1 - Two nop instructions. 1222 */ 1223 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S; 1224 1225 } else if ((csa->prob.spu_status_R & status_I) == status_I) { 1226 1227 /* SPU_Status[I]=1 - Illegal instruction followed 1228 * by 'br -4'. 1229 */ 1230 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_I; 1231 1232 } 1233 } 1234 1235 static inline void setup_spu_status_part2(struct spu_state *csa, 1236 struct spu *spu) 1237 { 1238 u32 mask; 1239 1240 /* Restore, Step 28: 1241 * If the CSA.SPU_Status[I,S,H,P,R]=0 then 1242 * add a 'br *' instruction to the end of 1243 * the SPU based restore code. 1244 * 1245 * NOTE: Rather than modifying the SPU executable, we 1246 * instead add a new 'stopped_status' field to the 1247 * LSCSA. The SPU-side restore reads this field and 1248 * takes the appropriate action when exiting. 1249 */ 1250 mask = SPU_STATUS_INVALID_INSTR | 1251 SPU_STATUS_SINGLE_STEP | 1252 SPU_STATUS_STOPPED_BY_HALT | 1253 SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING; 1254 if (!(csa->prob.spu_status_R & mask)) { 1255 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_R; 1256 } 1257 } 1258 1259 static inline void restore_mfc_rag(struct spu_state *csa, struct spu *spu) 1260 { 1261 /* Restore, Step 29: 1262 * Restore RA_GROUP_ID register and the 1263 * RA_ENABLE reigster from the CSA. 1264 */ 1265 spu_resource_allocation_groupID_set(spu, 1266 csa->priv1.resource_allocation_groupID_RW); 1267 spu_resource_allocation_enable_set(spu, 1268 csa->priv1.resource_allocation_enable_RW); 1269 } 1270 1271 static inline void send_restore_code(struct spu_state *csa, struct spu *spu) 1272 { 1273 unsigned long addr = (unsigned long)&spu_restore_code[0]; 1274 unsigned int ls_offset = 0x0; 1275 unsigned int size = sizeof(spu_restore_code); 1276 unsigned int tag = 0; 1277 unsigned int rclass = 0; 1278 unsigned int cmd = MFC_GETFS_CMD; 1279 1280 /* Restore, Step 37: 1281 * Issue MFC DMA command to copy context 1282 * restore code to local storage. 1283 */ 1284 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 1285 } 1286 1287 static inline void setup_decr(struct spu_state *csa, struct spu *spu) 1288 { 1289 /* Restore, Step 34: 1290 * If CSA.MFC_CNTL[Ds]=1 (decrementer was 1291 * running) then adjust decrementer, set 1292 * decrementer running status in LSCSA, 1293 * and set decrementer "wrapped" status 1294 * in LSCSA. 1295 */ 1296 if (csa->priv2.mfc_control_RW & MFC_CNTL_DECREMENTER_RUNNING) { 1297 cycles_t resume_time = get_cycles(); 1298 cycles_t delta_time = resume_time - csa->suspend_time; 1299 1300 csa->lscsa->decr.slot[0] -= delta_time; 1301 } 1302 } 1303 1304 static inline void setup_ppu_mb(struct spu_state *csa, struct spu *spu) 1305 { 1306 /* Restore, Step 35: 1307 * Copy the CSA.PU_MB data into the LSCSA. 1308 */ 1309 csa->lscsa->ppu_mb.slot[0] = csa->prob.pu_mb_R; 1310 } 1311 1312 static inline void setup_ppuint_mb(struct spu_state *csa, struct spu *spu) 1313 { 1314 /* Restore, Step 36: 1315 * Copy the CSA.PUINT_MB data into the LSCSA. 1316 */ 1317 csa->lscsa->ppuint_mb.slot[0] = csa->priv2.puint_mb_R; 1318 } 1319 1320 static inline int check_restore_status(struct spu_state *csa, struct spu *spu) 1321 { 1322 struct spu_problem __iomem *prob = spu->problem; 1323 u32 complete; 1324 1325 /* Restore, Step 40: 1326 * If SPU_Status[P]=1 and SPU_Status[SC] = "success", 1327 * context restore succeeded, otherwise context restore 1328 * failed. 1329 */ 1330 complete = ((SPU_RESTORE_COMPLETE << SPU_STOP_STATUS_SHIFT) | 1331 SPU_STATUS_STOPPED_BY_STOP); 1332 return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0; 1333 } 1334 1335 static inline void restore_spu_privcntl(struct spu_state *csa, struct spu *spu) 1336 { 1337 struct spu_priv2 __iomem *priv2 = spu->priv2; 1338 1339 /* Restore, Step 41: 1340 * Restore SPU_PrivCntl from the CSA. 1341 */ 1342 out_be64(&priv2->spu_privcntl_RW, csa->priv2.spu_privcntl_RW); 1343 eieio(); 1344 } 1345 1346 static inline void restore_status_part1(struct spu_state *csa, struct spu *spu) 1347 { 1348 struct spu_problem __iomem *prob = spu->problem; 1349 u32 mask; 1350 1351 /* Restore, Step 42: 1352 * If any CSA.SPU_Status[I,S,H,P]=1, then 1353 * restore the error or single step state. 1354 */ 1355 mask = SPU_STATUS_INVALID_INSTR | 1356 SPU_STATUS_SINGLE_STEP | 1357 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP; 1358 if (csa->prob.spu_status_R & mask) { 1359 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1360 eieio(); 1361 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1362 SPU_STATUS_RUNNING); 1363 } 1364 } 1365 1366 static inline void restore_status_part2(struct spu_state *csa, struct spu *spu) 1367 { 1368 struct spu_problem __iomem *prob = spu->problem; 1369 u32 mask; 1370 1371 /* Restore, Step 43: 1372 * If all CSA.SPU_Status[I,S,H,P,R]=0 then write 1373 * SPU_RunCntl[R0R1]='01', wait for SPU_Status[R]=1, 1374 * then write '00' to SPU_RunCntl[R0R1] and wait 1375 * for SPU_Status[R]=0. 1376 */ 1377 mask = SPU_STATUS_INVALID_INSTR | 1378 SPU_STATUS_SINGLE_STEP | 1379 SPU_STATUS_STOPPED_BY_HALT | 1380 SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING; 1381 if (!(csa->prob.spu_status_R & mask)) { 1382 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1383 eieio(); 1384 POLL_WHILE_FALSE(in_be32(&prob->spu_status_R) & 1385 SPU_STATUS_RUNNING); 1386 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1387 eieio(); 1388 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1389 SPU_STATUS_RUNNING); 1390 } 1391 } 1392 1393 static inline void restore_ls_16kb(struct spu_state *csa, struct spu *spu) 1394 { 1395 unsigned long addr = (unsigned long)&csa->lscsa->ls[0]; 1396 unsigned int ls_offset = 0x0; 1397 unsigned int size = 16384; 1398 unsigned int tag = 0; 1399 unsigned int rclass = 0; 1400 unsigned int cmd = MFC_GET_CMD; 1401 1402 /* Restore, Step 44: 1403 * Issue a DMA command to restore the first 1404 * 16kb of local storage from CSA. 1405 */ 1406 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 1407 } 1408 1409 static inline void clear_interrupts(struct spu_state *csa, struct spu *spu) 1410 { 1411 /* Restore, Step 49: 1412 * Write INT_MASK_class0 with value of 0. 1413 * Write INT_MASK_class1 with value of 0. 1414 * Write INT_MASK_class2 with value of 0. 1415 * Write INT_STAT_class0 with value of -1. 1416 * Write INT_STAT_class1 with value of -1. 1417 * Write INT_STAT_class2 with value of -1. 1418 */ 1419 spin_lock_irq(&spu->register_lock); 1420 spu_int_mask_set(spu, 0, 0ul); 1421 spu_int_mask_set(spu, 1, 0ul); 1422 spu_int_mask_set(spu, 2, 0ul); 1423 spu_int_stat_clear(spu, 0, ~0ul); 1424 spu_int_stat_clear(spu, 1, ~0ul); 1425 spu_int_stat_clear(spu, 2, ~0ul); 1426 spin_unlock_irq(&spu->register_lock); 1427 } 1428 1429 static inline void restore_mfc_queues(struct spu_state *csa, struct spu *spu) 1430 { 1431 struct spu_priv2 __iomem *priv2 = spu->priv2; 1432 int i; 1433 1434 /* Restore, Step 50: 1435 * If MFC_Cntl[Se]!=0 then restore 1436 * MFC command queues. 1437 */ 1438 if ((csa->priv2.mfc_control_RW & MFC_CNTL_DMA_QUEUES_EMPTY_MASK) == 0) { 1439 for (i = 0; i < 8; i++) { 1440 out_be64(&priv2->puq[i].mfc_cq_data0_RW, 1441 csa->priv2.puq[i].mfc_cq_data0_RW); 1442 out_be64(&priv2->puq[i].mfc_cq_data1_RW, 1443 csa->priv2.puq[i].mfc_cq_data1_RW); 1444 out_be64(&priv2->puq[i].mfc_cq_data2_RW, 1445 csa->priv2.puq[i].mfc_cq_data2_RW); 1446 out_be64(&priv2->puq[i].mfc_cq_data3_RW, 1447 csa->priv2.puq[i].mfc_cq_data3_RW); 1448 } 1449 for (i = 0; i < 16; i++) { 1450 out_be64(&priv2->spuq[i].mfc_cq_data0_RW, 1451 csa->priv2.spuq[i].mfc_cq_data0_RW); 1452 out_be64(&priv2->spuq[i].mfc_cq_data1_RW, 1453 csa->priv2.spuq[i].mfc_cq_data1_RW); 1454 out_be64(&priv2->spuq[i].mfc_cq_data2_RW, 1455 csa->priv2.spuq[i].mfc_cq_data2_RW); 1456 out_be64(&priv2->spuq[i].mfc_cq_data3_RW, 1457 csa->priv2.spuq[i].mfc_cq_data3_RW); 1458 } 1459 } 1460 eieio(); 1461 } 1462 1463 static inline void restore_ppu_querymask(struct spu_state *csa, struct spu *spu) 1464 { 1465 struct spu_problem __iomem *prob = spu->problem; 1466 1467 /* Restore, Step 51: 1468 * Restore the PPU_QueryMask register from CSA. 1469 */ 1470 out_be32(&prob->dma_querymask_RW, csa->prob.dma_querymask_RW); 1471 eieio(); 1472 } 1473 1474 static inline void restore_ppu_querytype(struct spu_state *csa, struct spu *spu) 1475 { 1476 struct spu_problem __iomem *prob = spu->problem; 1477 1478 /* Restore, Step 52: 1479 * Restore the PPU_QueryType register from CSA. 1480 */ 1481 out_be32(&prob->dma_querytype_RW, csa->prob.dma_querytype_RW); 1482 eieio(); 1483 } 1484 1485 static inline void restore_mfc_csr_tsq(struct spu_state *csa, struct spu *spu) 1486 { 1487 struct spu_priv2 __iomem *priv2 = spu->priv2; 1488 1489 /* Restore, Step 53: 1490 * Restore the MFC_CSR_TSQ register from CSA. 1491 */ 1492 out_be64(&priv2->spu_tag_status_query_RW, 1493 csa->priv2.spu_tag_status_query_RW); 1494 eieio(); 1495 } 1496 1497 static inline void restore_mfc_csr_cmd(struct spu_state *csa, struct spu *spu) 1498 { 1499 struct spu_priv2 __iomem *priv2 = spu->priv2; 1500 1501 /* Restore, Step 54: 1502 * Restore the MFC_CSR_CMD1 and MFC_CSR_CMD2 1503 * registers from CSA. 1504 */ 1505 out_be64(&priv2->spu_cmd_buf1_RW, csa->priv2.spu_cmd_buf1_RW); 1506 out_be64(&priv2->spu_cmd_buf2_RW, csa->priv2.spu_cmd_buf2_RW); 1507 eieio(); 1508 } 1509 1510 static inline void restore_mfc_csr_ato(struct spu_state *csa, struct spu *spu) 1511 { 1512 struct spu_priv2 __iomem *priv2 = spu->priv2; 1513 1514 /* Restore, Step 55: 1515 * Restore the MFC_CSR_ATO register from CSA. 1516 */ 1517 out_be64(&priv2->spu_atomic_status_RW, csa->priv2.spu_atomic_status_RW); 1518 } 1519 1520 static inline void restore_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 1521 { 1522 /* Restore, Step 56: 1523 * Restore the MFC_TCLASS_ID register from CSA. 1524 */ 1525 spu_mfc_tclass_id_set(spu, csa->priv1.mfc_tclass_id_RW); 1526 eieio(); 1527 } 1528 1529 static inline void set_llr_event(struct spu_state *csa, struct spu *spu) 1530 { 1531 u64 ch0_cnt, ch0_data; 1532 u64 ch1_data; 1533 1534 /* Restore, Step 57: 1535 * Set the Lock Line Reservation Lost Event by: 1536 * 1. OR CSA.SPU_Event_Status with bit 21 (Lr) set to 1. 1537 * 2. If CSA.SPU_Channel_0_Count=0 and 1538 * CSA.SPU_Wr_Event_Mask[Lr]=1 and 1539 * CSA.SPU_Event_Status[Lr]=0 then set 1540 * CSA.SPU_Event_Status_Count=1. 1541 */ 1542 ch0_cnt = csa->spu_chnlcnt_RW[0]; 1543 ch0_data = csa->spu_chnldata_RW[0]; 1544 ch1_data = csa->spu_chnldata_RW[1]; 1545 csa->spu_chnldata_RW[0] |= MFC_LLR_LOST_EVENT; 1546 if ((ch0_cnt == 0) && !(ch0_data & MFC_LLR_LOST_EVENT) && 1547 (ch1_data & MFC_LLR_LOST_EVENT)) { 1548 csa->spu_chnlcnt_RW[0] = 1; 1549 } 1550 } 1551 1552 static inline void restore_decr_wrapped(struct spu_state *csa, struct spu *spu) 1553 { 1554 /* Restore, Step 58: 1555 * If the status of the CSA software decrementer 1556 * "wrapped" flag is set, OR in a '1' to 1557 * CSA.SPU_Event_Status[Tm]. 1558 */ 1559 if (csa->lscsa->decr_status.slot[0] == 1) { 1560 csa->spu_chnldata_RW[0] |= 0x20; 1561 } 1562 if ((csa->lscsa->decr_status.slot[0] == 1) && 1563 (csa->spu_chnlcnt_RW[0] == 0 && 1564 ((csa->spu_chnldata_RW[2] & 0x20) == 0x0) && 1565 ((csa->spu_chnldata_RW[0] & 0x20) != 0x1))) { 1566 csa->spu_chnlcnt_RW[0] = 1; 1567 } 1568 } 1569 1570 static inline void restore_ch_part1(struct spu_state *csa, struct spu *spu) 1571 { 1572 struct spu_priv2 __iomem *priv2 = spu->priv2; 1573 u64 idx, ch_indices[7] = { 0UL, 1UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 1574 int i; 1575 1576 /* Restore, Step 59: 1577 * Restore the following CH: [0,1,3,4,24,25,27] 1578 */ 1579 for (i = 0; i < 7; i++) { 1580 idx = ch_indices[i]; 1581 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1582 eieio(); 1583 out_be64(&priv2->spu_chnldata_RW, csa->spu_chnldata_RW[idx]); 1584 out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[idx]); 1585 eieio(); 1586 } 1587 } 1588 1589 static inline void restore_ch_part2(struct spu_state *csa, struct spu *spu) 1590 { 1591 struct spu_priv2 __iomem *priv2 = spu->priv2; 1592 u64 ch_indices[3] = { 9UL, 21UL, 23UL }; 1593 u64 ch_counts[3] = { 1UL, 16UL, 1UL }; 1594 u64 idx; 1595 int i; 1596 1597 /* Restore, Step 60: 1598 * Restore the following CH: [9,21,23]. 1599 */ 1600 ch_counts[0] = 1UL; 1601 ch_counts[1] = csa->spu_chnlcnt_RW[21]; 1602 ch_counts[2] = 1UL; 1603 for (i = 0; i < 3; i++) { 1604 idx = ch_indices[i]; 1605 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1606 eieio(); 1607 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 1608 eieio(); 1609 } 1610 } 1611 1612 static inline void restore_spu_lslr(struct spu_state *csa, struct spu *spu) 1613 { 1614 struct spu_priv2 __iomem *priv2 = spu->priv2; 1615 1616 /* Restore, Step 61: 1617 * Restore the SPU_LSLR register from CSA. 1618 */ 1619 out_be64(&priv2->spu_lslr_RW, csa->priv2.spu_lslr_RW); 1620 eieio(); 1621 } 1622 1623 static inline void restore_spu_cfg(struct spu_state *csa, struct spu *spu) 1624 { 1625 struct spu_priv2 __iomem *priv2 = spu->priv2; 1626 1627 /* Restore, Step 62: 1628 * Restore the SPU_Cfg register from CSA. 1629 */ 1630 out_be64(&priv2->spu_cfg_RW, csa->priv2.spu_cfg_RW); 1631 eieio(); 1632 } 1633 1634 static inline void restore_pm_trace(struct spu_state *csa, struct spu *spu) 1635 { 1636 /* Restore, Step 63: 1637 * Restore PM_Trace_Tag_Wait_Mask from CSA. 1638 * Not performed by this implementation. 1639 */ 1640 } 1641 1642 static inline void restore_spu_npc(struct spu_state *csa, struct spu *spu) 1643 { 1644 struct spu_problem __iomem *prob = spu->problem; 1645 1646 /* Restore, Step 64: 1647 * Restore SPU_NPC from CSA. 1648 */ 1649 out_be32(&prob->spu_npc_RW, csa->prob.spu_npc_RW); 1650 eieio(); 1651 } 1652 1653 static inline void restore_spu_mb(struct spu_state *csa, struct spu *spu) 1654 { 1655 struct spu_priv2 __iomem *priv2 = spu->priv2; 1656 int i; 1657 1658 /* Restore, Step 65: 1659 * Restore MFC_RdSPU_MB from CSA. 1660 */ 1661 out_be64(&priv2->spu_chnlcntptr_RW, 29UL); 1662 eieio(); 1663 out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[29]); 1664 for (i = 0; i < 4; i++) { 1665 out_be64(&priv2->spu_chnldata_RW, csa->spu_mailbox_data[i]); 1666 } 1667 eieio(); 1668 } 1669 1670 static inline void check_ppu_mb_stat(struct spu_state *csa, struct spu *spu) 1671 { 1672 struct spu_problem __iomem *prob = spu->problem; 1673 u32 dummy = 0; 1674 1675 /* Restore, Step 66: 1676 * If CSA.MB_Stat[P]=0 (mailbox empty) then 1677 * read from the PPU_MB register. 1678 */ 1679 if ((csa->prob.mb_stat_R & 0xFF) == 0) { 1680 dummy = in_be32(&prob->pu_mb_R); 1681 eieio(); 1682 } 1683 } 1684 1685 static inline void check_ppuint_mb_stat(struct spu_state *csa, struct spu *spu) 1686 { 1687 struct spu_priv2 __iomem *priv2 = spu->priv2; 1688 u64 dummy = 0UL; 1689 1690 /* Restore, Step 66: 1691 * If CSA.MB_Stat[I]=0 (mailbox empty) then 1692 * read from the PPUINT_MB register. 1693 */ 1694 if ((csa->prob.mb_stat_R & 0xFF0000) == 0) { 1695 dummy = in_be64(&priv2->puint_mb_R); 1696 eieio(); 1697 spu_int_stat_clear(spu, 2, CLASS2_ENABLE_MAILBOX_INTR); 1698 eieio(); 1699 } 1700 } 1701 1702 static inline void restore_mfc_slbs(struct spu_state *csa, struct spu *spu) 1703 { 1704 struct spu_priv2 __iomem *priv2 = spu->priv2; 1705 int i; 1706 1707 /* Restore, Step 68: 1708 * If MFC_SR1[R]='1', restore SLBs from CSA. 1709 */ 1710 if (csa->priv1.mfc_sr1_RW & MFC_STATE1_RELOCATE_MASK) { 1711 for (i = 0; i < 8; i++) { 1712 out_be64(&priv2->slb_index_W, i); 1713 eieio(); 1714 out_be64(&priv2->slb_esid_RW, csa->slb_esid_RW[i]); 1715 out_be64(&priv2->slb_vsid_RW, csa->slb_vsid_RW[i]); 1716 eieio(); 1717 } 1718 out_be64(&priv2->slb_index_W, csa->priv2.slb_index_W); 1719 eieio(); 1720 } 1721 } 1722 1723 static inline void restore_mfc_sr1(struct spu_state *csa, struct spu *spu) 1724 { 1725 /* Restore, Step 69: 1726 * Restore the MFC_SR1 register from CSA. 1727 */ 1728 spu_mfc_sr1_set(spu, csa->priv1.mfc_sr1_RW); 1729 eieio(); 1730 } 1731 1732 static inline void restore_other_spu_access(struct spu_state *csa, 1733 struct spu *spu) 1734 { 1735 /* Restore, Step 70: 1736 * Restore other SPU mappings to this SPU. TBD. 1737 */ 1738 } 1739 1740 static inline void restore_spu_runcntl(struct spu_state *csa, struct spu *spu) 1741 { 1742 struct spu_problem __iomem *prob = spu->problem; 1743 1744 /* Restore, Step 71: 1745 * If CSA.SPU_Status[R]=1 then write 1746 * SPU_RunCntl[R0R1]='01'. 1747 */ 1748 if (csa->prob.spu_status_R & SPU_STATUS_RUNNING) { 1749 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1750 eieio(); 1751 } 1752 } 1753 1754 static inline void restore_mfc_cntl(struct spu_state *csa, struct spu *spu) 1755 { 1756 struct spu_priv2 __iomem *priv2 = spu->priv2; 1757 1758 /* Restore, Step 72: 1759 * Restore the MFC_CNTL register for the CSA. 1760 */ 1761 out_be64(&priv2->mfc_control_RW, csa->priv2.mfc_control_RW); 1762 eieio(); 1763 } 1764 1765 static inline void enable_user_access(struct spu_state *csa, struct spu *spu) 1766 { 1767 /* Restore, Step 73: 1768 * Enable user-space access (if provided) to this 1769 * SPU by mapping the virtual pages assigned to 1770 * the SPU memory-mapped I/O (MMIO) for problem 1771 * state. TBD. 1772 */ 1773 } 1774 1775 static inline void reset_switch_active(struct spu_state *csa, struct spu *spu) 1776 { 1777 /* Restore, Step 74: 1778 * Reset the "context switch active" flag. 1779 */ 1780 clear_bit(SPU_CONTEXT_SWITCH_ACTIVE, &spu->flags); 1781 mb(); 1782 } 1783 1784 static inline void reenable_interrupts(struct spu_state *csa, struct spu *spu) 1785 { 1786 /* Restore, Step 75: 1787 * Re-enable SPU interrupts. 1788 */ 1789 spin_lock_irq(&spu->register_lock); 1790 spu_int_mask_set(spu, 0, csa->priv1.int_mask_class0_RW); 1791 spu_int_mask_set(spu, 1, csa->priv1.int_mask_class1_RW); 1792 spu_int_mask_set(spu, 2, csa->priv1.int_mask_class2_RW); 1793 spin_unlock_irq(&spu->register_lock); 1794 } 1795 1796 static int quiece_spu(struct spu_state *prev, struct spu *spu) 1797 { 1798 /* 1799 * Combined steps 2-18 of SPU context save sequence, which 1800 * quiesce the SPU state (disable SPU execution, MFC command 1801 * queues, decrementer, SPU interrupts, etc.). 1802 * 1803 * Returns 0 on success. 1804 * 2 if failed step 2. 1805 * 6 if failed step 6. 1806 */ 1807 1808 if (check_spu_isolate(prev, spu)) { /* Step 2. */ 1809 return 2; 1810 } 1811 disable_interrupts(prev, spu); /* Step 3. */ 1812 set_watchdog_timer(prev, spu); /* Step 4. */ 1813 inhibit_user_access(prev, spu); /* Step 5. */ 1814 if (check_spu_isolate(prev, spu)) { /* Step 6. */ 1815 return 6; 1816 } 1817 set_switch_pending(prev, spu); /* Step 7. */ 1818 save_mfc_cntl(prev, spu); /* Step 8. */ 1819 save_spu_runcntl(prev, spu); /* Step 9. */ 1820 save_mfc_sr1(prev, spu); /* Step 10. */ 1821 save_spu_status(prev, spu); /* Step 11. */ 1822 save_mfc_decr(prev, spu); /* Step 12. */ 1823 halt_mfc_decr(prev, spu); /* Step 13. */ 1824 save_timebase(prev, spu); /* Step 14. */ 1825 remove_other_spu_access(prev, spu); /* Step 15. */ 1826 do_mfc_mssync(prev, spu); /* Step 16. */ 1827 issue_mfc_tlbie(prev, spu); /* Step 17. */ 1828 handle_pending_interrupts(prev, spu); /* Step 18. */ 1829 1830 return 0; 1831 } 1832 1833 static void save_csa(struct spu_state *prev, struct spu *spu) 1834 { 1835 /* 1836 * Combine steps 19-44 of SPU context save sequence, which 1837 * save regions of the privileged & problem state areas. 1838 */ 1839 1840 save_mfc_queues(prev, spu); /* Step 19. */ 1841 save_ppu_querymask(prev, spu); /* Step 20. */ 1842 save_ppu_querytype(prev, spu); /* Step 21. */ 1843 save_mfc_csr_tsq(prev, spu); /* Step 22. */ 1844 save_mfc_csr_cmd(prev, spu); /* Step 23. */ 1845 save_mfc_csr_ato(prev, spu); /* Step 24. */ 1846 save_mfc_tclass_id(prev, spu); /* Step 25. */ 1847 set_mfc_tclass_id(prev, spu); /* Step 26. */ 1848 purge_mfc_queue(prev, spu); /* Step 27. */ 1849 wait_purge_complete(prev, spu); /* Step 28. */ 1850 save_mfc_slbs(prev, spu); /* Step 29. */ 1851 setup_mfc_sr1(prev, spu); /* Step 30. */ 1852 save_spu_npc(prev, spu); /* Step 31. */ 1853 save_spu_privcntl(prev, spu); /* Step 32. */ 1854 reset_spu_privcntl(prev, spu); /* Step 33. */ 1855 save_spu_lslr(prev, spu); /* Step 34. */ 1856 reset_spu_lslr(prev, spu); /* Step 35. */ 1857 save_spu_cfg(prev, spu); /* Step 36. */ 1858 save_pm_trace(prev, spu); /* Step 37. */ 1859 save_mfc_rag(prev, spu); /* Step 38. */ 1860 save_ppu_mb_stat(prev, spu); /* Step 39. */ 1861 save_ppu_mb(prev, spu); /* Step 40. */ 1862 save_ppuint_mb(prev, spu); /* Step 41. */ 1863 save_ch_part1(prev, spu); /* Step 42. */ 1864 save_spu_mb(prev, spu); /* Step 43. */ 1865 save_mfc_cmd(prev, spu); /* Step 44. */ 1866 reset_ch(prev, spu); /* Step 45. */ 1867 } 1868 1869 static void save_lscsa(struct spu_state *prev, struct spu *spu) 1870 { 1871 /* 1872 * Perform steps 46-57 of SPU context save sequence, 1873 * which save regions of the local store and register 1874 * file. 1875 */ 1876 1877 resume_mfc_queue(prev, spu); /* Step 46. */ 1878 setup_mfc_slbs(prev, spu); /* Step 47. */ 1879 set_switch_active(prev, spu); /* Step 48. */ 1880 enable_interrupts(prev, spu); /* Step 49. */ 1881 save_ls_16kb(prev, spu); /* Step 50. */ 1882 set_spu_npc(prev, spu); /* Step 51. */ 1883 set_signot1(prev, spu); /* Step 52. */ 1884 set_signot2(prev, spu); /* Step 53. */ 1885 send_save_code(prev, spu); /* Step 54. */ 1886 set_ppu_querymask(prev, spu); /* Step 55. */ 1887 wait_tag_complete(prev, spu); /* Step 56. */ 1888 wait_spu_stopped(prev, spu); /* Step 57. */ 1889 } 1890 1891 static void harvest(struct spu_state *prev, struct spu *spu) 1892 { 1893 /* 1894 * Perform steps 2-25 of SPU context restore sequence, 1895 * which resets an SPU either after a failed save, or 1896 * when using SPU for first time. 1897 */ 1898 1899 disable_interrupts(prev, spu); /* Step 2. */ 1900 inhibit_user_access(prev, spu); /* Step 3. */ 1901 terminate_spu_app(prev, spu); /* Step 4. */ 1902 set_switch_pending(prev, spu); /* Step 5. */ 1903 remove_other_spu_access(prev, spu); /* Step 6. */ 1904 suspend_mfc(prev, spu); /* Step 7. */ 1905 wait_suspend_mfc_complete(prev, spu); /* Step 8. */ 1906 if (!suspend_spe(prev, spu)) /* Step 9. */ 1907 clear_spu_status(prev, spu); /* Step 10. */ 1908 do_mfc_mssync(prev, spu); /* Step 11. */ 1909 issue_mfc_tlbie(prev, spu); /* Step 12. */ 1910 handle_pending_interrupts(prev, spu); /* Step 13. */ 1911 purge_mfc_queue(prev, spu); /* Step 14. */ 1912 wait_purge_complete(prev, spu); /* Step 15. */ 1913 reset_spu_privcntl(prev, spu); /* Step 16. */ 1914 reset_spu_lslr(prev, spu); /* Step 17. */ 1915 setup_mfc_sr1(prev, spu); /* Step 18. */ 1916 invalidate_slbs(prev, spu); /* Step 19. */ 1917 reset_ch_part1(prev, spu); /* Step 20. */ 1918 reset_ch_part2(prev, spu); /* Step 21. */ 1919 enable_interrupts(prev, spu); /* Step 22. */ 1920 set_switch_active(prev, spu); /* Step 23. */ 1921 set_mfc_tclass_id(prev, spu); /* Step 24. */ 1922 resume_mfc_queue(prev, spu); /* Step 25. */ 1923 } 1924 1925 static void restore_lscsa(struct spu_state *next, struct spu *spu) 1926 { 1927 /* 1928 * Perform steps 26-40 of SPU context restore sequence, 1929 * which restores regions of the local store and register 1930 * file. 1931 */ 1932 1933 set_watchdog_timer(next, spu); /* Step 26. */ 1934 setup_spu_status_part1(next, spu); /* Step 27. */ 1935 setup_spu_status_part2(next, spu); /* Step 28. */ 1936 restore_mfc_rag(next, spu); /* Step 29. */ 1937 setup_mfc_slbs(next, spu); /* Step 30. */ 1938 set_spu_npc(next, spu); /* Step 31. */ 1939 set_signot1(next, spu); /* Step 32. */ 1940 set_signot2(next, spu); /* Step 33. */ 1941 setup_decr(next, spu); /* Step 34. */ 1942 setup_ppu_mb(next, spu); /* Step 35. */ 1943 setup_ppuint_mb(next, spu); /* Step 36. */ 1944 send_restore_code(next, spu); /* Step 37. */ 1945 set_ppu_querymask(next, spu); /* Step 38. */ 1946 wait_tag_complete(next, spu); /* Step 39. */ 1947 wait_spu_stopped(next, spu); /* Step 40. */ 1948 } 1949 1950 static void restore_csa(struct spu_state *next, struct spu *spu) 1951 { 1952 /* 1953 * Combine steps 41-76 of SPU context restore sequence, which 1954 * restore regions of the privileged & problem state areas. 1955 */ 1956 1957 restore_spu_privcntl(next, spu); /* Step 41. */ 1958 restore_status_part1(next, spu); /* Step 42. */ 1959 restore_status_part2(next, spu); /* Step 43. */ 1960 restore_ls_16kb(next, spu); /* Step 44. */ 1961 wait_tag_complete(next, spu); /* Step 45. */ 1962 suspend_mfc(next, spu); /* Step 46. */ 1963 wait_suspend_mfc_complete(next, spu); /* Step 47. */ 1964 issue_mfc_tlbie(next, spu); /* Step 48. */ 1965 clear_interrupts(next, spu); /* Step 49. */ 1966 restore_mfc_queues(next, spu); /* Step 50. */ 1967 restore_ppu_querymask(next, spu); /* Step 51. */ 1968 restore_ppu_querytype(next, spu); /* Step 52. */ 1969 restore_mfc_csr_tsq(next, spu); /* Step 53. */ 1970 restore_mfc_csr_cmd(next, spu); /* Step 54. */ 1971 restore_mfc_csr_ato(next, spu); /* Step 55. */ 1972 restore_mfc_tclass_id(next, spu); /* Step 56. */ 1973 set_llr_event(next, spu); /* Step 57. */ 1974 restore_decr_wrapped(next, spu); /* Step 58. */ 1975 restore_ch_part1(next, spu); /* Step 59. */ 1976 restore_ch_part2(next, spu); /* Step 60. */ 1977 restore_spu_lslr(next, spu); /* Step 61. */ 1978 restore_spu_cfg(next, spu); /* Step 62. */ 1979 restore_pm_trace(next, spu); /* Step 63. */ 1980 restore_spu_npc(next, spu); /* Step 64. */ 1981 restore_spu_mb(next, spu); /* Step 65. */ 1982 check_ppu_mb_stat(next, spu); /* Step 66. */ 1983 check_ppuint_mb_stat(next, spu); /* Step 67. */ 1984 restore_mfc_slbs(next, spu); /* Step 68. */ 1985 restore_mfc_sr1(next, spu); /* Step 69. */ 1986 restore_other_spu_access(next, spu); /* Step 70. */ 1987 restore_spu_runcntl(next, spu); /* Step 71. */ 1988 restore_mfc_cntl(next, spu); /* Step 72. */ 1989 enable_user_access(next, spu); /* Step 73. */ 1990 reset_switch_active(next, spu); /* Step 74. */ 1991 reenable_interrupts(next, spu); /* Step 75. */ 1992 } 1993 1994 static int __do_spu_save(struct spu_state *prev, struct spu *spu) 1995 { 1996 int rc; 1997 1998 /* 1999 * SPU context save can be broken into three phases: 2000 * 2001 * (a) quiesce [steps 2-16]. 2002 * (b) save of CSA, performed by PPE [steps 17-42] 2003 * (c) save of LSCSA, mostly performed by SPU [steps 43-52]. 2004 * 2005 * Returns 0 on success. 2006 * 2,6 if failed to quiece SPU 2007 * 53 if SPU-side of save failed. 2008 */ 2009 2010 rc = quiece_spu(prev, spu); /* Steps 2-16. */ 2011 switch (rc) { 2012 default: 2013 case 2: 2014 case 6: 2015 harvest(prev, spu); 2016 return rc; 2017 break; 2018 case 0: 2019 break; 2020 } 2021 save_csa(prev, spu); /* Steps 17-43. */ 2022 save_lscsa(prev, spu); /* Steps 44-53. */ 2023 return check_save_status(prev, spu); /* Step 54. */ 2024 } 2025 2026 static int __do_spu_restore(struct spu_state *next, struct spu *spu) 2027 { 2028 int rc; 2029 2030 /* 2031 * SPU context restore can be broken into three phases: 2032 * 2033 * (a) harvest (or reset) SPU [steps 2-24]. 2034 * (b) restore LSCSA [steps 25-40], mostly performed by SPU. 2035 * (c) restore CSA [steps 41-76], performed by PPE. 2036 * 2037 * The 'harvest' step is not performed here, but rather 2038 * as needed below. 2039 */ 2040 2041 restore_lscsa(next, spu); /* Steps 24-39. */ 2042 rc = check_restore_status(next, spu); /* Step 40. */ 2043 switch (rc) { 2044 default: 2045 /* Failed. Return now. */ 2046 return rc; 2047 break; 2048 case 0: 2049 /* Fall through to next step. */ 2050 break; 2051 } 2052 restore_csa(next, spu); 2053 2054 return 0; 2055 } 2056 2057 /** 2058 * spu_save - SPU context save, with locking. 2059 * @prev: pointer to SPU context save area, to be saved. 2060 * @spu: pointer to SPU iomem structure. 2061 * 2062 * Acquire locks, perform the save operation then return. 2063 */ 2064 int spu_save(struct spu_state *prev, struct spu *spu) 2065 { 2066 int rc; 2067 2068 acquire_spu_lock(spu); /* Step 1. */ 2069 rc = __do_spu_save(prev, spu); /* Steps 2-53. */ 2070 release_spu_lock(spu); 2071 if (rc) { 2072 panic("%s failed on SPU[%d], rc=%d.\n", 2073 __func__, spu->number, rc); 2074 } 2075 return rc; 2076 } 2077 2078 /** 2079 * spu_restore - SPU context restore, with harvest and locking. 2080 * @new: pointer to SPU context save area, to be restored. 2081 * @spu: pointer to SPU iomem structure. 2082 * 2083 * Perform harvest + restore, as we may not be coming 2084 * from a previous succesful save operation, and the 2085 * hardware state is unknown. 2086 */ 2087 int spu_restore(struct spu_state *new, struct spu *spu) 2088 { 2089 int rc; 2090 2091 acquire_spu_lock(spu); 2092 harvest(NULL, spu); 2093 spu->stop_code = 0; 2094 spu->dar = 0; 2095 spu->dsisr = 0; 2096 spu->slb_replace = 0; 2097 spu->class_0_pending = 0; 2098 rc = __do_spu_restore(new, spu); 2099 release_spu_lock(spu); 2100 if (rc) { 2101 panic("%s failed on SPU[%d] rc=%d.\n", 2102 __func__, spu->number, rc); 2103 } 2104 return rc; 2105 } 2106 2107 /** 2108 * spu_harvest - SPU harvest (reset) operation 2109 * @spu: pointer to SPU iomem structure. 2110 * 2111 * Perform SPU harvest (reset) operation. 2112 */ 2113 void spu_harvest(struct spu *spu) 2114 { 2115 acquire_spu_lock(spu); 2116 harvest(NULL, spu); 2117 release_spu_lock(spu); 2118 } 2119 2120 static void init_prob(struct spu_state *csa) 2121 { 2122 csa->spu_chnlcnt_RW[9] = 1; 2123 csa->spu_chnlcnt_RW[21] = 16; 2124 csa->spu_chnlcnt_RW[23] = 1; 2125 csa->spu_chnlcnt_RW[28] = 1; 2126 csa->spu_chnlcnt_RW[30] = 1; 2127 csa->prob.spu_runcntl_RW = SPU_RUNCNTL_STOP; 2128 } 2129 2130 static void init_priv1(struct spu_state *csa) 2131 { 2132 /* Enable decode, relocate, tlbie response, master runcntl. */ 2133 csa->priv1.mfc_sr1_RW = MFC_STATE1_LOCAL_STORAGE_DECODE_MASK | 2134 MFC_STATE1_MASTER_RUN_CONTROL_MASK | 2135 MFC_STATE1_PROBLEM_STATE_MASK | 2136 MFC_STATE1_RELOCATE_MASK | MFC_STATE1_BUS_TLBIE_MASK; 2137 2138 /* Set storage description. */ 2139 csa->priv1.mfc_sdr_RW = mfspr(SPRN_SDR1); 2140 2141 /* Enable OS-specific set of interrupts. */ 2142 csa->priv1.int_mask_class0_RW = CLASS0_ENABLE_DMA_ALIGNMENT_INTR | 2143 CLASS0_ENABLE_INVALID_DMA_COMMAND_INTR | 2144 CLASS0_ENABLE_SPU_ERROR_INTR; 2145 csa->priv1.int_mask_class1_RW = CLASS1_ENABLE_SEGMENT_FAULT_INTR | 2146 CLASS1_ENABLE_STORAGE_FAULT_INTR; 2147 csa->priv1.int_mask_class2_RW = CLASS2_ENABLE_SPU_STOP_INTR | 2148 CLASS2_ENABLE_SPU_HALT_INTR | 2149 CLASS2_ENABLE_SPU_DMA_TAG_GROUP_COMPLETE_INTR; 2150 } 2151 2152 static void init_priv2(struct spu_state *csa) 2153 { 2154 csa->priv2.spu_lslr_RW = LS_ADDR_MASK; 2155 csa->priv2.mfc_control_RW = MFC_CNTL_RESUME_DMA_QUEUE | 2156 MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION | 2157 MFC_CNTL_DMA_QUEUES_EMPTY_MASK; 2158 } 2159 2160 /** 2161 * spu_alloc_csa - allocate and initialize an SPU context save area. 2162 * 2163 * Allocate and initialize the contents of an SPU context save area. 2164 * This includes enabling address translation, interrupt masks, etc., 2165 * as appropriate for the given OS environment. 2166 * 2167 * Note that storage for the 'lscsa' is allocated separately, 2168 * as it is by far the largest of the context save regions, 2169 * and may need to be pinned or otherwise specially aligned. 2170 */ 2171 void spu_init_csa(struct spu_state *csa) 2172 { 2173 struct spu_lscsa *lscsa; 2174 unsigned char *p; 2175 2176 if (!csa) 2177 return; 2178 memset(csa, 0, sizeof(struct spu_state)); 2179 2180 lscsa = vmalloc(sizeof(struct spu_lscsa)); 2181 if (!lscsa) 2182 return; 2183 2184 memset(lscsa, 0, sizeof(struct spu_lscsa)); 2185 csa->lscsa = lscsa; 2186 csa->register_lock = SPIN_LOCK_UNLOCKED; 2187 2188 /* Set LS pages reserved to allow for user-space mapping. */ 2189 for (p = lscsa->ls; p < lscsa->ls + LS_SIZE; p += PAGE_SIZE) 2190 SetPageReserved(vmalloc_to_page(p)); 2191 2192 init_prob(csa); 2193 init_priv1(csa); 2194 init_priv2(csa); 2195 } 2196 2197 void spu_fini_csa(struct spu_state *csa) 2198 { 2199 /* Clear reserved bit before vfree. */ 2200 unsigned char *p; 2201 for (p = csa->lscsa->ls; p < csa->lscsa->ls + LS_SIZE; p += PAGE_SIZE) 2202 ClearPageReserved(vmalloc_to_page(p)); 2203 2204 vfree(csa->lscsa); 2205 } 2206