1 /* 2 * spu_switch.c 3 * 4 * (C) Copyright IBM Corp. 2005 5 * 6 * Author: Mark Nutter <mnutter@us.ibm.com> 7 * 8 * Host-side part of SPU context switch sequence outlined in 9 * Synergistic Processor Element, Book IV. 10 * 11 * A fully premptive switch of an SPE is very expensive in terms 12 * of time and system resources. SPE Book IV indicates that SPE 13 * allocation should follow a "serially reusable device" model, 14 * in which the SPE is assigned a task until it completes. When 15 * this is not possible, this sequence may be used to premptively 16 * save, and then later (optionally) restore the context of a 17 * program executing on an SPE. 18 * 19 * 20 * This program is free software; you can redistribute it and/or modify 21 * it under the terms of the GNU General Public License as published by 22 * the Free Software Foundation; either version 2, or (at your option) 23 * any later version. 24 * 25 * This program is distributed in the hope that it will be useful, 26 * but WITHOUT ANY WARRANTY; without even the implied warranty of 27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 28 * GNU General Public License for more details. 29 * 30 * You should have received a copy of the GNU General Public License 31 * along with this program; if not, write to the Free Software 32 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 33 */ 34 35 #include <linux/module.h> 36 #include <linux/errno.h> 37 #include <linux/sched.h> 38 #include <linux/kernel.h> 39 #include <linux/mm.h> 40 #include <linux/vmalloc.h> 41 #include <linux/smp.h> 42 #include <linux/smp_lock.h> 43 #include <linux/stddef.h> 44 #include <linux/unistd.h> 45 46 #include <asm/io.h> 47 #include <asm/spu.h> 48 #include <asm/spu_priv1.h> 49 #include <asm/spu_csa.h> 50 #include <asm/mmu_context.h> 51 52 #include "spu_save_dump.h" 53 #include "spu_restore_dump.h" 54 55 #if 0 56 #define POLL_WHILE_TRUE(_c) { \ 57 do { \ 58 } while (_c); \ 59 } 60 #else 61 #define RELAX_SPIN_COUNT 1000 62 #define POLL_WHILE_TRUE(_c) { \ 63 do { \ 64 int _i; \ 65 for (_i=0; _i<RELAX_SPIN_COUNT && (_c); _i++) { \ 66 cpu_relax(); \ 67 } \ 68 if (unlikely(_c)) yield(); \ 69 else break; \ 70 } while (_c); \ 71 } 72 #endif /* debug */ 73 74 #define POLL_WHILE_FALSE(_c) POLL_WHILE_TRUE(!(_c)) 75 76 static inline void acquire_spu_lock(struct spu *spu) 77 { 78 /* Save, Step 1: 79 * Restore, Step 1: 80 * Acquire SPU-specific mutual exclusion lock. 81 * TBD. 82 */ 83 } 84 85 static inline void release_spu_lock(struct spu *spu) 86 { 87 /* Restore, Step 76: 88 * Release SPU-specific mutual exclusion lock. 89 * TBD. 90 */ 91 } 92 93 static inline int check_spu_isolate(struct spu_state *csa, struct spu *spu) 94 { 95 struct spu_problem __iomem *prob = spu->problem; 96 u32 isolate_state; 97 98 /* Save, Step 2: 99 * Save, Step 6: 100 * If SPU_Status[E,L,IS] any field is '1', this 101 * SPU is in isolate state and cannot be context 102 * saved at this time. 103 */ 104 isolate_state = SPU_STATUS_ISOLATED_STATE | 105 SPU_STATUS_ISOLATED_LOAD_STATUS | SPU_STATUS_ISOLATED_EXIT_STATUS; 106 return (in_be32(&prob->spu_status_R) & isolate_state) ? 1 : 0; 107 } 108 109 static inline void disable_interrupts(struct spu_state *csa, struct spu *spu) 110 { 111 /* Save, Step 3: 112 * Restore, Step 2: 113 * Save INT_Mask_class0 in CSA. 114 * Write INT_MASK_class0 with value of 0. 115 * Save INT_Mask_class1 in CSA. 116 * Write INT_MASK_class1 with value of 0. 117 * Save INT_Mask_class2 in CSA. 118 * Write INT_MASK_class2 with value of 0. 119 */ 120 spin_lock_irq(&spu->register_lock); 121 if (csa) { 122 csa->priv1.int_mask_class0_RW = spu_int_mask_get(spu, 0); 123 csa->priv1.int_mask_class1_RW = spu_int_mask_get(spu, 1); 124 csa->priv1.int_mask_class2_RW = spu_int_mask_get(spu, 2); 125 } 126 spu_int_mask_set(spu, 0, 0ul); 127 spu_int_mask_set(spu, 1, 0ul); 128 spu_int_mask_set(spu, 2, 0ul); 129 eieio(); 130 spin_unlock_irq(&spu->register_lock); 131 } 132 133 static inline void set_watchdog_timer(struct spu_state *csa, struct spu *spu) 134 { 135 /* Save, Step 4: 136 * Restore, Step 25. 137 * Set a software watchdog timer, which specifies the 138 * maximum allowable time for a context save sequence. 139 * 140 * For present, this implementation will not set a global 141 * watchdog timer, as virtualization & variable system load 142 * may cause unpredictable execution times. 143 */ 144 } 145 146 static inline void inhibit_user_access(struct spu_state *csa, struct spu *spu) 147 { 148 /* Save, Step 5: 149 * Restore, Step 3: 150 * Inhibit user-space access (if provided) to this 151 * SPU by unmapping the virtual pages assigned to 152 * the SPU memory-mapped I/O (MMIO) for problem 153 * state. TBD. 154 */ 155 } 156 157 static inline void set_switch_pending(struct spu_state *csa, struct spu *spu) 158 { 159 /* Save, Step 7: 160 * Restore, Step 5: 161 * Set a software context switch pending flag. 162 */ 163 set_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags); 164 mb(); 165 } 166 167 static inline void save_mfc_cntl(struct spu_state *csa, struct spu *spu) 168 { 169 struct spu_priv2 __iomem *priv2 = spu->priv2; 170 171 /* Save, Step 8: 172 * Suspend DMA and save MFC_CNTL. 173 */ 174 switch (in_be64(&priv2->mfc_control_RW) & 175 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) { 176 case MFC_CNTL_SUSPEND_IN_PROGRESS: 177 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 178 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) == 179 MFC_CNTL_SUSPEND_COMPLETE); 180 /* fall through */ 181 case MFC_CNTL_SUSPEND_COMPLETE: 182 if (csa) { 183 csa->priv2.mfc_control_RW = 184 in_be64(&priv2->mfc_control_RW) | 185 MFC_CNTL_SUSPEND_DMA_QUEUE; 186 } 187 break; 188 case MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION: 189 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE); 190 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 191 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) == 192 MFC_CNTL_SUSPEND_COMPLETE); 193 if (csa) { 194 csa->priv2.mfc_control_RW = 195 in_be64(&priv2->mfc_control_RW) & 196 ~MFC_CNTL_SUSPEND_DMA_QUEUE; 197 } 198 break; 199 } 200 } 201 202 static inline void save_spu_runcntl(struct spu_state *csa, struct spu *spu) 203 { 204 struct spu_problem __iomem *prob = spu->problem; 205 206 /* Save, Step 9: 207 * Save SPU_Runcntl in the CSA. This value contains 208 * the "Application Desired State". 209 */ 210 csa->prob.spu_runcntl_RW = in_be32(&prob->spu_runcntl_RW); 211 } 212 213 static inline void save_mfc_sr1(struct spu_state *csa, struct spu *spu) 214 { 215 /* Save, Step 10: 216 * Save MFC_SR1 in the CSA. 217 */ 218 csa->priv1.mfc_sr1_RW = spu_mfc_sr1_get(spu); 219 } 220 221 static inline void save_spu_status(struct spu_state *csa, struct spu *spu) 222 { 223 struct spu_problem __iomem *prob = spu->problem; 224 225 /* Save, Step 11: 226 * Read SPU_Status[R], and save to CSA. 227 */ 228 if ((in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) == 0) { 229 csa->prob.spu_status_R = in_be32(&prob->spu_status_R); 230 } else { 231 u32 stopped; 232 233 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 234 eieio(); 235 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 236 SPU_STATUS_RUNNING); 237 stopped = 238 SPU_STATUS_INVALID_INSTR | SPU_STATUS_SINGLE_STEP | 239 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP; 240 if ((in_be32(&prob->spu_status_R) & stopped) == 0) 241 csa->prob.spu_status_R = SPU_STATUS_RUNNING; 242 else 243 csa->prob.spu_status_R = in_be32(&prob->spu_status_R); 244 } 245 } 246 247 static inline void save_mfc_decr(struct spu_state *csa, struct spu *spu) 248 { 249 struct spu_priv2 __iomem *priv2 = spu->priv2; 250 251 /* Save, Step 12: 252 * Read MFC_CNTL[Ds]. Update saved copy of 253 * CSA.MFC_CNTL[Ds]. 254 */ 255 if (in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DECREMENTER_RUNNING) { 256 csa->priv2.mfc_control_RW |= MFC_CNTL_DECREMENTER_RUNNING; 257 csa->suspend_time = get_cycles(); 258 out_be64(&priv2->spu_chnlcntptr_RW, 7ULL); 259 eieio(); 260 csa->spu_chnldata_RW[7] = in_be64(&priv2->spu_chnldata_RW); 261 eieio(); 262 } else { 263 csa->priv2.mfc_control_RW &= ~MFC_CNTL_DECREMENTER_RUNNING; 264 } 265 } 266 267 static inline void halt_mfc_decr(struct spu_state *csa, struct spu *spu) 268 { 269 struct spu_priv2 __iomem *priv2 = spu->priv2; 270 271 /* Save, Step 13: 272 * Write MFC_CNTL[Dh] set to a '1' to halt 273 * the decrementer. 274 */ 275 out_be64(&priv2->mfc_control_RW, MFC_CNTL_DECREMENTER_HALTED); 276 eieio(); 277 } 278 279 static inline void save_timebase(struct spu_state *csa, struct spu *spu) 280 { 281 /* Save, Step 14: 282 * Read PPE Timebase High and Timebase low registers 283 * and save in CSA. TBD. 284 */ 285 csa->suspend_time = get_cycles(); 286 } 287 288 static inline void remove_other_spu_access(struct spu_state *csa, 289 struct spu *spu) 290 { 291 /* Save, Step 15: 292 * Remove other SPU access to this SPU by unmapping 293 * this SPU's pages from their address space. TBD. 294 */ 295 } 296 297 static inline void do_mfc_mssync(struct spu_state *csa, struct spu *spu) 298 { 299 struct spu_problem __iomem *prob = spu->problem; 300 301 /* Save, Step 16: 302 * Restore, Step 11. 303 * Write SPU_MSSync register. Poll SPU_MSSync[P] 304 * for a value of 0. 305 */ 306 out_be64(&prob->spc_mssync_RW, 1UL); 307 POLL_WHILE_TRUE(in_be64(&prob->spc_mssync_RW) & MS_SYNC_PENDING); 308 } 309 310 static inline void issue_mfc_tlbie(struct spu_state *csa, struct spu *spu) 311 { 312 /* Save, Step 17: 313 * Restore, Step 12. 314 * Restore, Step 48. 315 * Write TLB_Invalidate_Entry[IS,VPN,L,Lp]=0 register. 316 * Then issue a PPE sync instruction. 317 */ 318 spu_tlb_invalidate(spu); 319 mb(); 320 } 321 322 static inline void handle_pending_interrupts(struct spu_state *csa, 323 struct spu *spu) 324 { 325 /* Save, Step 18: 326 * Handle any pending interrupts from this SPU 327 * here. This is OS or hypervisor specific. One 328 * option is to re-enable interrupts to handle any 329 * pending interrupts, with the interrupt handlers 330 * recognizing the software Context Switch Pending 331 * flag, to ensure the SPU execution or MFC command 332 * queue is not restarted. TBD. 333 */ 334 } 335 336 static inline void save_mfc_queues(struct spu_state *csa, struct spu *spu) 337 { 338 struct spu_priv2 __iomem *priv2 = spu->priv2; 339 int i; 340 341 /* Save, Step 19: 342 * If MFC_Cntl[Se]=0 then save 343 * MFC command queues. 344 */ 345 if ((in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DMA_QUEUES_EMPTY) == 0) { 346 for (i = 0; i < 8; i++) { 347 csa->priv2.puq[i].mfc_cq_data0_RW = 348 in_be64(&priv2->puq[i].mfc_cq_data0_RW); 349 csa->priv2.puq[i].mfc_cq_data1_RW = 350 in_be64(&priv2->puq[i].mfc_cq_data1_RW); 351 csa->priv2.puq[i].mfc_cq_data2_RW = 352 in_be64(&priv2->puq[i].mfc_cq_data2_RW); 353 csa->priv2.puq[i].mfc_cq_data3_RW = 354 in_be64(&priv2->puq[i].mfc_cq_data3_RW); 355 } 356 for (i = 0; i < 16; i++) { 357 csa->priv2.spuq[i].mfc_cq_data0_RW = 358 in_be64(&priv2->spuq[i].mfc_cq_data0_RW); 359 csa->priv2.spuq[i].mfc_cq_data1_RW = 360 in_be64(&priv2->spuq[i].mfc_cq_data1_RW); 361 csa->priv2.spuq[i].mfc_cq_data2_RW = 362 in_be64(&priv2->spuq[i].mfc_cq_data2_RW); 363 csa->priv2.spuq[i].mfc_cq_data3_RW = 364 in_be64(&priv2->spuq[i].mfc_cq_data3_RW); 365 } 366 } 367 } 368 369 static inline void save_ppu_querymask(struct spu_state *csa, struct spu *spu) 370 { 371 struct spu_problem __iomem *prob = spu->problem; 372 373 /* Save, Step 20: 374 * Save the PPU_QueryMask register 375 * in the CSA. 376 */ 377 csa->prob.dma_querymask_RW = in_be32(&prob->dma_querymask_RW); 378 } 379 380 static inline void save_ppu_querytype(struct spu_state *csa, struct spu *spu) 381 { 382 struct spu_problem __iomem *prob = spu->problem; 383 384 /* Save, Step 21: 385 * Save the PPU_QueryType register 386 * in the CSA. 387 */ 388 csa->prob.dma_querytype_RW = in_be32(&prob->dma_querytype_RW); 389 } 390 391 static inline void save_mfc_csr_tsq(struct spu_state *csa, struct spu *spu) 392 { 393 struct spu_priv2 __iomem *priv2 = spu->priv2; 394 395 /* Save, Step 22: 396 * Save the MFC_CSR_TSQ register 397 * in the LSCSA. 398 */ 399 csa->priv2.spu_tag_status_query_RW = 400 in_be64(&priv2->spu_tag_status_query_RW); 401 } 402 403 static inline void save_mfc_csr_cmd(struct spu_state *csa, struct spu *spu) 404 { 405 struct spu_priv2 __iomem *priv2 = spu->priv2; 406 407 /* Save, Step 23: 408 * Save the MFC_CSR_CMD1 and MFC_CSR_CMD2 409 * registers in the CSA. 410 */ 411 csa->priv2.spu_cmd_buf1_RW = in_be64(&priv2->spu_cmd_buf1_RW); 412 csa->priv2.spu_cmd_buf2_RW = in_be64(&priv2->spu_cmd_buf2_RW); 413 } 414 415 static inline void save_mfc_csr_ato(struct spu_state *csa, struct spu *spu) 416 { 417 struct spu_priv2 __iomem *priv2 = spu->priv2; 418 419 /* Save, Step 24: 420 * Save the MFC_CSR_ATO register in 421 * the CSA. 422 */ 423 csa->priv2.spu_atomic_status_RW = in_be64(&priv2->spu_atomic_status_RW); 424 } 425 426 static inline void save_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 427 { 428 /* Save, Step 25: 429 * Save the MFC_TCLASS_ID register in 430 * the CSA. 431 */ 432 csa->priv1.mfc_tclass_id_RW = spu_mfc_tclass_id_get(spu); 433 } 434 435 static inline void set_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 436 { 437 /* Save, Step 26: 438 * Restore, Step 23. 439 * Write the MFC_TCLASS_ID register with 440 * the value 0x10000000. 441 */ 442 spu_mfc_tclass_id_set(spu, 0x10000000); 443 eieio(); 444 } 445 446 static inline void purge_mfc_queue(struct spu_state *csa, struct spu *spu) 447 { 448 struct spu_priv2 __iomem *priv2 = spu->priv2; 449 450 /* Save, Step 27: 451 * Restore, Step 14. 452 * Write MFC_CNTL[Pc]=1 (purge queue). 453 */ 454 out_be64(&priv2->mfc_control_RW, MFC_CNTL_PURGE_DMA_REQUEST); 455 eieio(); 456 } 457 458 static inline void wait_purge_complete(struct spu_state *csa, struct spu *spu) 459 { 460 struct spu_priv2 __iomem *priv2 = spu->priv2; 461 462 /* Save, Step 28: 463 * Poll MFC_CNTL[Ps] until value '11' is read 464 * (purge complete). 465 */ 466 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 467 MFC_CNTL_PURGE_DMA_STATUS_MASK) == 468 MFC_CNTL_PURGE_DMA_COMPLETE); 469 } 470 471 static inline void setup_mfc_sr1(struct spu_state *csa, struct spu *spu) 472 { 473 /* Save, Step 30: 474 * Restore, Step 18: 475 * Write MFC_SR1 with MFC_SR1[D=0,S=1] and 476 * MFC_SR1[TL,R,Pr,T] set correctly for the 477 * OS specific environment. 478 * 479 * Implementation note: The SPU-side code 480 * for save/restore is privileged, so the 481 * MFC_SR1[Pr] bit is not set. 482 * 483 */ 484 spu_mfc_sr1_set(spu, (MFC_STATE1_MASTER_RUN_CONTROL_MASK | 485 MFC_STATE1_RELOCATE_MASK | 486 MFC_STATE1_BUS_TLBIE_MASK)); 487 } 488 489 static inline void save_spu_npc(struct spu_state *csa, struct spu *spu) 490 { 491 struct spu_problem __iomem *prob = spu->problem; 492 493 /* Save, Step 31: 494 * Save SPU_NPC in the CSA. 495 */ 496 csa->prob.spu_npc_RW = in_be32(&prob->spu_npc_RW); 497 } 498 499 static inline void save_spu_privcntl(struct spu_state *csa, struct spu *spu) 500 { 501 struct spu_priv2 __iomem *priv2 = spu->priv2; 502 503 /* Save, Step 32: 504 * Save SPU_PrivCntl in the CSA. 505 */ 506 csa->priv2.spu_privcntl_RW = in_be64(&priv2->spu_privcntl_RW); 507 } 508 509 static inline void reset_spu_privcntl(struct spu_state *csa, struct spu *spu) 510 { 511 struct spu_priv2 __iomem *priv2 = spu->priv2; 512 513 /* Save, Step 33: 514 * Restore, Step 16: 515 * Write SPU_PrivCntl[S,Le,A] fields reset to 0. 516 */ 517 out_be64(&priv2->spu_privcntl_RW, 0UL); 518 eieio(); 519 } 520 521 static inline void save_spu_lslr(struct spu_state *csa, struct spu *spu) 522 { 523 struct spu_priv2 __iomem *priv2 = spu->priv2; 524 525 /* Save, Step 34: 526 * Save SPU_LSLR in the CSA. 527 */ 528 csa->priv2.spu_lslr_RW = in_be64(&priv2->spu_lslr_RW); 529 } 530 531 static inline void reset_spu_lslr(struct spu_state *csa, struct spu *spu) 532 { 533 struct spu_priv2 __iomem *priv2 = spu->priv2; 534 535 /* Save, Step 35: 536 * Restore, Step 17. 537 * Reset SPU_LSLR. 538 */ 539 out_be64(&priv2->spu_lslr_RW, LS_ADDR_MASK); 540 eieio(); 541 } 542 543 static inline void save_spu_cfg(struct spu_state *csa, struct spu *spu) 544 { 545 struct spu_priv2 __iomem *priv2 = spu->priv2; 546 547 /* Save, Step 36: 548 * Save SPU_Cfg in the CSA. 549 */ 550 csa->priv2.spu_cfg_RW = in_be64(&priv2->spu_cfg_RW); 551 } 552 553 static inline void save_pm_trace(struct spu_state *csa, struct spu *spu) 554 { 555 /* Save, Step 37: 556 * Save PM_Trace_Tag_Wait_Mask in the CSA. 557 * Not performed by this implementation. 558 */ 559 } 560 561 static inline void save_mfc_rag(struct spu_state *csa, struct spu *spu) 562 { 563 /* Save, Step 38: 564 * Save RA_GROUP_ID register and the 565 * RA_ENABLE reigster in the CSA. 566 */ 567 csa->priv1.resource_allocation_groupID_RW = 568 spu_resource_allocation_groupID_get(spu); 569 csa->priv1.resource_allocation_enable_RW = 570 spu_resource_allocation_enable_get(spu); 571 } 572 573 static inline void save_ppu_mb_stat(struct spu_state *csa, struct spu *spu) 574 { 575 struct spu_problem __iomem *prob = spu->problem; 576 577 /* Save, Step 39: 578 * Save MB_Stat register in the CSA. 579 */ 580 csa->prob.mb_stat_R = in_be32(&prob->mb_stat_R); 581 } 582 583 static inline void save_ppu_mb(struct spu_state *csa, struct spu *spu) 584 { 585 struct spu_problem __iomem *prob = spu->problem; 586 587 /* Save, Step 40: 588 * Save the PPU_MB register in the CSA. 589 */ 590 csa->prob.pu_mb_R = in_be32(&prob->pu_mb_R); 591 } 592 593 static inline void save_ppuint_mb(struct spu_state *csa, struct spu *spu) 594 { 595 struct spu_priv2 __iomem *priv2 = spu->priv2; 596 597 /* Save, Step 41: 598 * Save the PPUINT_MB register in the CSA. 599 */ 600 csa->priv2.puint_mb_R = in_be64(&priv2->puint_mb_R); 601 } 602 603 static inline void save_ch_part1(struct spu_state *csa, struct spu *spu) 604 { 605 struct spu_priv2 __iomem *priv2 = spu->priv2; 606 u64 idx, ch_indices[7] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 607 int i; 608 609 /* Save, Step 42: 610 */ 611 612 /* Save CH 1, without channel count */ 613 out_be64(&priv2->spu_chnlcntptr_RW, 1); 614 csa->spu_chnldata_RW[1] = in_be64(&priv2->spu_chnldata_RW); 615 616 /* Save the following CH: [0,3,4,24,25,27] */ 617 for (i = 0; i < 7; i++) { 618 idx = ch_indices[i]; 619 out_be64(&priv2->spu_chnlcntptr_RW, idx); 620 eieio(); 621 csa->spu_chnldata_RW[idx] = in_be64(&priv2->spu_chnldata_RW); 622 csa->spu_chnlcnt_RW[idx] = in_be64(&priv2->spu_chnlcnt_RW); 623 out_be64(&priv2->spu_chnldata_RW, 0UL); 624 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 625 eieio(); 626 } 627 } 628 629 static inline void save_spu_mb(struct spu_state *csa, struct spu *spu) 630 { 631 struct spu_priv2 __iomem *priv2 = spu->priv2; 632 int i; 633 634 /* Save, Step 43: 635 * Save SPU Read Mailbox Channel. 636 */ 637 out_be64(&priv2->spu_chnlcntptr_RW, 29UL); 638 eieio(); 639 csa->spu_chnlcnt_RW[29] = in_be64(&priv2->spu_chnlcnt_RW); 640 for (i = 0; i < 4; i++) { 641 csa->spu_mailbox_data[i] = in_be64(&priv2->spu_chnldata_RW); 642 } 643 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 644 eieio(); 645 } 646 647 static inline void save_mfc_cmd(struct spu_state *csa, struct spu *spu) 648 { 649 struct spu_priv2 __iomem *priv2 = spu->priv2; 650 651 /* Save, Step 44: 652 * Save MFC_CMD Channel. 653 */ 654 out_be64(&priv2->spu_chnlcntptr_RW, 21UL); 655 eieio(); 656 csa->spu_chnlcnt_RW[21] = in_be64(&priv2->spu_chnlcnt_RW); 657 eieio(); 658 } 659 660 static inline void reset_ch(struct spu_state *csa, struct spu *spu) 661 { 662 struct spu_priv2 __iomem *priv2 = spu->priv2; 663 u64 ch_indices[4] = { 21UL, 23UL, 28UL, 30UL }; 664 u64 ch_counts[4] = { 16UL, 1UL, 1UL, 1UL }; 665 u64 idx; 666 int i; 667 668 /* Save, Step 45: 669 * Reset the following CH: [21, 23, 28, 30] 670 */ 671 for (i = 0; i < 4; i++) { 672 idx = ch_indices[i]; 673 out_be64(&priv2->spu_chnlcntptr_RW, idx); 674 eieio(); 675 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 676 eieio(); 677 } 678 } 679 680 static inline void resume_mfc_queue(struct spu_state *csa, struct spu *spu) 681 { 682 struct spu_priv2 __iomem *priv2 = spu->priv2; 683 684 /* Save, Step 46: 685 * Restore, Step 25. 686 * Write MFC_CNTL[Sc]=0 (resume queue processing). 687 */ 688 out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESUME_DMA_QUEUE); 689 } 690 691 static inline void get_kernel_slb(u64 ea, u64 slb[2]) 692 { 693 u64 llp; 694 695 if (REGION_ID(ea) == KERNEL_REGION_ID) 696 llp = mmu_psize_defs[mmu_linear_psize].sllp; 697 else 698 llp = mmu_psize_defs[mmu_virtual_psize].sllp; 699 slb[0] = (get_kernel_vsid(ea) << SLB_VSID_SHIFT) | 700 SLB_VSID_KERNEL | llp; 701 slb[1] = (ea & ESID_MASK) | SLB_ESID_V; 702 } 703 704 static inline void load_mfc_slb(struct spu *spu, u64 slb[2], int slbe) 705 { 706 struct spu_priv2 __iomem *priv2 = spu->priv2; 707 708 out_be64(&priv2->slb_index_W, slbe); 709 eieio(); 710 out_be64(&priv2->slb_vsid_RW, slb[0]); 711 out_be64(&priv2->slb_esid_RW, slb[1]); 712 eieio(); 713 } 714 715 static inline void setup_mfc_slbs(struct spu_state *csa, struct spu *spu) 716 { 717 u64 code_slb[2]; 718 u64 lscsa_slb[2]; 719 720 /* Save, Step 47: 721 * Restore, Step 30. 722 * If MFC_SR1[R]=1, write 0 to SLB_Invalidate_All 723 * register, then initialize SLB_VSID and SLB_ESID 724 * to provide access to SPU context save code and 725 * LSCSA. 726 * 727 * This implementation places both the context 728 * switch code and LSCSA in kernel address space. 729 * 730 * Further this implementation assumes that the 731 * MFC_SR1[R]=1 (in other words, assume that 732 * translation is desired by OS environment). 733 */ 734 spu_invalidate_slbs(spu); 735 get_kernel_slb((unsigned long)&spu_save_code[0], code_slb); 736 get_kernel_slb((unsigned long)csa->lscsa, lscsa_slb); 737 load_mfc_slb(spu, code_slb, 0); 738 if ((lscsa_slb[0] != code_slb[0]) || (lscsa_slb[1] != code_slb[1])) 739 load_mfc_slb(spu, lscsa_slb, 1); 740 } 741 742 static inline void set_switch_active(struct spu_state *csa, struct spu *spu) 743 { 744 /* Save, Step 48: 745 * Restore, Step 23. 746 * Change the software context switch pending flag 747 * to context switch active. 748 */ 749 set_bit(SPU_CONTEXT_SWITCH_ACTIVE, &spu->flags); 750 clear_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags); 751 mb(); 752 } 753 754 static inline void enable_interrupts(struct spu_state *csa, struct spu *spu) 755 { 756 unsigned long class1_mask = CLASS1_ENABLE_SEGMENT_FAULT_INTR | 757 CLASS1_ENABLE_STORAGE_FAULT_INTR; 758 759 /* Save, Step 49: 760 * Restore, Step 22: 761 * Reset and then enable interrupts, as 762 * needed by OS. 763 * 764 * This implementation enables only class1 765 * (translation) interrupts. 766 */ 767 spin_lock_irq(&spu->register_lock); 768 spu_int_stat_clear(spu, 0, ~0ul); 769 spu_int_stat_clear(spu, 1, ~0ul); 770 spu_int_stat_clear(spu, 2, ~0ul); 771 spu_int_mask_set(spu, 0, 0ul); 772 spu_int_mask_set(spu, 1, class1_mask); 773 spu_int_mask_set(spu, 2, 0ul); 774 spin_unlock_irq(&spu->register_lock); 775 } 776 777 static inline int send_mfc_dma(struct spu *spu, unsigned long ea, 778 unsigned int ls_offset, unsigned int size, 779 unsigned int tag, unsigned int rclass, 780 unsigned int cmd) 781 { 782 struct spu_problem __iomem *prob = spu->problem; 783 union mfc_tag_size_class_cmd command; 784 unsigned int transfer_size; 785 volatile unsigned int status = 0x0; 786 787 while (size > 0) { 788 transfer_size = 789 (size > MFC_MAX_DMA_SIZE) ? MFC_MAX_DMA_SIZE : size; 790 command.u.mfc_size = transfer_size; 791 command.u.mfc_tag = tag; 792 command.u.mfc_rclassid = rclass; 793 command.u.mfc_cmd = cmd; 794 do { 795 out_be32(&prob->mfc_lsa_W, ls_offset); 796 out_be64(&prob->mfc_ea_W, ea); 797 out_be64(&prob->mfc_union_W.all64, command.all64); 798 status = 799 in_be32(&prob->mfc_union_W.by32.mfc_class_cmd32); 800 if (unlikely(status & 0x2)) { 801 cpu_relax(); 802 } 803 } while (status & 0x3); 804 size -= transfer_size; 805 ea += transfer_size; 806 ls_offset += transfer_size; 807 } 808 return 0; 809 } 810 811 static inline void save_ls_16kb(struct spu_state *csa, struct spu *spu) 812 { 813 unsigned long addr = (unsigned long)&csa->lscsa->ls[0]; 814 unsigned int ls_offset = 0x0; 815 unsigned int size = 16384; 816 unsigned int tag = 0; 817 unsigned int rclass = 0; 818 unsigned int cmd = MFC_PUT_CMD; 819 820 /* Save, Step 50: 821 * Issue a DMA command to copy the first 16K bytes 822 * of local storage to the CSA. 823 */ 824 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 825 } 826 827 static inline void set_spu_npc(struct spu_state *csa, struct spu *spu) 828 { 829 struct spu_problem __iomem *prob = spu->problem; 830 831 /* Save, Step 51: 832 * Restore, Step 31. 833 * Write SPU_NPC[IE]=0 and SPU_NPC[LSA] to entry 834 * point address of context save code in local 835 * storage. 836 * 837 * This implementation uses SPU-side save/restore 838 * programs with entry points at LSA of 0. 839 */ 840 out_be32(&prob->spu_npc_RW, 0); 841 eieio(); 842 } 843 844 static inline void set_signot1(struct spu_state *csa, struct spu *spu) 845 { 846 struct spu_problem __iomem *prob = spu->problem; 847 union { 848 u64 ull; 849 u32 ui[2]; 850 } addr64; 851 852 /* Save, Step 52: 853 * Restore, Step 32: 854 * Write SPU_Sig_Notify_1 register with upper 32-bits 855 * of the CSA.LSCSA effective address. 856 */ 857 addr64.ull = (u64) csa->lscsa; 858 out_be32(&prob->signal_notify1, addr64.ui[0]); 859 eieio(); 860 } 861 862 static inline void set_signot2(struct spu_state *csa, struct spu *spu) 863 { 864 struct spu_problem __iomem *prob = spu->problem; 865 union { 866 u64 ull; 867 u32 ui[2]; 868 } addr64; 869 870 /* Save, Step 53: 871 * Restore, Step 33: 872 * Write SPU_Sig_Notify_2 register with lower 32-bits 873 * of the CSA.LSCSA effective address. 874 */ 875 addr64.ull = (u64) csa->lscsa; 876 out_be32(&prob->signal_notify2, addr64.ui[1]); 877 eieio(); 878 } 879 880 static inline void send_save_code(struct spu_state *csa, struct spu *spu) 881 { 882 unsigned long addr = (unsigned long)&spu_save_code[0]; 883 unsigned int ls_offset = 0x0; 884 unsigned int size = sizeof(spu_save_code); 885 unsigned int tag = 0; 886 unsigned int rclass = 0; 887 unsigned int cmd = MFC_GETFS_CMD; 888 889 /* Save, Step 54: 890 * Issue a DMA command to copy context save code 891 * to local storage and start SPU. 892 */ 893 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 894 } 895 896 static inline void set_ppu_querymask(struct spu_state *csa, struct spu *spu) 897 { 898 struct spu_problem __iomem *prob = spu->problem; 899 900 /* Save, Step 55: 901 * Restore, Step 38. 902 * Write PPU_QueryMask=1 (enable Tag Group 0) 903 * and issue eieio instruction. 904 */ 905 out_be32(&prob->dma_querymask_RW, MFC_TAGID_TO_TAGMASK(0)); 906 eieio(); 907 } 908 909 static inline void wait_tag_complete(struct spu_state *csa, struct spu *spu) 910 { 911 struct spu_problem __iomem *prob = spu->problem; 912 u32 mask = MFC_TAGID_TO_TAGMASK(0); 913 unsigned long flags; 914 915 /* Save, Step 56: 916 * Restore, Step 39. 917 * Restore, Step 39. 918 * Restore, Step 46. 919 * Poll PPU_TagStatus[gn] until 01 (Tag group 0 complete) 920 * or write PPU_QueryType[TS]=01 and wait for Tag Group 921 * Complete Interrupt. Write INT_Stat_Class0 or 922 * INT_Stat_Class2 with value of 'handled'. 923 */ 924 POLL_WHILE_FALSE(in_be32(&prob->dma_tagstatus_R) & mask); 925 926 local_irq_save(flags); 927 spu_int_stat_clear(spu, 0, ~(0ul)); 928 spu_int_stat_clear(spu, 2, ~(0ul)); 929 local_irq_restore(flags); 930 } 931 932 static inline void wait_spu_stopped(struct spu_state *csa, struct spu *spu) 933 { 934 struct spu_problem __iomem *prob = spu->problem; 935 unsigned long flags; 936 937 /* Save, Step 57: 938 * Restore, Step 40. 939 * Poll until SPU_Status[R]=0 or wait for SPU Class 0 940 * or SPU Class 2 interrupt. Write INT_Stat_class0 941 * or INT_Stat_class2 with value of handled. 942 */ 943 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING); 944 945 local_irq_save(flags); 946 spu_int_stat_clear(spu, 0, ~(0ul)); 947 spu_int_stat_clear(spu, 2, ~(0ul)); 948 local_irq_restore(flags); 949 } 950 951 static inline int check_save_status(struct spu_state *csa, struct spu *spu) 952 { 953 struct spu_problem __iomem *prob = spu->problem; 954 u32 complete; 955 956 /* Save, Step 54: 957 * If SPU_Status[P]=1 and SPU_Status[SC] = "success", 958 * context save succeeded, otherwise context save 959 * failed. 960 */ 961 complete = ((SPU_SAVE_COMPLETE << SPU_STOP_STATUS_SHIFT) | 962 SPU_STATUS_STOPPED_BY_STOP); 963 return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0; 964 } 965 966 static inline void terminate_spu_app(struct spu_state *csa, struct spu *spu) 967 { 968 /* Restore, Step 4: 969 * If required, notify the "using application" that 970 * the SPU task has been terminated. TBD. 971 */ 972 } 973 974 static inline void suspend_mfc(struct spu_state *csa, struct spu *spu) 975 { 976 struct spu_priv2 __iomem *priv2 = spu->priv2; 977 978 /* Restore, Step 7: 979 * Restore, Step 47. 980 * Write MFC_Cntl[Dh,Sc]='1','1' to suspend 981 * the queue and halt the decrementer. 982 */ 983 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE | 984 MFC_CNTL_DECREMENTER_HALTED); 985 eieio(); 986 } 987 988 static inline void wait_suspend_mfc_complete(struct spu_state *csa, 989 struct spu *spu) 990 { 991 struct spu_priv2 __iomem *priv2 = spu->priv2; 992 993 /* Restore, Step 8: 994 * Restore, Step 47. 995 * Poll MFC_CNTL[Ss] until 11 is returned. 996 */ 997 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 998 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) == 999 MFC_CNTL_SUSPEND_COMPLETE); 1000 } 1001 1002 static inline int suspend_spe(struct spu_state *csa, struct spu *spu) 1003 { 1004 struct spu_problem __iomem *prob = spu->problem; 1005 1006 /* Restore, Step 9: 1007 * If SPU_Status[R]=1, stop SPU execution 1008 * and wait for stop to complete. 1009 * 1010 * Returns 1 if SPU_Status[R]=1 on entry. 1011 * 0 otherwise 1012 */ 1013 if (in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) { 1014 if (in_be32(&prob->spu_status_R) & 1015 SPU_STATUS_ISOLATED_EXIT_STATUS) { 1016 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1017 SPU_STATUS_RUNNING); 1018 } 1019 if ((in_be32(&prob->spu_status_R) & 1020 SPU_STATUS_ISOLATED_LOAD_STATUS) 1021 || (in_be32(&prob->spu_status_R) & 1022 SPU_STATUS_ISOLATED_STATE)) { 1023 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1024 eieio(); 1025 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1026 SPU_STATUS_RUNNING); 1027 out_be32(&prob->spu_runcntl_RW, 0x2); 1028 eieio(); 1029 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1030 SPU_STATUS_RUNNING); 1031 } 1032 if (in_be32(&prob->spu_status_R) & 1033 SPU_STATUS_WAITING_FOR_CHANNEL) { 1034 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1035 eieio(); 1036 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1037 SPU_STATUS_RUNNING); 1038 } 1039 return 1; 1040 } 1041 return 0; 1042 } 1043 1044 static inline void clear_spu_status(struct spu_state *csa, struct spu *spu) 1045 { 1046 struct spu_problem __iomem *prob = spu->problem; 1047 1048 /* Restore, Step 10: 1049 * If SPU_Status[R]=0 and SPU_Status[E,L,IS]=1, 1050 * release SPU from isolate state. 1051 */ 1052 if (!(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING)) { 1053 if (in_be32(&prob->spu_status_R) & 1054 SPU_STATUS_ISOLATED_EXIT_STATUS) { 1055 spu_mfc_sr1_set(spu, 1056 MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1057 eieio(); 1058 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1059 eieio(); 1060 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1061 SPU_STATUS_RUNNING); 1062 } 1063 if ((in_be32(&prob->spu_status_R) & 1064 SPU_STATUS_ISOLATED_LOAD_STATUS) 1065 || (in_be32(&prob->spu_status_R) & 1066 SPU_STATUS_ISOLATED_STATE)) { 1067 spu_mfc_sr1_set(spu, 1068 MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1069 eieio(); 1070 out_be32(&prob->spu_runcntl_RW, 0x2); 1071 eieio(); 1072 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1073 SPU_STATUS_RUNNING); 1074 } 1075 } 1076 } 1077 1078 static inline void reset_ch_part1(struct spu_state *csa, struct spu *spu) 1079 { 1080 struct spu_priv2 __iomem *priv2 = spu->priv2; 1081 u64 ch_indices[7] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 1082 u64 idx; 1083 int i; 1084 1085 /* Restore, Step 20: 1086 */ 1087 1088 /* Reset CH 1 */ 1089 out_be64(&priv2->spu_chnlcntptr_RW, 1); 1090 out_be64(&priv2->spu_chnldata_RW, 0UL); 1091 1092 /* Reset the following CH: [0,3,4,24,25,27] */ 1093 for (i = 0; i < 7; i++) { 1094 idx = ch_indices[i]; 1095 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1096 eieio(); 1097 out_be64(&priv2->spu_chnldata_RW, 0UL); 1098 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 1099 eieio(); 1100 } 1101 } 1102 1103 static inline void reset_ch_part2(struct spu_state *csa, struct spu *spu) 1104 { 1105 struct spu_priv2 __iomem *priv2 = spu->priv2; 1106 u64 ch_indices[5] = { 21UL, 23UL, 28UL, 29UL, 30UL }; 1107 u64 ch_counts[5] = { 16UL, 1UL, 1UL, 0UL, 1UL }; 1108 u64 idx; 1109 int i; 1110 1111 /* Restore, Step 21: 1112 * Reset the following CH: [21, 23, 28, 29, 30] 1113 */ 1114 for (i = 0; i < 5; i++) { 1115 idx = ch_indices[i]; 1116 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1117 eieio(); 1118 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 1119 eieio(); 1120 } 1121 } 1122 1123 static inline void setup_spu_status_part1(struct spu_state *csa, 1124 struct spu *spu) 1125 { 1126 u32 status_P = SPU_STATUS_STOPPED_BY_STOP; 1127 u32 status_I = SPU_STATUS_INVALID_INSTR; 1128 u32 status_H = SPU_STATUS_STOPPED_BY_HALT; 1129 u32 status_S = SPU_STATUS_SINGLE_STEP; 1130 u32 status_S_I = SPU_STATUS_SINGLE_STEP | SPU_STATUS_INVALID_INSTR; 1131 u32 status_S_P = SPU_STATUS_SINGLE_STEP | SPU_STATUS_STOPPED_BY_STOP; 1132 u32 status_P_H = SPU_STATUS_STOPPED_BY_HALT |SPU_STATUS_STOPPED_BY_STOP; 1133 u32 status_P_I = SPU_STATUS_STOPPED_BY_STOP |SPU_STATUS_INVALID_INSTR; 1134 u32 status_code; 1135 1136 /* Restore, Step 27: 1137 * If the CSA.SPU_Status[I,S,H,P]=1 then add the correct 1138 * instruction sequence to the end of the SPU based restore 1139 * code (after the "context restored" stop and signal) to 1140 * restore the correct SPU status. 1141 * 1142 * NOTE: Rather than modifying the SPU executable, we 1143 * instead add a new 'stopped_status' field to the 1144 * LSCSA. The SPU-side restore reads this field and 1145 * takes the appropriate action when exiting. 1146 */ 1147 1148 status_code = 1149 (csa->prob.spu_status_R >> SPU_STOP_STATUS_SHIFT) & 0xFFFF; 1150 if ((csa->prob.spu_status_R & status_P_I) == status_P_I) { 1151 1152 /* SPU_Status[P,I]=1 - Illegal Instruction followed 1153 * by Stop and Signal instruction, followed by 'br -4'. 1154 * 1155 */ 1156 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_I; 1157 csa->lscsa->stopped_status.slot[1] = status_code; 1158 1159 } else if ((csa->prob.spu_status_R & status_P_H) == status_P_H) { 1160 1161 /* SPU_Status[P,H]=1 - Halt Conditional, followed 1162 * by Stop and Signal instruction, followed by 1163 * 'br -4'. 1164 */ 1165 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_H; 1166 csa->lscsa->stopped_status.slot[1] = status_code; 1167 1168 } else if ((csa->prob.spu_status_R & status_S_P) == status_S_P) { 1169 1170 /* SPU_Status[S,P]=1 - Stop and Signal instruction 1171 * followed by 'br -4'. 1172 */ 1173 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_P; 1174 csa->lscsa->stopped_status.slot[1] = status_code; 1175 1176 } else if ((csa->prob.spu_status_R & status_S_I) == status_S_I) { 1177 1178 /* SPU_Status[S,I]=1 - Illegal instruction followed 1179 * by 'br -4'. 1180 */ 1181 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_I; 1182 csa->lscsa->stopped_status.slot[1] = status_code; 1183 1184 } else if ((csa->prob.spu_status_R & status_P) == status_P) { 1185 1186 /* SPU_Status[P]=1 - Stop and Signal instruction 1187 * followed by 'br -4'. 1188 */ 1189 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P; 1190 csa->lscsa->stopped_status.slot[1] = status_code; 1191 1192 } else if ((csa->prob.spu_status_R & status_H) == status_H) { 1193 1194 /* SPU_Status[H]=1 - Halt Conditional, followed 1195 * by 'br -4'. 1196 */ 1197 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_H; 1198 1199 } else if ((csa->prob.spu_status_R & status_S) == status_S) { 1200 1201 /* SPU_Status[S]=1 - Two nop instructions. 1202 */ 1203 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S; 1204 1205 } else if ((csa->prob.spu_status_R & status_I) == status_I) { 1206 1207 /* SPU_Status[I]=1 - Illegal instruction followed 1208 * by 'br -4'. 1209 */ 1210 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_I; 1211 1212 } 1213 } 1214 1215 static inline void setup_spu_status_part2(struct spu_state *csa, 1216 struct spu *spu) 1217 { 1218 u32 mask; 1219 1220 /* Restore, Step 28: 1221 * If the CSA.SPU_Status[I,S,H,P,R]=0 then 1222 * add a 'br *' instruction to the end of 1223 * the SPU based restore code. 1224 * 1225 * NOTE: Rather than modifying the SPU executable, we 1226 * instead add a new 'stopped_status' field to the 1227 * LSCSA. The SPU-side restore reads this field and 1228 * takes the appropriate action when exiting. 1229 */ 1230 mask = SPU_STATUS_INVALID_INSTR | 1231 SPU_STATUS_SINGLE_STEP | 1232 SPU_STATUS_STOPPED_BY_HALT | 1233 SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING; 1234 if (!(csa->prob.spu_status_R & mask)) { 1235 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_R; 1236 } 1237 } 1238 1239 static inline void restore_mfc_rag(struct spu_state *csa, struct spu *spu) 1240 { 1241 /* Restore, Step 29: 1242 * Restore RA_GROUP_ID register and the 1243 * RA_ENABLE reigster from the CSA. 1244 */ 1245 spu_resource_allocation_groupID_set(spu, 1246 csa->priv1.resource_allocation_groupID_RW); 1247 spu_resource_allocation_enable_set(spu, 1248 csa->priv1.resource_allocation_enable_RW); 1249 } 1250 1251 static inline void send_restore_code(struct spu_state *csa, struct spu *spu) 1252 { 1253 unsigned long addr = (unsigned long)&spu_restore_code[0]; 1254 unsigned int ls_offset = 0x0; 1255 unsigned int size = sizeof(spu_restore_code); 1256 unsigned int tag = 0; 1257 unsigned int rclass = 0; 1258 unsigned int cmd = MFC_GETFS_CMD; 1259 1260 /* Restore, Step 37: 1261 * Issue MFC DMA command to copy context 1262 * restore code to local storage. 1263 */ 1264 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 1265 } 1266 1267 static inline void setup_decr(struct spu_state *csa, struct spu *spu) 1268 { 1269 /* Restore, Step 34: 1270 * If CSA.MFC_CNTL[Ds]=1 (decrementer was 1271 * running) then adjust decrementer, set 1272 * decrementer running status in LSCSA, 1273 * and set decrementer "wrapped" status 1274 * in LSCSA. 1275 */ 1276 if (csa->priv2.mfc_control_RW & MFC_CNTL_DECREMENTER_RUNNING) { 1277 cycles_t resume_time = get_cycles(); 1278 cycles_t delta_time = resume_time - csa->suspend_time; 1279 1280 csa->lscsa->decr.slot[0] -= delta_time; 1281 } 1282 } 1283 1284 static inline void setup_ppu_mb(struct spu_state *csa, struct spu *spu) 1285 { 1286 /* Restore, Step 35: 1287 * Copy the CSA.PU_MB data into the LSCSA. 1288 */ 1289 csa->lscsa->ppu_mb.slot[0] = csa->prob.pu_mb_R; 1290 } 1291 1292 static inline void setup_ppuint_mb(struct spu_state *csa, struct spu *spu) 1293 { 1294 /* Restore, Step 36: 1295 * Copy the CSA.PUINT_MB data into the LSCSA. 1296 */ 1297 csa->lscsa->ppuint_mb.slot[0] = csa->priv2.puint_mb_R; 1298 } 1299 1300 static inline int check_restore_status(struct spu_state *csa, struct spu *spu) 1301 { 1302 struct spu_problem __iomem *prob = spu->problem; 1303 u32 complete; 1304 1305 /* Restore, Step 40: 1306 * If SPU_Status[P]=1 and SPU_Status[SC] = "success", 1307 * context restore succeeded, otherwise context restore 1308 * failed. 1309 */ 1310 complete = ((SPU_RESTORE_COMPLETE << SPU_STOP_STATUS_SHIFT) | 1311 SPU_STATUS_STOPPED_BY_STOP); 1312 return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0; 1313 } 1314 1315 static inline void restore_spu_privcntl(struct spu_state *csa, struct spu *spu) 1316 { 1317 struct spu_priv2 __iomem *priv2 = spu->priv2; 1318 1319 /* Restore, Step 41: 1320 * Restore SPU_PrivCntl from the CSA. 1321 */ 1322 out_be64(&priv2->spu_privcntl_RW, csa->priv2.spu_privcntl_RW); 1323 eieio(); 1324 } 1325 1326 static inline void restore_status_part1(struct spu_state *csa, struct spu *spu) 1327 { 1328 struct spu_problem __iomem *prob = spu->problem; 1329 u32 mask; 1330 1331 /* Restore, Step 42: 1332 * If any CSA.SPU_Status[I,S,H,P]=1, then 1333 * restore the error or single step state. 1334 */ 1335 mask = SPU_STATUS_INVALID_INSTR | 1336 SPU_STATUS_SINGLE_STEP | 1337 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP; 1338 if (csa->prob.spu_status_R & mask) { 1339 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1340 eieio(); 1341 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1342 SPU_STATUS_RUNNING); 1343 } 1344 } 1345 1346 static inline void restore_status_part2(struct spu_state *csa, struct spu *spu) 1347 { 1348 struct spu_problem __iomem *prob = spu->problem; 1349 u32 mask; 1350 1351 /* Restore, Step 43: 1352 * If all CSA.SPU_Status[I,S,H,P,R]=0 then write 1353 * SPU_RunCntl[R0R1]='01', wait for SPU_Status[R]=1, 1354 * then write '00' to SPU_RunCntl[R0R1] and wait 1355 * for SPU_Status[R]=0. 1356 */ 1357 mask = SPU_STATUS_INVALID_INSTR | 1358 SPU_STATUS_SINGLE_STEP | 1359 SPU_STATUS_STOPPED_BY_HALT | 1360 SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING; 1361 if (!(csa->prob.spu_status_R & mask)) { 1362 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1363 eieio(); 1364 POLL_WHILE_FALSE(in_be32(&prob->spu_status_R) & 1365 SPU_STATUS_RUNNING); 1366 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1367 eieio(); 1368 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1369 SPU_STATUS_RUNNING); 1370 } 1371 } 1372 1373 static inline void restore_ls_16kb(struct spu_state *csa, struct spu *spu) 1374 { 1375 unsigned long addr = (unsigned long)&csa->lscsa->ls[0]; 1376 unsigned int ls_offset = 0x0; 1377 unsigned int size = 16384; 1378 unsigned int tag = 0; 1379 unsigned int rclass = 0; 1380 unsigned int cmd = MFC_GET_CMD; 1381 1382 /* Restore, Step 44: 1383 * Issue a DMA command to restore the first 1384 * 16kb of local storage from CSA. 1385 */ 1386 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 1387 } 1388 1389 static inline void clear_interrupts(struct spu_state *csa, struct spu *spu) 1390 { 1391 /* Restore, Step 49: 1392 * Write INT_MASK_class0 with value of 0. 1393 * Write INT_MASK_class1 with value of 0. 1394 * Write INT_MASK_class2 with value of 0. 1395 * Write INT_STAT_class0 with value of -1. 1396 * Write INT_STAT_class1 with value of -1. 1397 * Write INT_STAT_class2 with value of -1. 1398 */ 1399 spin_lock_irq(&spu->register_lock); 1400 spu_int_mask_set(spu, 0, 0ul); 1401 spu_int_mask_set(spu, 1, 0ul); 1402 spu_int_mask_set(spu, 2, 0ul); 1403 spu_int_stat_clear(spu, 0, ~0ul); 1404 spu_int_stat_clear(spu, 1, ~0ul); 1405 spu_int_stat_clear(spu, 2, ~0ul); 1406 spin_unlock_irq(&spu->register_lock); 1407 } 1408 1409 static inline void restore_mfc_queues(struct spu_state *csa, struct spu *spu) 1410 { 1411 struct spu_priv2 __iomem *priv2 = spu->priv2; 1412 int i; 1413 1414 /* Restore, Step 50: 1415 * If MFC_Cntl[Se]!=0 then restore 1416 * MFC command queues. 1417 */ 1418 if ((csa->priv2.mfc_control_RW & MFC_CNTL_DMA_QUEUES_EMPTY_MASK) == 0) { 1419 for (i = 0; i < 8; i++) { 1420 out_be64(&priv2->puq[i].mfc_cq_data0_RW, 1421 csa->priv2.puq[i].mfc_cq_data0_RW); 1422 out_be64(&priv2->puq[i].mfc_cq_data1_RW, 1423 csa->priv2.puq[i].mfc_cq_data1_RW); 1424 out_be64(&priv2->puq[i].mfc_cq_data2_RW, 1425 csa->priv2.puq[i].mfc_cq_data2_RW); 1426 out_be64(&priv2->puq[i].mfc_cq_data3_RW, 1427 csa->priv2.puq[i].mfc_cq_data3_RW); 1428 } 1429 for (i = 0; i < 16; i++) { 1430 out_be64(&priv2->spuq[i].mfc_cq_data0_RW, 1431 csa->priv2.spuq[i].mfc_cq_data0_RW); 1432 out_be64(&priv2->spuq[i].mfc_cq_data1_RW, 1433 csa->priv2.spuq[i].mfc_cq_data1_RW); 1434 out_be64(&priv2->spuq[i].mfc_cq_data2_RW, 1435 csa->priv2.spuq[i].mfc_cq_data2_RW); 1436 out_be64(&priv2->spuq[i].mfc_cq_data3_RW, 1437 csa->priv2.spuq[i].mfc_cq_data3_RW); 1438 } 1439 } 1440 eieio(); 1441 } 1442 1443 static inline void restore_ppu_querymask(struct spu_state *csa, struct spu *spu) 1444 { 1445 struct spu_problem __iomem *prob = spu->problem; 1446 1447 /* Restore, Step 51: 1448 * Restore the PPU_QueryMask register from CSA. 1449 */ 1450 out_be32(&prob->dma_querymask_RW, csa->prob.dma_querymask_RW); 1451 eieio(); 1452 } 1453 1454 static inline void restore_ppu_querytype(struct spu_state *csa, struct spu *spu) 1455 { 1456 struct spu_problem __iomem *prob = spu->problem; 1457 1458 /* Restore, Step 52: 1459 * Restore the PPU_QueryType register from CSA. 1460 */ 1461 out_be32(&prob->dma_querytype_RW, csa->prob.dma_querytype_RW); 1462 eieio(); 1463 } 1464 1465 static inline void restore_mfc_csr_tsq(struct spu_state *csa, struct spu *spu) 1466 { 1467 struct spu_priv2 __iomem *priv2 = spu->priv2; 1468 1469 /* Restore, Step 53: 1470 * Restore the MFC_CSR_TSQ register from CSA. 1471 */ 1472 out_be64(&priv2->spu_tag_status_query_RW, 1473 csa->priv2.spu_tag_status_query_RW); 1474 eieio(); 1475 } 1476 1477 static inline void restore_mfc_csr_cmd(struct spu_state *csa, struct spu *spu) 1478 { 1479 struct spu_priv2 __iomem *priv2 = spu->priv2; 1480 1481 /* Restore, Step 54: 1482 * Restore the MFC_CSR_CMD1 and MFC_CSR_CMD2 1483 * registers from CSA. 1484 */ 1485 out_be64(&priv2->spu_cmd_buf1_RW, csa->priv2.spu_cmd_buf1_RW); 1486 out_be64(&priv2->spu_cmd_buf2_RW, csa->priv2.spu_cmd_buf2_RW); 1487 eieio(); 1488 } 1489 1490 static inline void restore_mfc_csr_ato(struct spu_state *csa, struct spu *spu) 1491 { 1492 struct spu_priv2 __iomem *priv2 = spu->priv2; 1493 1494 /* Restore, Step 55: 1495 * Restore the MFC_CSR_ATO register from CSA. 1496 */ 1497 out_be64(&priv2->spu_atomic_status_RW, csa->priv2.spu_atomic_status_RW); 1498 } 1499 1500 static inline void restore_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 1501 { 1502 /* Restore, Step 56: 1503 * Restore the MFC_TCLASS_ID register from CSA. 1504 */ 1505 spu_mfc_tclass_id_set(spu, csa->priv1.mfc_tclass_id_RW); 1506 eieio(); 1507 } 1508 1509 static inline void set_llr_event(struct spu_state *csa, struct spu *spu) 1510 { 1511 u64 ch0_cnt, ch0_data; 1512 u64 ch1_data; 1513 1514 /* Restore, Step 57: 1515 * Set the Lock Line Reservation Lost Event by: 1516 * 1. OR CSA.SPU_Event_Status with bit 21 (Lr) set to 1. 1517 * 2. If CSA.SPU_Channel_0_Count=0 and 1518 * CSA.SPU_Wr_Event_Mask[Lr]=1 and 1519 * CSA.SPU_Event_Status[Lr]=0 then set 1520 * CSA.SPU_Event_Status_Count=1. 1521 */ 1522 ch0_cnt = csa->spu_chnlcnt_RW[0]; 1523 ch0_data = csa->spu_chnldata_RW[0]; 1524 ch1_data = csa->spu_chnldata_RW[1]; 1525 csa->spu_chnldata_RW[0] |= MFC_LLR_LOST_EVENT; 1526 if ((ch0_cnt == 0) && !(ch0_data & MFC_LLR_LOST_EVENT) && 1527 (ch1_data & MFC_LLR_LOST_EVENT)) { 1528 csa->spu_chnlcnt_RW[0] = 1; 1529 } 1530 } 1531 1532 static inline void restore_decr_wrapped(struct spu_state *csa, struct spu *spu) 1533 { 1534 /* Restore, Step 58: 1535 * If the status of the CSA software decrementer 1536 * "wrapped" flag is set, OR in a '1' to 1537 * CSA.SPU_Event_Status[Tm]. 1538 */ 1539 if (csa->lscsa->decr_status.slot[0] == 1) { 1540 csa->spu_chnldata_RW[0] |= 0x20; 1541 } 1542 if ((csa->lscsa->decr_status.slot[0] == 1) && 1543 (csa->spu_chnlcnt_RW[0] == 0 && 1544 ((csa->spu_chnldata_RW[2] & 0x20) == 0x0) && 1545 ((csa->spu_chnldata_RW[0] & 0x20) != 0x1))) { 1546 csa->spu_chnlcnt_RW[0] = 1; 1547 } 1548 } 1549 1550 static inline void restore_ch_part1(struct spu_state *csa, struct spu *spu) 1551 { 1552 struct spu_priv2 __iomem *priv2 = spu->priv2; 1553 u64 idx, ch_indices[7] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 1554 int i; 1555 1556 /* Restore, Step 59: 1557 */ 1558 1559 /* Restore CH 1 without count */ 1560 out_be64(&priv2->spu_chnlcntptr_RW, 1); 1561 out_be64(&priv2->spu_chnldata_RW, csa->spu_chnldata_RW[1]); 1562 1563 /* Restore the following CH: [0,3,4,24,25,27] */ 1564 for (i = 0; i < 7; i++) { 1565 idx = ch_indices[i]; 1566 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1567 eieio(); 1568 out_be64(&priv2->spu_chnldata_RW, csa->spu_chnldata_RW[idx]); 1569 out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[idx]); 1570 eieio(); 1571 } 1572 } 1573 1574 static inline void restore_ch_part2(struct spu_state *csa, struct spu *spu) 1575 { 1576 struct spu_priv2 __iomem *priv2 = spu->priv2; 1577 u64 ch_indices[3] = { 9UL, 21UL, 23UL }; 1578 u64 ch_counts[3] = { 1UL, 16UL, 1UL }; 1579 u64 idx; 1580 int i; 1581 1582 /* Restore, Step 60: 1583 * Restore the following CH: [9,21,23]. 1584 */ 1585 ch_counts[0] = 1UL; 1586 ch_counts[1] = csa->spu_chnlcnt_RW[21]; 1587 ch_counts[2] = 1UL; 1588 for (i = 0; i < 3; i++) { 1589 idx = ch_indices[i]; 1590 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1591 eieio(); 1592 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 1593 eieio(); 1594 } 1595 } 1596 1597 static inline void restore_spu_lslr(struct spu_state *csa, struct spu *spu) 1598 { 1599 struct spu_priv2 __iomem *priv2 = spu->priv2; 1600 1601 /* Restore, Step 61: 1602 * Restore the SPU_LSLR register from CSA. 1603 */ 1604 out_be64(&priv2->spu_lslr_RW, csa->priv2.spu_lslr_RW); 1605 eieio(); 1606 } 1607 1608 static inline void restore_spu_cfg(struct spu_state *csa, struct spu *spu) 1609 { 1610 struct spu_priv2 __iomem *priv2 = spu->priv2; 1611 1612 /* Restore, Step 62: 1613 * Restore the SPU_Cfg register from CSA. 1614 */ 1615 out_be64(&priv2->spu_cfg_RW, csa->priv2.spu_cfg_RW); 1616 eieio(); 1617 } 1618 1619 static inline void restore_pm_trace(struct spu_state *csa, struct spu *spu) 1620 { 1621 /* Restore, Step 63: 1622 * Restore PM_Trace_Tag_Wait_Mask from CSA. 1623 * Not performed by this implementation. 1624 */ 1625 } 1626 1627 static inline void restore_spu_npc(struct spu_state *csa, struct spu *spu) 1628 { 1629 struct spu_problem __iomem *prob = spu->problem; 1630 1631 /* Restore, Step 64: 1632 * Restore SPU_NPC from CSA. 1633 */ 1634 out_be32(&prob->spu_npc_RW, csa->prob.spu_npc_RW); 1635 eieio(); 1636 } 1637 1638 static inline void restore_spu_mb(struct spu_state *csa, struct spu *spu) 1639 { 1640 struct spu_priv2 __iomem *priv2 = spu->priv2; 1641 int i; 1642 1643 /* Restore, Step 65: 1644 * Restore MFC_RdSPU_MB from CSA. 1645 */ 1646 out_be64(&priv2->spu_chnlcntptr_RW, 29UL); 1647 eieio(); 1648 out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[29]); 1649 for (i = 0; i < 4; i++) { 1650 out_be64(&priv2->spu_chnldata_RW, csa->spu_mailbox_data[i]); 1651 } 1652 eieio(); 1653 } 1654 1655 static inline void check_ppu_mb_stat(struct spu_state *csa, struct spu *spu) 1656 { 1657 struct spu_problem __iomem *prob = spu->problem; 1658 u32 dummy = 0; 1659 1660 /* Restore, Step 66: 1661 * If CSA.MB_Stat[P]=0 (mailbox empty) then 1662 * read from the PPU_MB register. 1663 */ 1664 if ((csa->prob.mb_stat_R & 0xFF) == 0) { 1665 dummy = in_be32(&prob->pu_mb_R); 1666 eieio(); 1667 } 1668 } 1669 1670 static inline void check_ppuint_mb_stat(struct spu_state *csa, struct spu *spu) 1671 { 1672 struct spu_priv2 __iomem *priv2 = spu->priv2; 1673 u64 dummy = 0UL; 1674 1675 /* Restore, Step 66: 1676 * If CSA.MB_Stat[I]=0 (mailbox empty) then 1677 * read from the PPUINT_MB register. 1678 */ 1679 if ((csa->prob.mb_stat_R & 0xFF0000) == 0) { 1680 dummy = in_be64(&priv2->puint_mb_R); 1681 eieio(); 1682 spu_int_stat_clear(spu, 2, CLASS2_ENABLE_MAILBOX_INTR); 1683 eieio(); 1684 } 1685 } 1686 1687 static inline void restore_mfc_sr1(struct spu_state *csa, struct spu *spu) 1688 { 1689 /* Restore, Step 69: 1690 * Restore the MFC_SR1 register from CSA. 1691 */ 1692 spu_mfc_sr1_set(spu, csa->priv1.mfc_sr1_RW); 1693 eieio(); 1694 } 1695 1696 static inline void restore_other_spu_access(struct spu_state *csa, 1697 struct spu *spu) 1698 { 1699 /* Restore, Step 70: 1700 * Restore other SPU mappings to this SPU. TBD. 1701 */ 1702 } 1703 1704 static inline void restore_spu_runcntl(struct spu_state *csa, struct spu *spu) 1705 { 1706 struct spu_problem __iomem *prob = spu->problem; 1707 1708 /* Restore, Step 71: 1709 * If CSA.SPU_Status[R]=1 then write 1710 * SPU_RunCntl[R0R1]='01'. 1711 */ 1712 if (csa->prob.spu_status_R & SPU_STATUS_RUNNING) { 1713 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1714 eieio(); 1715 } 1716 } 1717 1718 static inline void restore_mfc_cntl(struct spu_state *csa, struct spu *spu) 1719 { 1720 struct spu_priv2 __iomem *priv2 = spu->priv2; 1721 1722 /* Restore, Step 72: 1723 * Restore the MFC_CNTL register for the CSA. 1724 */ 1725 out_be64(&priv2->mfc_control_RW, csa->priv2.mfc_control_RW); 1726 eieio(); 1727 /* 1728 * FIXME: this is to restart a DMA that we were processing 1729 * before the save. better remember the fault information 1730 * in the csa instead. 1731 */ 1732 if ((csa->priv2.mfc_control_RW & MFC_CNTL_SUSPEND_DMA_QUEUE_MASK)) { 1733 out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESTART_DMA_COMMAND); 1734 eieio(); 1735 } 1736 } 1737 1738 static inline void enable_user_access(struct spu_state *csa, struct spu *spu) 1739 { 1740 /* Restore, Step 73: 1741 * Enable user-space access (if provided) to this 1742 * SPU by mapping the virtual pages assigned to 1743 * the SPU memory-mapped I/O (MMIO) for problem 1744 * state. TBD. 1745 */ 1746 } 1747 1748 static inline void reset_switch_active(struct spu_state *csa, struct spu *spu) 1749 { 1750 /* Restore, Step 74: 1751 * Reset the "context switch active" flag. 1752 */ 1753 clear_bit(SPU_CONTEXT_SWITCH_ACTIVE, &spu->flags); 1754 mb(); 1755 } 1756 1757 static inline void reenable_interrupts(struct spu_state *csa, struct spu *spu) 1758 { 1759 /* Restore, Step 75: 1760 * Re-enable SPU interrupts. 1761 */ 1762 spin_lock_irq(&spu->register_lock); 1763 spu_int_mask_set(spu, 0, csa->priv1.int_mask_class0_RW); 1764 spu_int_mask_set(spu, 1, csa->priv1.int_mask_class1_RW); 1765 spu_int_mask_set(spu, 2, csa->priv1.int_mask_class2_RW); 1766 spin_unlock_irq(&spu->register_lock); 1767 } 1768 1769 static int quiece_spu(struct spu_state *prev, struct spu *spu) 1770 { 1771 /* 1772 * Combined steps 2-18 of SPU context save sequence, which 1773 * quiesce the SPU state (disable SPU execution, MFC command 1774 * queues, decrementer, SPU interrupts, etc.). 1775 * 1776 * Returns 0 on success. 1777 * 2 if failed step 2. 1778 * 6 if failed step 6. 1779 */ 1780 1781 if (check_spu_isolate(prev, spu)) { /* Step 2. */ 1782 return 2; 1783 } 1784 disable_interrupts(prev, spu); /* Step 3. */ 1785 set_watchdog_timer(prev, spu); /* Step 4. */ 1786 inhibit_user_access(prev, spu); /* Step 5. */ 1787 if (check_spu_isolate(prev, spu)) { /* Step 6. */ 1788 return 6; 1789 } 1790 set_switch_pending(prev, spu); /* Step 7. */ 1791 save_mfc_cntl(prev, spu); /* Step 8. */ 1792 save_spu_runcntl(prev, spu); /* Step 9. */ 1793 save_mfc_sr1(prev, spu); /* Step 10. */ 1794 save_spu_status(prev, spu); /* Step 11. */ 1795 save_mfc_decr(prev, spu); /* Step 12. */ 1796 halt_mfc_decr(prev, spu); /* Step 13. */ 1797 save_timebase(prev, spu); /* Step 14. */ 1798 remove_other_spu_access(prev, spu); /* Step 15. */ 1799 do_mfc_mssync(prev, spu); /* Step 16. */ 1800 issue_mfc_tlbie(prev, spu); /* Step 17. */ 1801 handle_pending_interrupts(prev, spu); /* Step 18. */ 1802 1803 return 0; 1804 } 1805 1806 static void save_csa(struct spu_state *prev, struct spu *spu) 1807 { 1808 /* 1809 * Combine steps 19-44 of SPU context save sequence, which 1810 * save regions of the privileged & problem state areas. 1811 */ 1812 1813 save_mfc_queues(prev, spu); /* Step 19. */ 1814 save_ppu_querymask(prev, spu); /* Step 20. */ 1815 save_ppu_querytype(prev, spu); /* Step 21. */ 1816 save_mfc_csr_tsq(prev, spu); /* Step 22. */ 1817 save_mfc_csr_cmd(prev, spu); /* Step 23. */ 1818 save_mfc_csr_ato(prev, spu); /* Step 24. */ 1819 save_mfc_tclass_id(prev, spu); /* Step 25. */ 1820 set_mfc_tclass_id(prev, spu); /* Step 26. */ 1821 purge_mfc_queue(prev, spu); /* Step 27. */ 1822 wait_purge_complete(prev, spu); /* Step 28. */ 1823 setup_mfc_sr1(prev, spu); /* Step 30. */ 1824 save_spu_npc(prev, spu); /* Step 31. */ 1825 save_spu_privcntl(prev, spu); /* Step 32. */ 1826 reset_spu_privcntl(prev, spu); /* Step 33. */ 1827 save_spu_lslr(prev, spu); /* Step 34. */ 1828 reset_spu_lslr(prev, spu); /* Step 35. */ 1829 save_spu_cfg(prev, spu); /* Step 36. */ 1830 save_pm_trace(prev, spu); /* Step 37. */ 1831 save_mfc_rag(prev, spu); /* Step 38. */ 1832 save_ppu_mb_stat(prev, spu); /* Step 39. */ 1833 save_ppu_mb(prev, spu); /* Step 40. */ 1834 save_ppuint_mb(prev, spu); /* Step 41. */ 1835 save_ch_part1(prev, spu); /* Step 42. */ 1836 save_spu_mb(prev, spu); /* Step 43. */ 1837 save_mfc_cmd(prev, spu); /* Step 44. */ 1838 reset_ch(prev, spu); /* Step 45. */ 1839 } 1840 1841 static void save_lscsa(struct spu_state *prev, struct spu *spu) 1842 { 1843 /* 1844 * Perform steps 46-57 of SPU context save sequence, 1845 * which save regions of the local store and register 1846 * file. 1847 */ 1848 1849 resume_mfc_queue(prev, spu); /* Step 46. */ 1850 setup_mfc_slbs(prev, spu); /* Step 47. */ 1851 set_switch_active(prev, spu); /* Step 48. */ 1852 enable_interrupts(prev, spu); /* Step 49. */ 1853 save_ls_16kb(prev, spu); /* Step 50. */ 1854 set_spu_npc(prev, spu); /* Step 51. */ 1855 set_signot1(prev, spu); /* Step 52. */ 1856 set_signot2(prev, spu); /* Step 53. */ 1857 send_save_code(prev, spu); /* Step 54. */ 1858 set_ppu_querymask(prev, spu); /* Step 55. */ 1859 wait_tag_complete(prev, spu); /* Step 56. */ 1860 wait_spu_stopped(prev, spu); /* Step 57. */ 1861 } 1862 1863 static void force_spu_isolate_exit(struct spu *spu) 1864 { 1865 struct spu_problem __iomem *prob = spu->problem; 1866 struct spu_priv2 __iomem *priv2 = spu->priv2; 1867 1868 /* Stop SPE execution and wait for completion. */ 1869 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1870 iobarrier_rw(); 1871 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING); 1872 1873 /* Restart SPE master runcntl. */ 1874 spu_mfc_sr1_set(spu, MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1875 iobarrier_w(); 1876 1877 /* Initiate isolate exit request and wait for completion. */ 1878 out_be64(&priv2->spu_privcntl_RW, 4LL); 1879 iobarrier_w(); 1880 out_be32(&prob->spu_runcntl_RW, 2); 1881 iobarrier_rw(); 1882 POLL_WHILE_FALSE((in_be32(&prob->spu_status_R) 1883 & SPU_STATUS_STOPPED_BY_STOP)); 1884 1885 /* Reset load request to normal. */ 1886 out_be64(&priv2->spu_privcntl_RW, SPU_PRIVCNT_LOAD_REQUEST_NORMAL); 1887 iobarrier_w(); 1888 } 1889 1890 /** 1891 * stop_spu_isolate 1892 * Check SPU run-control state and force isolated 1893 * exit function as necessary. 1894 */ 1895 static void stop_spu_isolate(struct spu *spu) 1896 { 1897 struct spu_problem __iomem *prob = spu->problem; 1898 1899 if (in_be32(&prob->spu_status_R) & SPU_STATUS_ISOLATED_STATE) { 1900 /* The SPU is in isolated state; the only way 1901 * to get it out is to perform an isolated 1902 * exit (clean) operation. 1903 */ 1904 force_spu_isolate_exit(spu); 1905 } 1906 } 1907 1908 static void harvest(struct spu_state *prev, struct spu *spu) 1909 { 1910 /* 1911 * Perform steps 2-25 of SPU context restore sequence, 1912 * which resets an SPU either after a failed save, or 1913 * when using SPU for first time. 1914 */ 1915 1916 disable_interrupts(prev, spu); /* Step 2. */ 1917 inhibit_user_access(prev, spu); /* Step 3. */ 1918 terminate_spu_app(prev, spu); /* Step 4. */ 1919 set_switch_pending(prev, spu); /* Step 5. */ 1920 stop_spu_isolate(spu); /* NEW. */ 1921 remove_other_spu_access(prev, spu); /* Step 6. */ 1922 suspend_mfc(prev, spu); /* Step 7. */ 1923 wait_suspend_mfc_complete(prev, spu); /* Step 8. */ 1924 if (!suspend_spe(prev, spu)) /* Step 9. */ 1925 clear_spu_status(prev, spu); /* Step 10. */ 1926 do_mfc_mssync(prev, spu); /* Step 11. */ 1927 issue_mfc_tlbie(prev, spu); /* Step 12. */ 1928 handle_pending_interrupts(prev, spu); /* Step 13. */ 1929 purge_mfc_queue(prev, spu); /* Step 14. */ 1930 wait_purge_complete(prev, spu); /* Step 15. */ 1931 reset_spu_privcntl(prev, spu); /* Step 16. */ 1932 reset_spu_lslr(prev, spu); /* Step 17. */ 1933 setup_mfc_sr1(prev, spu); /* Step 18. */ 1934 spu_invalidate_slbs(spu); /* Step 19. */ 1935 reset_ch_part1(prev, spu); /* Step 20. */ 1936 reset_ch_part2(prev, spu); /* Step 21. */ 1937 enable_interrupts(prev, spu); /* Step 22. */ 1938 set_switch_active(prev, spu); /* Step 23. */ 1939 set_mfc_tclass_id(prev, spu); /* Step 24. */ 1940 resume_mfc_queue(prev, spu); /* Step 25. */ 1941 } 1942 1943 static void restore_lscsa(struct spu_state *next, struct spu *spu) 1944 { 1945 /* 1946 * Perform steps 26-40 of SPU context restore sequence, 1947 * which restores regions of the local store and register 1948 * file. 1949 */ 1950 1951 set_watchdog_timer(next, spu); /* Step 26. */ 1952 setup_spu_status_part1(next, spu); /* Step 27. */ 1953 setup_spu_status_part2(next, spu); /* Step 28. */ 1954 restore_mfc_rag(next, spu); /* Step 29. */ 1955 setup_mfc_slbs(next, spu); /* Step 30. */ 1956 set_spu_npc(next, spu); /* Step 31. */ 1957 set_signot1(next, spu); /* Step 32. */ 1958 set_signot2(next, spu); /* Step 33. */ 1959 setup_decr(next, spu); /* Step 34. */ 1960 setup_ppu_mb(next, spu); /* Step 35. */ 1961 setup_ppuint_mb(next, spu); /* Step 36. */ 1962 send_restore_code(next, spu); /* Step 37. */ 1963 set_ppu_querymask(next, spu); /* Step 38. */ 1964 wait_tag_complete(next, spu); /* Step 39. */ 1965 wait_spu_stopped(next, spu); /* Step 40. */ 1966 } 1967 1968 static void restore_csa(struct spu_state *next, struct spu *spu) 1969 { 1970 /* 1971 * Combine steps 41-76 of SPU context restore sequence, which 1972 * restore regions of the privileged & problem state areas. 1973 */ 1974 1975 restore_spu_privcntl(next, spu); /* Step 41. */ 1976 restore_status_part1(next, spu); /* Step 42. */ 1977 restore_status_part2(next, spu); /* Step 43. */ 1978 restore_ls_16kb(next, spu); /* Step 44. */ 1979 wait_tag_complete(next, spu); /* Step 45. */ 1980 suspend_mfc(next, spu); /* Step 46. */ 1981 wait_suspend_mfc_complete(next, spu); /* Step 47. */ 1982 issue_mfc_tlbie(next, spu); /* Step 48. */ 1983 clear_interrupts(next, spu); /* Step 49. */ 1984 restore_mfc_queues(next, spu); /* Step 50. */ 1985 restore_ppu_querymask(next, spu); /* Step 51. */ 1986 restore_ppu_querytype(next, spu); /* Step 52. */ 1987 restore_mfc_csr_tsq(next, spu); /* Step 53. */ 1988 restore_mfc_csr_cmd(next, spu); /* Step 54. */ 1989 restore_mfc_csr_ato(next, spu); /* Step 55. */ 1990 restore_mfc_tclass_id(next, spu); /* Step 56. */ 1991 set_llr_event(next, spu); /* Step 57. */ 1992 restore_decr_wrapped(next, spu); /* Step 58. */ 1993 restore_ch_part1(next, spu); /* Step 59. */ 1994 restore_ch_part2(next, spu); /* Step 60. */ 1995 restore_spu_lslr(next, spu); /* Step 61. */ 1996 restore_spu_cfg(next, spu); /* Step 62. */ 1997 restore_pm_trace(next, spu); /* Step 63. */ 1998 restore_spu_npc(next, spu); /* Step 64. */ 1999 restore_spu_mb(next, spu); /* Step 65. */ 2000 check_ppu_mb_stat(next, spu); /* Step 66. */ 2001 check_ppuint_mb_stat(next, spu); /* Step 67. */ 2002 spu_invalidate_slbs(spu); /* Modified Step 68. */ 2003 restore_mfc_sr1(next, spu); /* Step 69. */ 2004 restore_other_spu_access(next, spu); /* Step 70. */ 2005 restore_spu_runcntl(next, spu); /* Step 71. */ 2006 restore_mfc_cntl(next, spu); /* Step 72. */ 2007 enable_user_access(next, spu); /* Step 73. */ 2008 reset_switch_active(next, spu); /* Step 74. */ 2009 reenable_interrupts(next, spu); /* Step 75. */ 2010 } 2011 2012 static int __do_spu_save(struct spu_state *prev, struct spu *spu) 2013 { 2014 int rc; 2015 2016 /* 2017 * SPU context save can be broken into three phases: 2018 * 2019 * (a) quiesce [steps 2-16]. 2020 * (b) save of CSA, performed by PPE [steps 17-42] 2021 * (c) save of LSCSA, mostly performed by SPU [steps 43-52]. 2022 * 2023 * Returns 0 on success. 2024 * 2,6 if failed to quiece SPU 2025 * 53 if SPU-side of save failed. 2026 */ 2027 2028 rc = quiece_spu(prev, spu); /* Steps 2-16. */ 2029 switch (rc) { 2030 default: 2031 case 2: 2032 case 6: 2033 harvest(prev, spu); 2034 return rc; 2035 break; 2036 case 0: 2037 break; 2038 } 2039 save_csa(prev, spu); /* Steps 17-43. */ 2040 save_lscsa(prev, spu); /* Steps 44-53. */ 2041 return check_save_status(prev, spu); /* Step 54. */ 2042 } 2043 2044 static int __do_spu_restore(struct spu_state *next, struct spu *spu) 2045 { 2046 int rc; 2047 2048 /* 2049 * SPU context restore can be broken into three phases: 2050 * 2051 * (a) harvest (or reset) SPU [steps 2-24]. 2052 * (b) restore LSCSA [steps 25-40], mostly performed by SPU. 2053 * (c) restore CSA [steps 41-76], performed by PPE. 2054 * 2055 * The 'harvest' step is not performed here, but rather 2056 * as needed below. 2057 */ 2058 2059 restore_lscsa(next, spu); /* Steps 24-39. */ 2060 rc = check_restore_status(next, spu); /* Step 40. */ 2061 switch (rc) { 2062 default: 2063 /* Failed. Return now. */ 2064 return rc; 2065 break; 2066 case 0: 2067 /* Fall through to next step. */ 2068 break; 2069 } 2070 restore_csa(next, spu); 2071 2072 return 0; 2073 } 2074 2075 /** 2076 * spu_save - SPU context save, with locking. 2077 * @prev: pointer to SPU context save area, to be saved. 2078 * @spu: pointer to SPU iomem structure. 2079 * 2080 * Acquire locks, perform the save operation then return. 2081 */ 2082 int spu_save(struct spu_state *prev, struct spu *spu) 2083 { 2084 int rc; 2085 2086 acquire_spu_lock(spu); /* Step 1. */ 2087 rc = __do_spu_save(prev, spu); /* Steps 2-53. */ 2088 release_spu_lock(spu); 2089 if (rc != 0 && rc != 2 && rc != 6) { 2090 panic("%s failed on SPU[%d], rc=%d.\n", 2091 __func__, spu->number, rc); 2092 } 2093 return 0; 2094 } 2095 EXPORT_SYMBOL_GPL(spu_save); 2096 2097 /** 2098 * spu_restore - SPU context restore, with harvest and locking. 2099 * @new: pointer to SPU context save area, to be restored. 2100 * @spu: pointer to SPU iomem structure. 2101 * 2102 * Perform harvest + restore, as we may not be coming 2103 * from a previous successful save operation, and the 2104 * hardware state is unknown. 2105 */ 2106 int spu_restore(struct spu_state *new, struct spu *spu) 2107 { 2108 int rc; 2109 2110 acquire_spu_lock(spu); 2111 harvest(NULL, spu); 2112 spu->dar = 0; 2113 spu->dsisr = 0; 2114 spu->slb_replace = 0; 2115 spu->class_0_pending = 0; 2116 rc = __do_spu_restore(new, spu); 2117 release_spu_lock(spu); 2118 if (rc) { 2119 panic("%s failed on SPU[%d] rc=%d.\n", 2120 __func__, spu->number, rc); 2121 } 2122 return rc; 2123 } 2124 EXPORT_SYMBOL_GPL(spu_restore); 2125 2126 /** 2127 * spu_harvest - SPU harvest (reset) operation 2128 * @spu: pointer to SPU iomem structure. 2129 * 2130 * Perform SPU harvest (reset) operation. 2131 */ 2132 void spu_harvest(struct spu *spu) 2133 { 2134 acquire_spu_lock(spu); 2135 harvest(NULL, spu); 2136 release_spu_lock(spu); 2137 } 2138 2139 static void init_prob(struct spu_state *csa) 2140 { 2141 csa->spu_chnlcnt_RW[9] = 1; 2142 csa->spu_chnlcnt_RW[21] = 16; 2143 csa->spu_chnlcnt_RW[23] = 1; 2144 csa->spu_chnlcnt_RW[28] = 1; 2145 csa->spu_chnlcnt_RW[30] = 1; 2146 csa->prob.spu_runcntl_RW = SPU_RUNCNTL_STOP; 2147 csa->prob.mb_stat_R = 0x000400; 2148 } 2149 2150 static void init_priv1(struct spu_state *csa) 2151 { 2152 /* Enable decode, relocate, tlbie response, master runcntl. */ 2153 csa->priv1.mfc_sr1_RW = MFC_STATE1_LOCAL_STORAGE_DECODE_MASK | 2154 MFC_STATE1_MASTER_RUN_CONTROL_MASK | 2155 MFC_STATE1_PROBLEM_STATE_MASK | 2156 MFC_STATE1_RELOCATE_MASK | MFC_STATE1_BUS_TLBIE_MASK; 2157 2158 /* Enable OS-specific set of interrupts. */ 2159 csa->priv1.int_mask_class0_RW = CLASS0_ENABLE_DMA_ALIGNMENT_INTR | 2160 CLASS0_ENABLE_INVALID_DMA_COMMAND_INTR | 2161 CLASS0_ENABLE_SPU_ERROR_INTR; 2162 csa->priv1.int_mask_class1_RW = CLASS1_ENABLE_SEGMENT_FAULT_INTR | 2163 CLASS1_ENABLE_STORAGE_FAULT_INTR; 2164 csa->priv1.int_mask_class2_RW = CLASS2_ENABLE_SPU_STOP_INTR | 2165 CLASS2_ENABLE_SPU_HALT_INTR | 2166 CLASS2_ENABLE_SPU_DMA_TAG_GROUP_COMPLETE_INTR; 2167 } 2168 2169 static void init_priv2(struct spu_state *csa) 2170 { 2171 csa->priv2.spu_lslr_RW = LS_ADDR_MASK; 2172 csa->priv2.mfc_control_RW = MFC_CNTL_RESUME_DMA_QUEUE | 2173 MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION | 2174 MFC_CNTL_DMA_QUEUES_EMPTY_MASK; 2175 } 2176 2177 /** 2178 * spu_alloc_csa - allocate and initialize an SPU context save area. 2179 * 2180 * Allocate and initialize the contents of an SPU context save area. 2181 * This includes enabling address translation, interrupt masks, etc., 2182 * as appropriate for the given OS environment. 2183 * 2184 * Note that storage for the 'lscsa' is allocated separately, 2185 * as it is by far the largest of the context save regions, 2186 * and may need to be pinned or otherwise specially aligned. 2187 */ 2188 void spu_init_csa(struct spu_state *csa) 2189 { 2190 struct spu_lscsa *lscsa; 2191 unsigned char *p; 2192 2193 if (!csa) 2194 return; 2195 memset(csa, 0, sizeof(struct spu_state)); 2196 2197 lscsa = vmalloc(sizeof(struct spu_lscsa)); 2198 if (!lscsa) 2199 return; 2200 2201 memset(lscsa, 0, sizeof(struct spu_lscsa)); 2202 csa->lscsa = lscsa; 2203 spin_lock_init(&csa->register_lock); 2204 2205 /* Set LS pages reserved to allow for user-space mapping. */ 2206 for (p = lscsa->ls; p < lscsa->ls + LS_SIZE; p += PAGE_SIZE) 2207 SetPageReserved(vmalloc_to_page(p)); 2208 2209 init_prob(csa); 2210 init_priv1(csa); 2211 init_priv2(csa); 2212 } 2213 EXPORT_SYMBOL_GPL(spu_init_csa); 2214 2215 void spu_fini_csa(struct spu_state *csa) 2216 { 2217 /* Clear reserved bit before vfree. */ 2218 unsigned char *p; 2219 for (p = csa->lscsa->ls; p < csa->lscsa->ls + LS_SIZE; p += PAGE_SIZE) 2220 ClearPageReserved(vmalloc_to_page(p)); 2221 2222 vfree(csa->lscsa); 2223 } 2224 EXPORT_SYMBOL_GPL(spu_fini_csa); 2225