1 /***********************license start*************** 2 * Author: Cavium Networks 3 * 4 * Contact: support@caviumnetworks.com 5 * This file is part of the OCTEON SDK 6 * 7 * Copyright (c) 2003-2008 Cavium Networks 8 * 9 * This file is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License, Version 2, as 11 * published by the Free Software Foundation. 12 * 13 * This file is distributed in the hope that it will be useful, but 14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty 15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or 16 * NONINFRINGEMENT. See the GNU General Public License for more 17 * details. 18 * 19 * You should have received a copy of the GNU General Public License 20 * along with this file; if not, write to the Free Software 21 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 22 * or visit http://www.gnu.org/licenses/. 23 * 24 * This file may also be available under a different license from Cavium. 25 * Contact Cavium Networks for more information 26 ***********************license end**************************************/ 27 28 /** 29 * Interface to the hardware Packet Order / Work unit. 30 * 31 * New, starting with SDK 1.7.0, cvmx-pow supports a number of 32 * extended consistency checks. The define 33 * CVMX_ENABLE_POW_CHECKS controls the runtime insertion of POW 34 * internal state checks to find common programming errors. If 35 * CVMX_ENABLE_POW_CHECKS is not defined, checks are by default 36 * enabled. For example, cvmx-pow will check for the following 37 * program errors or POW state inconsistency. 38 * - Requesting a POW operation with an active tag switch in 39 * progress. 40 * - Waiting for a tag switch to complete for an excessively 41 * long period. This is normally a sign of an error in locking 42 * causing deadlock. 43 * - Illegal tag switches from NULL_NULL. 44 * - Illegal tag switches from NULL. 45 * - Illegal deschedule request. 46 * - WQE pointer not matching the one attached to the core by 47 * the POW. 48 * 49 */ 50 51 #ifndef __CVMX_POW_H__ 52 #define __CVMX_POW_H__ 53 54 #include <asm/octeon/cvmx-pow-defs.h> 55 56 #include <asm/octeon/cvmx-scratch.h> 57 #include <asm/octeon/cvmx-wqe.h> 58 59 /* Default to having all POW constancy checks turned on */ 60 #ifndef CVMX_ENABLE_POW_CHECKS 61 #define CVMX_ENABLE_POW_CHECKS 1 62 #endif 63 64 enum cvmx_pow_tag_type { 65 /* Tag ordering is maintained */ 66 CVMX_POW_TAG_TYPE_ORDERED = 0L, 67 /* Tag ordering is maintained, and at most one PP has the tag */ 68 CVMX_POW_TAG_TYPE_ATOMIC = 1L, 69 /* 70 * The work queue entry from the order - NEVER tag switch from 71 * NULL to NULL 72 */ 73 CVMX_POW_TAG_TYPE_NULL = 2L, 74 /* A tag switch to NULL, and there is no space reserved in POW 75 * - NEVER tag switch to NULL_NULL 76 * - NEVER tag switch from NULL_NULL 77 * - NULL_NULL is entered at the beginning of time and on a deschedule. 78 * - NULL_NULL can be exited by a new work request. A NULL_SWITCH 79 * load can also switch the state to NULL 80 */ 81 CVMX_POW_TAG_TYPE_NULL_NULL = 3L 82 }; 83 84 /** 85 * Wait flag values for pow functions. 86 */ 87 typedef enum { 88 CVMX_POW_WAIT = 1, 89 CVMX_POW_NO_WAIT = 0, 90 } cvmx_pow_wait_t; 91 92 /** 93 * POW tag operations. These are used in the data stored to the POW. 94 */ 95 typedef enum { 96 /* 97 * switch the tag (only) for this PP 98 * - the previous tag should be non-NULL in this case 99 * - tag switch response required 100 * - fields used: op, type, tag 101 */ 102 CVMX_POW_TAG_OP_SWTAG = 0L, 103 /* 104 * switch the tag for this PP, with full information 105 * - this should be used when the previous tag is NULL 106 * - tag switch response required 107 * - fields used: address, op, grp, type, tag 108 */ 109 CVMX_POW_TAG_OP_SWTAG_FULL = 1L, 110 /* 111 * switch the tag (and/or group) for this PP and de-schedule 112 * - OK to keep the tag the same and only change the group 113 * - fields used: op, no_sched, grp, type, tag 114 */ 115 CVMX_POW_TAG_OP_SWTAG_DESCH = 2L, 116 /* 117 * just de-schedule 118 * - fields used: op, no_sched 119 */ 120 CVMX_POW_TAG_OP_DESCH = 3L, 121 /* 122 * create an entirely new work queue entry 123 * - fields used: address, op, qos, grp, type, tag 124 */ 125 CVMX_POW_TAG_OP_ADDWQ = 4L, 126 /* 127 * just update the work queue pointer and grp for this PP 128 * - fields used: address, op, grp 129 */ 130 CVMX_POW_TAG_OP_UPDATE_WQP_GRP = 5L, 131 /* 132 * set the no_sched bit on the de-schedule list 133 * 134 * - does nothing if the selected entry is not on the 135 * de-schedule list 136 * 137 * - does nothing if the stored work queue pointer does not 138 * match the address field 139 * 140 * - fields used: address, index, op 141 * 142 * Before issuing a *_NSCHED operation, SW must guarantee 143 * that all prior deschedules and set/clr NSCHED operations 144 * are complete and all prior switches are complete. The 145 * hardware provides the opsdone bit and swdone bit for SW 146 * polling. After issuing a *_NSCHED operation, SW must 147 * guarantee that the set/clr NSCHED is complete before any 148 * subsequent operations. 149 */ 150 CVMX_POW_TAG_OP_SET_NSCHED = 6L, 151 /* 152 * clears the no_sched bit on the de-schedule list 153 * 154 * - does nothing if the selected entry is not on the 155 * de-schedule list 156 * 157 * - does nothing if the stored work queue pointer does not 158 * match the address field 159 * 160 * - fields used: address, index, op 161 * 162 * Before issuing a *_NSCHED operation, SW must guarantee that 163 * all prior deschedules and set/clr NSCHED operations are 164 * complete and all prior switches are complete. The hardware 165 * provides the opsdone bit and swdone bit for SW 166 * polling. After issuing a *_NSCHED operation, SW must 167 * guarantee that the set/clr NSCHED is complete before any 168 * subsequent operations. 169 */ 170 CVMX_POW_TAG_OP_CLR_NSCHED = 7L, 171 /* do nothing */ 172 CVMX_POW_TAG_OP_NOP = 15L 173 } cvmx_pow_tag_op_t; 174 175 /** 176 * This structure defines the store data on a store to POW 177 */ 178 typedef union { 179 uint64_t u64; 180 struct { 181 #ifdef __BIG_ENDIAN_BITFIELD 182 /* 183 * Don't reschedule this entry. no_sched is used for 184 * CVMX_POW_TAG_OP_SWTAG_DESCH and 185 * CVMX_POW_TAG_OP_DESCH 186 */ 187 uint64_t no_sched:1; 188 uint64_t unused:2; 189 /* Tontains index of entry for a CVMX_POW_TAG_OP_*_NSCHED */ 190 uint64_t index:13; 191 /* The operation to perform */ 192 cvmx_pow_tag_op_t op:4; 193 uint64_t unused2:2; 194 /* 195 * The QOS level for the packet. qos is only used for 196 * CVMX_POW_TAG_OP_ADDWQ 197 */ 198 uint64_t qos:3; 199 /* 200 * The group that the work queue entry will be 201 * scheduled to grp is used for CVMX_POW_TAG_OP_ADDWQ, 202 * CVMX_POW_TAG_OP_SWTAG_FULL, 203 * CVMX_POW_TAG_OP_SWTAG_DESCH, and 204 * CVMX_POW_TAG_OP_UPDATE_WQP_GRP 205 */ 206 uint64_t grp:4; 207 /* 208 * The type of the tag. type is used for everything 209 * except CVMX_POW_TAG_OP_DESCH, 210 * CVMX_POW_TAG_OP_UPDATE_WQP_GRP, and 211 * CVMX_POW_TAG_OP_*_NSCHED 212 */ 213 uint64_t type:3; 214 /* 215 * The actual tag. tag is used for everything except 216 * CVMX_POW_TAG_OP_DESCH, 217 * CVMX_POW_TAG_OP_UPDATE_WQP_GRP, and 218 * CVMX_POW_TAG_OP_*_NSCHED 219 */ 220 uint64_t tag:32; 221 #else 222 uint64_t tag:32; 223 uint64_t type:3; 224 uint64_t grp:4; 225 uint64_t qos:3; 226 uint64_t unused2:2; 227 cvmx_pow_tag_op_t op:4; 228 uint64_t index:13; 229 uint64_t unused:2; 230 uint64_t no_sched:1; 231 #endif 232 } s; 233 } cvmx_pow_tag_req_t; 234 235 /** 236 * This structure describes the address to load stuff from POW 237 */ 238 typedef union { 239 uint64_t u64; 240 241 /** 242 * Address for new work request loads (did<2:0> == 0) 243 */ 244 struct { 245 #ifdef __BIG_ENDIAN_BITFIELD 246 /* Mips64 address region. Should be CVMX_IO_SEG */ 247 uint64_t mem_region:2; 248 /* Must be zero */ 249 uint64_t reserved_49_61:13; 250 /* Must be one */ 251 uint64_t is_io:1; 252 /* the ID of POW -- did<2:0> == 0 in this case */ 253 uint64_t did:8; 254 /* Must be zero */ 255 uint64_t reserved_4_39:36; 256 /* 257 * If set, don't return load response until work is 258 * available. 259 */ 260 uint64_t wait:1; 261 /* Must be zero */ 262 uint64_t reserved_0_2:3; 263 #else 264 uint64_t reserved_0_2:3; 265 uint64_t wait:1; 266 uint64_t reserved_4_39:36; 267 uint64_t did:8; 268 uint64_t is_io:1; 269 uint64_t reserved_49_61:13; 270 uint64_t mem_region:2; 271 #endif 272 } swork; 273 274 /** 275 * Address for loads to get POW internal status 276 */ 277 struct { 278 #ifdef __BIG_ENDIAN_BITFIELD 279 /* Mips64 address region. Should be CVMX_IO_SEG */ 280 uint64_t mem_region:2; 281 /* Must be zero */ 282 uint64_t reserved_49_61:13; 283 /* Must be one */ 284 uint64_t is_io:1; 285 /* the ID of POW -- did<2:0> == 1 in this case */ 286 uint64_t did:8; 287 /* Must be zero */ 288 uint64_t reserved_10_39:30; 289 /* The core id to get status for */ 290 uint64_t coreid:4; 291 /* 292 * If set and get_cur is set, return reverse tag-list 293 * pointer rather than forward tag-list pointer. 294 */ 295 uint64_t get_rev:1; 296 /* 297 * If set, return current status rather than pending 298 * status. 299 */ 300 uint64_t get_cur:1; 301 /* 302 * If set, get the work-queue pointer rather than 303 * tag/type. 304 */ 305 uint64_t get_wqp:1; 306 /* Must be zero */ 307 uint64_t reserved_0_2:3; 308 #else 309 uint64_t reserved_0_2:3; 310 uint64_t get_wqp:1; 311 uint64_t get_cur:1; 312 uint64_t get_rev:1; 313 uint64_t coreid:4; 314 uint64_t reserved_10_39:30; 315 uint64_t did:8; 316 uint64_t is_io:1; 317 uint64_t reserved_49_61:13; 318 uint64_t mem_region:2; 319 #endif 320 } sstatus; 321 322 /** 323 * Address for memory loads to get POW internal state 324 */ 325 struct { 326 #ifdef __BIG_ENDIAN_BITFIELD 327 /* Mips64 address region. Should be CVMX_IO_SEG */ 328 uint64_t mem_region:2; 329 /* Must be zero */ 330 uint64_t reserved_49_61:13; 331 /* Must be one */ 332 uint64_t is_io:1; 333 /* the ID of POW -- did<2:0> == 2 in this case */ 334 uint64_t did:8; 335 /* Must be zero */ 336 uint64_t reserved_16_39:24; 337 /* POW memory index */ 338 uint64_t index:11; 339 /* 340 * If set, return deschedule information rather than 341 * the standard response for work-queue index (invalid 342 * if the work-queue entry is not on the deschedule 343 * list). 344 */ 345 uint64_t get_des:1; 346 /* 347 * If set, get the work-queue pointer rather than 348 * tag/type (no effect when get_des set). 349 */ 350 uint64_t get_wqp:1; 351 /* Must be zero */ 352 uint64_t reserved_0_2:3; 353 #else 354 uint64_t reserved_0_2:3; 355 uint64_t get_wqp:1; 356 uint64_t get_des:1; 357 uint64_t index:11; 358 uint64_t reserved_16_39:24; 359 uint64_t did:8; 360 uint64_t is_io:1; 361 uint64_t reserved_49_61:13; 362 uint64_t mem_region:2; 363 #endif 364 } smemload; 365 366 /** 367 * Address for index/pointer loads 368 */ 369 struct { 370 #ifdef __BIG_ENDIAN_BITFIELD 371 /* Mips64 address region. Should be CVMX_IO_SEG */ 372 uint64_t mem_region:2; 373 /* Must be zero */ 374 uint64_t reserved_49_61:13; 375 /* Must be one */ 376 uint64_t is_io:1; 377 /* the ID of POW -- did<2:0> == 3 in this case */ 378 uint64_t did:8; 379 /* Must be zero */ 380 uint64_t reserved_9_39:31; 381 /* 382 * when {get_rmt ==0 AND get_des_get_tail == 0}, this 383 * field selects one of eight POW internal-input 384 * queues (0-7), one per QOS level; values 8-15 are 385 * illegal in this case; when {get_rmt ==0 AND 386 * get_des_get_tail == 1}, this field selects one of 387 * 16 deschedule lists (per group); when get_rmt ==1, 388 * this field selects one of 16 memory-input queue 389 * lists. The two memory-input queue lists associated 390 * with each QOS level are: 391 * 392 * - qosgrp = 0, qosgrp = 8: QOS0 393 * - qosgrp = 1, qosgrp = 9: QOS1 394 * - qosgrp = 2, qosgrp = 10: QOS2 395 * - qosgrp = 3, qosgrp = 11: QOS3 396 * - qosgrp = 4, qosgrp = 12: QOS4 397 * - qosgrp = 5, qosgrp = 13: QOS5 398 * - qosgrp = 6, qosgrp = 14: QOS6 399 * - qosgrp = 7, qosgrp = 15: QOS7 400 */ 401 uint64_t qosgrp:4; 402 /* 403 * If set and get_rmt is clear, return deschedule list 404 * indexes rather than indexes for the specified qos 405 * level; if set and get_rmt is set, return the tail 406 * pointer rather than the head pointer for the 407 * specified qos level. 408 */ 409 uint64_t get_des_get_tail:1; 410 /* 411 * If set, return remote pointers rather than the 412 * local indexes for the specified qos level. 413 */ 414 uint64_t get_rmt:1; 415 /* Must be zero */ 416 uint64_t reserved_0_2:3; 417 #else 418 uint64_t reserved_0_2:3; 419 uint64_t get_rmt:1; 420 uint64_t get_des_get_tail:1; 421 uint64_t qosgrp:4; 422 uint64_t reserved_9_39:31; 423 uint64_t did:8; 424 uint64_t is_io:1; 425 uint64_t reserved_49_61:13; 426 uint64_t mem_region:2; 427 #endif 428 } sindexload; 429 430 /** 431 * address for NULL_RD request (did<2:0> == 4) when this is read, 432 * HW attempts to change the state to NULL if it is NULL_NULL (the 433 * hardware cannot switch from NULL_NULL to NULL if a POW entry is 434 * not available - software may need to recover by finishing 435 * another piece of work before a POW entry can ever become 436 * available.) 437 */ 438 struct { 439 #ifdef __BIG_ENDIAN_BITFIELD 440 /* Mips64 address region. Should be CVMX_IO_SEG */ 441 uint64_t mem_region:2; 442 /* Must be zero */ 443 uint64_t reserved_49_61:13; 444 /* Must be one */ 445 uint64_t is_io:1; 446 /* the ID of POW -- did<2:0> == 4 in this case */ 447 uint64_t did:8; 448 /* Must be zero */ 449 uint64_t reserved_0_39:40; 450 #else 451 uint64_t reserved_0_39:40; 452 uint64_t did:8; 453 uint64_t is_io:1; 454 uint64_t reserved_49_61:13; 455 uint64_t mem_region:2; 456 #endif 457 } snull_rd; 458 } cvmx_pow_load_addr_t; 459 460 /** 461 * This structure defines the response to a load/SENDSINGLE to POW 462 * (except CSR reads) 463 */ 464 typedef union { 465 uint64_t u64; 466 467 /** 468 * Response to new work request loads 469 */ 470 struct { 471 #ifdef __BIG_ENDIAN_BITFIELD 472 /* 473 * Set when no new work queue entry was returned. * 474 * If there was de-scheduled work, the HW will 475 * definitely return it. When this bit is set, it 476 * could mean either mean: 477 * 478 * - There was no work, or 479 * 480 * - There was no work that the HW could find. This 481 * case can happen, regardless of the wait bit value 482 * in the original request, when there is work in 483 * the IQ's that is too deep down the list. 484 */ 485 uint64_t no_work:1; 486 /* Must be zero */ 487 uint64_t reserved_40_62:23; 488 /* 36 in O1 -- the work queue pointer */ 489 uint64_t addr:40; 490 #else 491 uint64_t addr:40; 492 uint64_t reserved_40_62:23; 493 uint64_t no_work:1; 494 #endif 495 } s_work; 496 497 /** 498 * Result for a POW Status Load (when get_cur==0 and get_wqp==0) 499 */ 500 struct { 501 #ifdef __BIG_ENDIAN_BITFIELD 502 uint64_t reserved_62_63:2; 503 /* Set when there is a pending non-NULL SWTAG or 504 * SWTAG_FULL, and the POW entry has not left the list 505 * for the original tag. */ 506 uint64_t pend_switch:1; 507 /* Set when SWTAG_FULL and pend_switch is set. */ 508 uint64_t pend_switch_full:1; 509 /* 510 * Set when there is a pending NULL SWTAG, or an 511 * implicit switch to NULL. 512 */ 513 uint64_t pend_switch_null:1; 514 /* Set when there is a pending DESCHED or SWTAG_DESCHED. */ 515 uint64_t pend_desched:1; 516 /* 517 * Set when there is a pending SWTAG_DESCHED and 518 * pend_desched is set. 519 */ 520 uint64_t pend_desched_switch:1; 521 /* Set when nosched is desired and pend_desched is set. */ 522 uint64_t pend_nosched:1; 523 /* Set when there is a pending GET_WORK. */ 524 uint64_t pend_new_work:1; 525 /* 526 * When pend_new_work is set, this bit indicates that 527 * the wait bit was set. 528 */ 529 uint64_t pend_new_work_wait:1; 530 /* Set when there is a pending NULL_RD. */ 531 uint64_t pend_null_rd:1; 532 /* Set when there is a pending CLR_NSCHED. */ 533 uint64_t pend_nosched_clr:1; 534 uint64_t reserved_51:1; 535 /* This is the index when pend_nosched_clr is set. */ 536 uint64_t pend_index:11; 537 /* 538 * This is the new_grp when (pend_desched AND 539 * pend_desched_switch) is set. 540 */ 541 uint64_t pend_grp:4; 542 uint64_t reserved_34_35:2; 543 /* 544 * This is the tag type when pend_switch or 545 * (pend_desched AND pend_desched_switch) are set. 546 */ 547 uint64_t pend_type:2; 548 /* 549 * - this is the tag when pend_switch or (pend_desched 550 * AND pend_desched_switch) are set. 551 */ 552 uint64_t pend_tag:32; 553 #else 554 uint64_t pend_tag:32; 555 uint64_t pend_type:2; 556 uint64_t reserved_34_35:2; 557 uint64_t pend_grp:4; 558 uint64_t pend_index:11; 559 uint64_t reserved_51:1; 560 uint64_t pend_nosched_clr:1; 561 uint64_t pend_null_rd:1; 562 uint64_t pend_new_work_wait:1; 563 uint64_t pend_new_work:1; 564 uint64_t pend_nosched:1; 565 uint64_t pend_desched_switch:1; 566 uint64_t pend_desched:1; 567 uint64_t pend_switch_null:1; 568 uint64_t pend_switch_full:1; 569 uint64_t pend_switch:1; 570 uint64_t reserved_62_63:2; 571 #endif 572 } s_sstatus0; 573 574 /** 575 * Result for a POW Status Load (when get_cur==0 and get_wqp==1) 576 */ 577 struct { 578 #ifdef __BIG_ENDIAN_BITFIELD 579 uint64_t reserved_62_63:2; 580 /* 581 * Set when there is a pending non-NULL SWTAG or 582 * SWTAG_FULL, and the POW entry has not left the list 583 * for the original tag. 584 */ 585 uint64_t pend_switch:1; 586 /* Set when SWTAG_FULL and pend_switch is set. */ 587 uint64_t pend_switch_full:1; 588 /* 589 * Set when there is a pending NULL SWTAG, or an 590 * implicit switch to NULL. 591 */ 592 uint64_t pend_switch_null:1; 593 /* 594 * Set when there is a pending DESCHED or 595 * SWTAG_DESCHED. 596 */ 597 uint64_t pend_desched:1; 598 /* 599 * Set when there is a pending SWTAG_DESCHED and 600 * pend_desched is set. 601 */ 602 uint64_t pend_desched_switch:1; 603 /* Set when nosched is desired and pend_desched is set. */ 604 uint64_t pend_nosched:1; 605 /* Set when there is a pending GET_WORK. */ 606 uint64_t pend_new_work:1; 607 /* 608 * When pend_new_work is set, this bit indicates that 609 * the wait bit was set. 610 */ 611 uint64_t pend_new_work_wait:1; 612 /* Set when there is a pending NULL_RD. */ 613 uint64_t pend_null_rd:1; 614 /* Set when there is a pending CLR_NSCHED. */ 615 uint64_t pend_nosched_clr:1; 616 uint64_t reserved_51:1; 617 /* This is the index when pend_nosched_clr is set. */ 618 uint64_t pend_index:11; 619 /* 620 * This is the new_grp when (pend_desched AND 621 * pend_desched_switch) is set. 622 */ 623 uint64_t pend_grp:4; 624 /* This is the wqp when pend_nosched_clr is set. */ 625 uint64_t pend_wqp:36; 626 #else 627 uint64_t pend_wqp:36; 628 uint64_t pend_grp:4; 629 uint64_t pend_index:11; 630 uint64_t reserved_51:1; 631 uint64_t pend_nosched_clr:1; 632 uint64_t pend_null_rd:1; 633 uint64_t pend_new_work_wait:1; 634 uint64_t pend_new_work:1; 635 uint64_t pend_nosched:1; 636 uint64_t pend_desched_switch:1; 637 uint64_t pend_desched:1; 638 uint64_t pend_switch_null:1; 639 uint64_t pend_switch_full:1; 640 uint64_t pend_switch:1; 641 uint64_t reserved_62_63:2; 642 #endif 643 } s_sstatus1; 644 645 /** 646 * Result for a POW Status Load (when get_cur==1, get_wqp==0, and 647 * get_rev==0) 648 */ 649 struct { 650 #ifdef __BIG_ENDIAN_BITFIELD 651 uint64_t reserved_62_63:2; 652 /* 653 * Points to the next POW entry in the tag list when 654 * tail == 0 (and tag_type is not NULL or NULL_NULL). 655 */ 656 uint64_t link_index:11; 657 /* The POW entry attached to the core. */ 658 uint64_t index:11; 659 /* 660 * The group attached to the core (updated when new 661 * tag list entered on SWTAG_FULL). 662 */ 663 uint64_t grp:4; 664 /* 665 * Set when this POW entry is at the head of its tag 666 * list (also set when in the NULL or NULL_NULL 667 * state). 668 */ 669 uint64_t head:1; 670 /* 671 * Set when this POW entry is at the tail of its tag 672 * list (also set when in the NULL or NULL_NULL 673 * state). 674 */ 675 uint64_t tail:1; 676 /* 677 * The tag type attached to the core (updated when new 678 * tag list entered on SWTAG, SWTAG_FULL, or 679 * SWTAG_DESCHED). 680 */ 681 uint64_t tag_type:2; 682 /* 683 * The tag attached to the core (updated when new tag 684 * list entered on SWTAG, SWTAG_FULL, or 685 * SWTAG_DESCHED). 686 */ 687 uint64_t tag:32; 688 #else 689 uint64_t tag:32; 690 uint64_t tag_type:2; 691 uint64_t tail:1; 692 uint64_t head:1; 693 uint64_t grp:4; 694 uint64_t index:11; 695 uint64_t link_index:11; 696 uint64_t reserved_62_63:2; 697 #endif 698 } s_sstatus2; 699 700 /** 701 * Result for a POW Status Load (when get_cur==1, get_wqp==0, and get_rev==1) 702 */ 703 struct { 704 #ifdef __BIG_ENDIAN_BITFIELD 705 uint64_t reserved_62_63:2; 706 /* 707 * Points to the prior POW entry in the tag list when 708 * head == 0 (and tag_type is not NULL or 709 * NULL_NULL). This field is unpredictable when the 710 * core's state is NULL or NULL_NULL. 711 */ 712 uint64_t revlink_index:11; 713 /* The POW entry attached to the core. */ 714 uint64_t index:11; 715 /* 716 * The group attached to the core (updated when new 717 * tag list entered on SWTAG_FULL). 718 */ 719 uint64_t grp:4; 720 /* Set when this POW entry is at the head of its tag 721 * list (also set when in the NULL or NULL_NULL 722 * state). 723 */ 724 uint64_t head:1; 725 /* 726 * Set when this POW entry is at the tail of its tag 727 * list (also set when in the NULL or NULL_NULL 728 * state). 729 */ 730 uint64_t tail:1; 731 /* 732 * The tag type attached to the core (updated when new 733 * tag list entered on SWTAG, SWTAG_FULL, or 734 * SWTAG_DESCHED). 735 */ 736 uint64_t tag_type:2; 737 /* 738 * The tag attached to the core (updated when new tag 739 * list entered on SWTAG, SWTAG_FULL, or 740 * SWTAG_DESCHED). 741 */ 742 uint64_t tag:32; 743 #else 744 uint64_t tag:32; 745 uint64_t tag_type:2; 746 uint64_t tail:1; 747 uint64_t head:1; 748 uint64_t grp:4; 749 uint64_t index:11; 750 uint64_t revlink_index:11; 751 uint64_t reserved_62_63:2; 752 #endif 753 } s_sstatus3; 754 755 /** 756 * Result for a POW Status Load (when get_cur==1, get_wqp==1, and 757 * get_rev==0) 758 */ 759 struct { 760 #ifdef __BIG_ENDIAN_BITFIELD 761 uint64_t reserved_62_63:2; 762 /* 763 * Points to the next POW entry in the tag list when 764 * tail == 0 (and tag_type is not NULL or NULL_NULL). 765 */ 766 uint64_t link_index:11; 767 /* The POW entry attached to the core. */ 768 uint64_t index:11; 769 /* 770 * The group attached to the core (updated when new 771 * tag list entered on SWTAG_FULL). 772 */ 773 uint64_t grp:4; 774 /* 775 * The wqp attached to the core (updated when new tag 776 * list entered on SWTAG_FULL). 777 */ 778 uint64_t wqp:36; 779 #else 780 uint64_t wqp:36; 781 uint64_t grp:4; 782 uint64_t index:11; 783 uint64_t link_index:11; 784 uint64_t reserved_62_63:2; 785 #endif 786 } s_sstatus4; 787 788 /** 789 * Result for a POW Status Load (when get_cur==1, get_wqp==1, and 790 * get_rev==1) 791 */ 792 struct { 793 #ifdef __BIG_ENDIAN_BITFIELD 794 uint64_t reserved_62_63:2; 795 /* 796 * Points to the prior POW entry in the tag list when 797 * head == 0 (and tag_type is not NULL or 798 * NULL_NULL). This field is unpredictable when the 799 * core's state is NULL or NULL_NULL. 800 */ 801 uint64_t revlink_index:11; 802 /* The POW entry attached to the core. */ 803 uint64_t index:11; 804 /* 805 * The group attached to the core (updated when new 806 * tag list entered on SWTAG_FULL). 807 */ 808 uint64_t grp:4; 809 /* 810 * The wqp attached to the core (updated when new tag 811 * list entered on SWTAG_FULL). 812 */ 813 uint64_t wqp:36; 814 #else 815 uint64_t wqp:36; 816 uint64_t grp:4; 817 uint64_t index:11; 818 uint64_t revlink_index:11; 819 uint64_t reserved_62_63:2; 820 #endif 821 } s_sstatus5; 822 823 /** 824 * Result For POW Memory Load (get_des == 0 and get_wqp == 0) 825 */ 826 struct { 827 #ifdef __BIG_ENDIAN_BITFIELD 828 uint64_t reserved_51_63:13; 829 /* 830 * The next entry in the input, free, descheduled_head 831 * list (unpredictable if entry is the tail of the 832 * list). 833 */ 834 uint64_t next_index:11; 835 /* The group of the POW entry. */ 836 uint64_t grp:4; 837 uint64_t reserved_35:1; 838 /* 839 * Set when this POW entry is at the tail of its tag 840 * list (also set when in the NULL or NULL_NULL 841 * state). 842 */ 843 uint64_t tail:1; 844 /* The tag type of the POW entry. */ 845 uint64_t tag_type:2; 846 /* The tag of the POW entry. */ 847 uint64_t tag:32; 848 #else 849 uint64_t tag:32; 850 uint64_t tag_type:2; 851 uint64_t tail:1; 852 uint64_t reserved_35:1; 853 uint64_t grp:4; 854 uint64_t next_index:11; 855 uint64_t reserved_51_63:13; 856 #endif 857 } s_smemload0; 858 859 /** 860 * Result For POW Memory Load (get_des == 0 and get_wqp == 1) 861 */ 862 struct { 863 #ifdef __BIG_ENDIAN_BITFIELD 864 uint64_t reserved_51_63:13; 865 /* 866 * The next entry in the input, free, descheduled_head 867 * list (unpredictable if entry is the tail of the 868 * list). 869 */ 870 uint64_t next_index:11; 871 /* The group of the POW entry. */ 872 uint64_t grp:4; 873 /* The WQP held in the POW entry. */ 874 uint64_t wqp:36; 875 #else 876 uint64_t wqp:36; 877 uint64_t grp:4; 878 uint64_t next_index:11; 879 uint64_t reserved_51_63:13; 880 #endif 881 } s_smemload1; 882 883 /** 884 * Result For POW Memory Load (get_des == 1) 885 */ 886 struct { 887 #ifdef __BIG_ENDIAN_BITFIELD 888 uint64_t reserved_51_63:13; 889 /* 890 * The next entry in the tag list connected to the 891 * descheduled head. 892 */ 893 uint64_t fwd_index:11; 894 /* The group of the POW entry. */ 895 uint64_t grp:4; 896 /* The nosched bit for the POW entry. */ 897 uint64_t nosched:1; 898 /* There is a pending tag switch */ 899 uint64_t pend_switch:1; 900 /* 901 * The next tag type for the new tag list when 902 * pend_switch is set. 903 */ 904 uint64_t pend_type:2; 905 /* 906 * The next tag for the new tag list when pend_switch 907 * is set. 908 */ 909 uint64_t pend_tag:32; 910 #else 911 uint64_t pend_tag:32; 912 uint64_t pend_type:2; 913 uint64_t pend_switch:1; 914 uint64_t nosched:1; 915 uint64_t grp:4; 916 uint64_t fwd_index:11; 917 uint64_t reserved_51_63:13; 918 #endif 919 } s_smemload2; 920 921 /** 922 * Result For POW Index/Pointer Load (get_rmt == 0/get_des_get_tail == 0) 923 */ 924 struct { 925 #ifdef __BIG_ENDIAN_BITFIELD 926 uint64_t reserved_52_63:12; 927 /* 928 * set when there is one or more POW entries on the 929 * free list. 930 */ 931 uint64_t free_val:1; 932 /* 933 * set when there is exactly one POW entry on the free 934 * list. 935 */ 936 uint64_t free_one:1; 937 uint64_t reserved_49:1; 938 /* 939 * when free_val is set, indicates the first entry on 940 * the free list. 941 */ 942 uint64_t free_head:11; 943 uint64_t reserved_37:1; 944 /* 945 * when free_val is set, indicates the last entry on 946 * the free list. 947 */ 948 uint64_t free_tail:11; 949 /* 950 * set when there is one or more POW entries on the 951 * input Q list selected by qosgrp. 952 */ 953 uint64_t loc_val:1; 954 /* 955 * set when there is exactly one POW entry on the 956 * input Q list selected by qosgrp. 957 */ 958 uint64_t loc_one:1; 959 uint64_t reserved_23:1; 960 /* 961 * when loc_val is set, indicates the first entry on 962 * the input Q list selected by qosgrp. 963 */ 964 uint64_t loc_head:11; 965 uint64_t reserved_11:1; 966 /* 967 * when loc_val is set, indicates the last entry on 968 * the input Q list selected by qosgrp. 969 */ 970 uint64_t loc_tail:11; 971 #else 972 uint64_t loc_tail:11; 973 uint64_t reserved_11:1; 974 uint64_t loc_head:11; 975 uint64_t reserved_23:1; 976 uint64_t loc_one:1; 977 uint64_t loc_val:1; 978 uint64_t free_tail:11; 979 uint64_t reserved_37:1; 980 uint64_t free_head:11; 981 uint64_t reserved_49:1; 982 uint64_t free_one:1; 983 uint64_t free_val:1; 984 uint64_t reserved_52_63:12; 985 #endif 986 } sindexload0; 987 988 /** 989 * Result For POW Index/Pointer Load (get_rmt == 0/get_des_get_tail == 1) 990 */ 991 struct { 992 #ifdef __BIG_ENDIAN_BITFIELD 993 uint64_t reserved_52_63:12; 994 /* 995 * set when there is one or more POW entries on the 996 * nosched list. 997 */ 998 uint64_t nosched_val:1; 999 /* 1000 * set when there is exactly one POW entry on the 1001 * nosched list. 1002 */ 1003 uint64_t nosched_one:1; 1004 uint64_t reserved_49:1; 1005 /* 1006 * when nosched_val is set, indicates the first entry 1007 * on the nosched list. 1008 */ 1009 uint64_t nosched_head:11; 1010 uint64_t reserved_37:1; 1011 /* 1012 * when nosched_val is set, indicates the last entry 1013 * on the nosched list. 1014 */ 1015 uint64_t nosched_tail:11; 1016 /* 1017 * set when there is one or more descheduled heads on 1018 * the descheduled list selected by qosgrp. 1019 */ 1020 uint64_t des_val:1; 1021 /* 1022 * set when there is exactly one descheduled head on 1023 * the descheduled list selected by qosgrp. 1024 */ 1025 uint64_t des_one:1; 1026 uint64_t reserved_23:1; 1027 /* 1028 * when des_val is set, indicates the first 1029 * descheduled head on the descheduled list selected 1030 * by qosgrp. 1031 */ 1032 uint64_t des_head:11; 1033 uint64_t reserved_11:1; 1034 /* 1035 * when des_val is set, indicates the last descheduled 1036 * head on the descheduled list selected by qosgrp. 1037 */ 1038 uint64_t des_tail:11; 1039 #else 1040 uint64_t des_tail:11; 1041 uint64_t reserved_11:1; 1042 uint64_t des_head:11; 1043 uint64_t reserved_23:1; 1044 uint64_t des_one:1; 1045 uint64_t des_val:1; 1046 uint64_t nosched_tail:11; 1047 uint64_t reserved_37:1; 1048 uint64_t nosched_head:11; 1049 uint64_t reserved_49:1; 1050 uint64_t nosched_one:1; 1051 uint64_t nosched_val:1; 1052 uint64_t reserved_52_63:12; 1053 #endif 1054 } sindexload1; 1055 1056 /** 1057 * Result For POW Index/Pointer Load (get_rmt == 1/get_des_get_tail == 0) 1058 */ 1059 struct { 1060 #ifdef __BIG_ENDIAN_BITFIELD 1061 uint64_t reserved_39_63:25; 1062 /* 1063 * Set when this DRAM list is the current head 1064 * (i.e. is the next to be reloaded when the POW 1065 * hardware reloads a POW entry from DRAM). The POW 1066 * hardware alternates between the two DRAM lists 1067 * associated with a QOS level when it reloads work 1068 * from DRAM into the POW unit. 1069 */ 1070 uint64_t rmt_is_head:1; 1071 /* 1072 * Set when the DRAM portion of the input Q list 1073 * selected by qosgrp contains one or more pieces of 1074 * work. 1075 */ 1076 uint64_t rmt_val:1; 1077 /* 1078 * Set when the DRAM portion of the input Q list 1079 * selected by qosgrp contains exactly one piece of 1080 * work. 1081 */ 1082 uint64_t rmt_one:1; 1083 /* 1084 * When rmt_val is set, indicates the first piece of 1085 * work on the DRAM input Q list selected by 1086 * qosgrp. 1087 */ 1088 uint64_t rmt_head:36; 1089 #else 1090 uint64_t rmt_head:36; 1091 uint64_t rmt_one:1; 1092 uint64_t rmt_val:1; 1093 uint64_t rmt_is_head:1; 1094 uint64_t reserved_39_63:25; 1095 #endif 1096 } sindexload2; 1097 1098 /** 1099 * Result For POW Index/Pointer Load (get_rmt == 1100 * 1/get_des_get_tail == 1) 1101 */ 1102 struct { 1103 #ifdef __BIG_ENDIAN_BITFIELD 1104 uint64_t reserved_39_63:25; 1105 /* 1106 * set when this DRAM list is the current head 1107 * (i.e. is the next to be reloaded when the POW 1108 * hardware reloads a POW entry from DRAM). The POW 1109 * hardware alternates between the two DRAM lists 1110 * associated with a QOS level when it reloads work 1111 * from DRAM into the POW unit. 1112 */ 1113 uint64_t rmt_is_head:1; 1114 /* 1115 * set when the DRAM portion of the input Q list 1116 * selected by qosgrp contains one or more pieces of 1117 * work. 1118 */ 1119 uint64_t rmt_val:1; 1120 /* 1121 * set when the DRAM portion of the input Q list 1122 * selected by qosgrp contains exactly one piece of 1123 * work. 1124 */ 1125 uint64_t rmt_one:1; 1126 /* 1127 * when rmt_val is set, indicates the last piece of 1128 * work on the DRAM input Q list selected by 1129 * qosgrp. 1130 */ 1131 uint64_t rmt_tail:36; 1132 #else 1133 uint64_t rmt_tail:36; 1134 uint64_t rmt_one:1; 1135 uint64_t rmt_val:1; 1136 uint64_t rmt_is_head:1; 1137 uint64_t reserved_39_63:25; 1138 #endif 1139 } sindexload3; 1140 1141 /** 1142 * Response to NULL_RD request loads 1143 */ 1144 struct { 1145 #ifdef __BIG_ENDIAN_BITFIELD 1146 uint64_t unused:62; 1147 /* of type cvmx_pow_tag_type_t. state is one of the 1148 * following: 1149 * 1150 * - CVMX_POW_TAG_TYPE_ORDERED 1151 * - CVMX_POW_TAG_TYPE_ATOMIC 1152 * - CVMX_POW_TAG_TYPE_NULL 1153 * - CVMX_POW_TAG_TYPE_NULL_NULL 1154 */ 1155 uint64_t state:2; 1156 #else 1157 uint64_t state:2; 1158 uint64_t unused:62; 1159 #endif 1160 } s_null_rd; 1161 1162 } cvmx_pow_tag_load_resp_t; 1163 1164 /** 1165 * This structure describes the address used for stores to the POW. 1166 * The store address is meaningful on stores to the POW. The 1167 * hardware assumes that an aligned 64-bit store was used for all 1168 * these stores. Note the assumption that the work queue entry is 1169 * aligned on an 8-byte boundary (since the low-order 3 address bits 1170 * must be zero). Note that not all fields are used by all 1171 * operations. 1172 * 1173 * NOTE: The following is the behavior of the pending switch bit at the PP 1174 * for POW stores (i.e. when did<7:3> == 0xc) 1175 * - did<2:0> == 0 => pending switch bit is set 1176 * - did<2:0> == 1 => no affect on the pending switch bit 1177 * - did<2:0> == 3 => pending switch bit is cleared 1178 * - did<2:0> == 7 => no affect on the pending switch bit 1179 * - did<2:0> == others => must not be used 1180 * - No other loads/stores have an affect on the pending switch bit 1181 * - The switch bus from POW can clear the pending switch bit 1182 * 1183 * NOTE: did<2:0> == 2 is used by the HW for a special single-cycle 1184 * ADDWQ command that only contains the pointer). SW must never use 1185 * did<2:0> == 2. 1186 */ 1187 typedef union { 1188 /** 1189 * Unsigned 64 bit integer representation of store address 1190 */ 1191 uint64_t u64; 1192 1193 struct { 1194 #ifdef __BIG_ENDIAN_BITFIELD 1195 /* Memory region. Should be CVMX_IO_SEG in most cases */ 1196 uint64_t mem_reg:2; 1197 uint64_t reserved_49_61:13; /* Must be zero */ 1198 uint64_t is_io:1; /* Must be one */ 1199 /* Device ID of POW. Note that different sub-dids are used. */ 1200 uint64_t did:8; 1201 uint64_t reserved_36_39:4; /* Must be zero */ 1202 /* Address field. addr<2:0> must be zero */ 1203 uint64_t addr:36; 1204 #else 1205 uint64_t addr:36; 1206 uint64_t reserved_36_39:4; 1207 uint64_t did:8; 1208 uint64_t is_io:1; 1209 uint64_t reserved_49_61:13; 1210 uint64_t mem_reg:2; 1211 #endif 1212 } stag; 1213 } cvmx_pow_tag_store_addr_t; 1214 1215 /** 1216 * decode of the store data when an IOBDMA SENDSINGLE is sent to POW 1217 */ 1218 typedef union { 1219 uint64_t u64; 1220 1221 struct { 1222 #ifdef __BIG_ENDIAN_BITFIELD 1223 /* 1224 * the (64-bit word) location in scratchpad to write 1225 * to (if len != 0) 1226 */ 1227 uint64_t scraddr:8; 1228 /* the number of words in the response (0 => no response) */ 1229 uint64_t len:8; 1230 /* the ID of the device on the non-coherent bus */ 1231 uint64_t did:8; 1232 uint64_t unused:36; 1233 /* if set, don't return load response until work is available */ 1234 uint64_t wait:1; 1235 uint64_t unused2:3; 1236 #else 1237 uint64_t unused2:3; 1238 uint64_t wait:1; 1239 uint64_t unused:36; 1240 uint64_t did:8; 1241 uint64_t len:8; 1242 uint64_t scraddr:8; 1243 #endif 1244 } s; 1245 1246 } cvmx_pow_iobdma_store_t; 1247 1248 /* CSR typedefs have been moved to cvmx-csr-*.h */ 1249 1250 /** 1251 * Get the POW tag for this core. This returns the current 1252 * tag type, tag, group, and POW entry index associated with 1253 * this core. Index is only valid if the tag type isn't NULL_NULL. 1254 * If a tag switch is pending this routine returns the tag before 1255 * the tag switch, not after. 1256 * 1257 * Returns Current tag 1258 */ 1259 static inline cvmx_pow_tag_req_t cvmx_pow_get_current_tag(void) 1260 { 1261 cvmx_pow_load_addr_t load_addr; 1262 cvmx_pow_tag_load_resp_t load_resp; 1263 cvmx_pow_tag_req_t result; 1264 1265 load_addr.u64 = 0; 1266 load_addr.sstatus.mem_region = CVMX_IO_SEG; 1267 load_addr.sstatus.is_io = 1; 1268 load_addr.sstatus.did = CVMX_OCT_DID_TAG_TAG1; 1269 load_addr.sstatus.coreid = cvmx_get_core_num(); 1270 load_addr.sstatus.get_cur = 1; 1271 load_resp.u64 = cvmx_read_csr(load_addr.u64); 1272 result.u64 = 0; 1273 result.s.grp = load_resp.s_sstatus2.grp; 1274 result.s.index = load_resp.s_sstatus2.index; 1275 result.s.type = load_resp.s_sstatus2.tag_type; 1276 result.s.tag = load_resp.s_sstatus2.tag; 1277 return result; 1278 } 1279 1280 /** 1281 * Get the POW WQE for this core. This returns the work queue 1282 * entry currently associated with this core. 1283 * 1284 * Returns WQE pointer 1285 */ 1286 static inline struct cvmx_wqe *cvmx_pow_get_current_wqp(void) 1287 { 1288 cvmx_pow_load_addr_t load_addr; 1289 cvmx_pow_tag_load_resp_t load_resp; 1290 1291 load_addr.u64 = 0; 1292 load_addr.sstatus.mem_region = CVMX_IO_SEG; 1293 load_addr.sstatus.is_io = 1; 1294 load_addr.sstatus.did = CVMX_OCT_DID_TAG_TAG1; 1295 load_addr.sstatus.coreid = cvmx_get_core_num(); 1296 load_addr.sstatus.get_cur = 1; 1297 load_addr.sstatus.get_wqp = 1; 1298 load_resp.u64 = cvmx_read_csr(load_addr.u64); 1299 return (struct cvmx_wqe *) cvmx_phys_to_ptr(load_resp.s_sstatus4.wqp); 1300 } 1301 1302 #ifndef CVMX_MF_CHORD 1303 #define CVMX_MF_CHORD(dest) CVMX_RDHWR(dest, 30) 1304 #endif 1305 1306 /** 1307 * Print a warning if a tag switch is pending for this core 1308 * 1309 * @function: Function name checking for a pending tag switch 1310 */ 1311 static inline void __cvmx_pow_warn_if_pending_switch(const char *function) 1312 { 1313 uint64_t switch_complete; 1314 CVMX_MF_CHORD(switch_complete); 1315 if (!switch_complete) 1316 pr_warn("%s called with tag switch in progress\n", function); 1317 } 1318 1319 /** 1320 * Waits for a tag switch to complete by polling the completion bit. 1321 * Note that switches to NULL complete immediately and do not need 1322 * to be waited for. 1323 */ 1324 static inline void cvmx_pow_tag_sw_wait(void) 1325 { 1326 const uint64_t MAX_CYCLES = 1ull << 31; 1327 uint64_t switch_complete; 1328 uint64_t start_cycle = cvmx_get_cycle(); 1329 while (1) { 1330 CVMX_MF_CHORD(switch_complete); 1331 if (unlikely(switch_complete)) 1332 break; 1333 if (unlikely(cvmx_get_cycle() > start_cycle + MAX_CYCLES)) { 1334 pr_warn("Tag switch is taking a long time, possible deadlock\n"); 1335 start_cycle = -MAX_CYCLES - 1; 1336 } 1337 } 1338 } 1339 1340 /** 1341 * Synchronous work request. Requests work from the POW. 1342 * This function does NOT wait for previous tag switches to complete, 1343 * so the caller must ensure that there is not a pending tag switch. 1344 * 1345 * @wait: When set, call stalls until work becomes avaiable, or times out. 1346 * If not set, returns immediately. 1347 * 1348 * Returns: the WQE pointer from POW. Returns NULL if no work 1349 * was available. 1350 */ 1351 static inline struct cvmx_wqe *cvmx_pow_work_request_sync_nocheck(cvmx_pow_wait_t 1352 wait) 1353 { 1354 cvmx_pow_load_addr_t ptr; 1355 cvmx_pow_tag_load_resp_t result; 1356 1357 if (CVMX_ENABLE_POW_CHECKS) 1358 __cvmx_pow_warn_if_pending_switch(__func__); 1359 1360 ptr.u64 = 0; 1361 ptr.swork.mem_region = CVMX_IO_SEG; 1362 ptr.swork.is_io = 1; 1363 ptr.swork.did = CVMX_OCT_DID_TAG_SWTAG; 1364 ptr.swork.wait = wait; 1365 1366 result.u64 = cvmx_read_csr(ptr.u64); 1367 1368 if (result.s_work.no_work) 1369 return NULL; 1370 else 1371 return (struct cvmx_wqe *) cvmx_phys_to_ptr(result.s_work.addr); 1372 } 1373 1374 /** 1375 * Synchronous work request. Requests work from the POW. 1376 * This function waits for any previous tag switch to complete before 1377 * requesting the new work. 1378 * 1379 * @wait: When set, call stalls until work becomes avaiable, or times out. 1380 * If not set, returns immediately. 1381 * 1382 * Returns: the WQE pointer from POW. Returns NULL if no work 1383 * was available. 1384 */ 1385 static inline struct cvmx_wqe *cvmx_pow_work_request_sync(cvmx_pow_wait_t wait) 1386 { 1387 if (CVMX_ENABLE_POW_CHECKS) 1388 __cvmx_pow_warn_if_pending_switch(__func__); 1389 1390 /* Must not have a switch pending when requesting work */ 1391 cvmx_pow_tag_sw_wait(); 1392 return cvmx_pow_work_request_sync_nocheck(wait); 1393 1394 } 1395 1396 /** 1397 * Synchronous null_rd request. Requests a switch out of NULL_NULL POW state. 1398 * This function waits for any previous tag switch to complete before 1399 * requesting the null_rd. 1400 * 1401 * Returns: the POW state of type cvmx_pow_tag_type_t. 1402 */ 1403 static inline enum cvmx_pow_tag_type cvmx_pow_work_request_null_rd(void) 1404 { 1405 cvmx_pow_load_addr_t ptr; 1406 cvmx_pow_tag_load_resp_t result; 1407 1408 if (CVMX_ENABLE_POW_CHECKS) 1409 __cvmx_pow_warn_if_pending_switch(__func__); 1410 1411 /* Must not have a switch pending when requesting work */ 1412 cvmx_pow_tag_sw_wait(); 1413 1414 ptr.u64 = 0; 1415 ptr.snull_rd.mem_region = CVMX_IO_SEG; 1416 ptr.snull_rd.is_io = 1; 1417 ptr.snull_rd.did = CVMX_OCT_DID_TAG_NULL_RD; 1418 1419 result.u64 = cvmx_read_csr(ptr.u64); 1420 1421 return (enum cvmx_pow_tag_type) result.s_null_rd.state; 1422 } 1423 1424 /** 1425 * Asynchronous work request. Work is requested from the POW unit, 1426 * and should later be checked with function 1427 * cvmx_pow_work_response_async. This function does NOT wait for 1428 * previous tag switches to complete, so the caller must ensure that 1429 * there is not a pending tag switch. 1430 * 1431 * @scr_addr: Scratch memory address that response will be returned 1432 * to, which is either a valid WQE, or a response with the 1433 * invalid bit set. Byte address, must be 8 byte aligned. 1434 * 1435 * @wait: 1 to cause response to wait for work to become available (or 1436 * timeout), 0 to cause response to return immediately 1437 */ 1438 static inline void cvmx_pow_work_request_async_nocheck(int scr_addr, 1439 cvmx_pow_wait_t wait) 1440 { 1441 cvmx_pow_iobdma_store_t data; 1442 1443 if (CVMX_ENABLE_POW_CHECKS) 1444 __cvmx_pow_warn_if_pending_switch(__func__); 1445 1446 /* scr_addr must be 8 byte aligned */ 1447 data.s.scraddr = scr_addr >> 3; 1448 data.s.len = 1; 1449 data.s.did = CVMX_OCT_DID_TAG_SWTAG; 1450 data.s.wait = wait; 1451 cvmx_send_single(data.u64); 1452 } 1453 1454 /** 1455 * Asynchronous work request. Work is requested from the POW unit, 1456 * and should later be checked with function 1457 * cvmx_pow_work_response_async. This function waits for any previous 1458 * tag switch to complete before requesting the new work. 1459 * 1460 * @scr_addr: Scratch memory address that response will be returned 1461 * to, which is either a valid WQE, or a response with the 1462 * invalid bit set. Byte address, must be 8 byte aligned. 1463 * 1464 * @wait: 1 to cause response to wait for work to become available (or 1465 * timeout), 0 to cause response to return immediately 1466 */ 1467 static inline void cvmx_pow_work_request_async(int scr_addr, 1468 cvmx_pow_wait_t wait) 1469 { 1470 if (CVMX_ENABLE_POW_CHECKS) 1471 __cvmx_pow_warn_if_pending_switch(__func__); 1472 1473 /* Must not have a switch pending when requesting work */ 1474 cvmx_pow_tag_sw_wait(); 1475 cvmx_pow_work_request_async_nocheck(scr_addr, wait); 1476 } 1477 1478 /** 1479 * Gets result of asynchronous work request. Performs a IOBDMA sync 1480 * to wait for the response. 1481 * 1482 * @scr_addr: Scratch memory address to get result from Byte address, 1483 * must be 8 byte aligned. 1484 * 1485 * Returns: the WQE from the scratch register, or NULL if no 1486 * work was available. 1487 */ 1488 static inline struct cvmx_wqe *cvmx_pow_work_response_async(int scr_addr) 1489 { 1490 cvmx_pow_tag_load_resp_t result; 1491 1492 CVMX_SYNCIOBDMA; 1493 result.u64 = cvmx_scratch_read64(scr_addr); 1494 1495 if (result.s_work.no_work) 1496 return NULL; 1497 else 1498 return (struct cvmx_wqe *) cvmx_phys_to_ptr(result.s_work.addr); 1499 } 1500 1501 /** 1502 * Checks if a work queue entry pointer returned by a work 1503 * request is valid. It may be invalid due to no work 1504 * being available or due to a timeout. 1505 * 1506 * @wqe_ptr: pointer to a work queue entry returned by the POW 1507 * 1508 * Returns 0 if pointer is valid 1509 * 1 if invalid (no work was returned) 1510 */ 1511 static inline uint64_t cvmx_pow_work_invalid(struct cvmx_wqe *wqe_ptr) 1512 { 1513 return wqe_ptr == NULL; 1514 } 1515 1516 /** 1517 * Starts a tag switch to the provided tag value and tag type. 1518 * Completion for the tag switch must be checked for separately. This 1519 * function does NOT update the work queue entry in dram to match tag 1520 * value and type, so the application must keep track of these if they 1521 * are important to the application. This tag switch command must not 1522 * be used for switches to NULL, as the tag switch pending bit will be 1523 * set by the switch request, but never cleared by the hardware. 1524 * 1525 * NOTE: This should not be used when switching from a NULL tag. Use 1526 * cvmx_pow_tag_sw_full() instead. 1527 * 1528 * This function does no checks, so the caller must ensure that any 1529 * previous tag switch has completed. 1530 * 1531 * @tag: new tag value 1532 * @tag_type: new tag type (ordered or atomic) 1533 */ 1534 static inline void cvmx_pow_tag_sw_nocheck(uint32_t tag, 1535 enum cvmx_pow_tag_type tag_type) 1536 { 1537 cvmx_addr_t ptr; 1538 cvmx_pow_tag_req_t tag_req; 1539 1540 if (CVMX_ENABLE_POW_CHECKS) { 1541 cvmx_pow_tag_req_t current_tag; 1542 __cvmx_pow_warn_if_pending_switch(__func__); 1543 current_tag = cvmx_pow_get_current_tag(); 1544 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL_NULL) 1545 pr_warn("%s called with NULL_NULL tag\n", __func__); 1546 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL) 1547 pr_warn("%s called with NULL tag\n", __func__); 1548 if ((current_tag.s.type == tag_type) 1549 && (current_tag.s.tag == tag)) 1550 pr_warn("%s called to perform a tag switch to the same tag\n", 1551 __func__); 1552 if (tag_type == CVMX_POW_TAG_TYPE_NULL) 1553 pr_warn("%s called to perform a tag switch to NULL. Use cvmx_pow_tag_sw_null() instead\n", 1554 __func__); 1555 } 1556 1557 /* 1558 * Note that WQE in DRAM is not updated here, as the POW does 1559 * not read from DRAM once the WQE is in flight. See hardware 1560 * manual for complete details. It is the application's 1561 * responsibility to keep track of the current tag value if 1562 * that is important. 1563 */ 1564 1565 tag_req.u64 = 0; 1566 tag_req.s.op = CVMX_POW_TAG_OP_SWTAG; 1567 tag_req.s.tag = tag; 1568 tag_req.s.type = tag_type; 1569 1570 ptr.u64 = 0; 1571 ptr.sio.mem_region = CVMX_IO_SEG; 1572 ptr.sio.is_io = 1; 1573 ptr.sio.did = CVMX_OCT_DID_TAG_SWTAG; 1574 1575 /* once this store arrives at POW, it will attempt the switch 1576 software must wait for the switch to complete separately */ 1577 cvmx_write_io(ptr.u64, tag_req.u64); 1578 } 1579 1580 /** 1581 * Starts a tag switch to the provided tag value and tag type. 1582 * Completion for the tag switch must be checked for separately. This 1583 * function does NOT update the work queue entry in dram to match tag 1584 * value and type, so the application must keep track of these if they 1585 * are important to the application. This tag switch command must not 1586 * be used for switches to NULL, as the tag switch pending bit will be 1587 * set by the switch request, but never cleared by the hardware. 1588 * 1589 * NOTE: This should not be used when switching from a NULL tag. Use 1590 * cvmx_pow_tag_sw_full() instead. 1591 * 1592 * This function waits for any previous tag switch to complete, and also 1593 * displays an error on tag switches to NULL. 1594 * 1595 * @tag: new tag value 1596 * @tag_type: new tag type (ordered or atomic) 1597 */ 1598 static inline void cvmx_pow_tag_sw(uint32_t tag, 1599 enum cvmx_pow_tag_type tag_type) 1600 { 1601 if (CVMX_ENABLE_POW_CHECKS) 1602 __cvmx_pow_warn_if_pending_switch(__func__); 1603 1604 /* 1605 * Note that WQE in DRAM is not updated here, as the POW does 1606 * not read from DRAM once the WQE is in flight. See hardware 1607 * manual for complete details. It is the application's 1608 * responsibility to keep track of the current tag value if 1609 * that is important. 1610 */ 1611 1612 /* 1613 * Ensure that there is not a pending tag switch, as a tag 1614 * switch cannot be started if a previous switch is still 1615 * pending. 1616 */ 1617 cvmx_pow_tag_sw_wait(); 1618 cvmx_pow_tag_sw_nocheck(tag, tag_type); 1619 } 1620 1621 /** 1622 * Starts a tag switch to the provided tag value and tag type. 1623 * Completion for the tag switch must be checked for separately. This 1624 * function does NOT update the work queue entry in dram to match tag 1625 * value and type, so the application must keep track of these if they 1626 * are important to the application. This tag switch command must not 1627 * be used for switches to NULL, as the tag switch pending bit will be 1628 * set by the switch request, but never cleared by the hardware. 1629 * 1630 * This function must be used for tag switches from NULL. 1631 * 1632 * This function does no checks, so the caller must ensure that any 1633 * previous tag switch has completed. 1634 * 1635 * @wqp: pointer to work queue entry to submit. This entry is 1636 * updated to match the other parameters 1637 * @tag: tag value to be assigned to work queue entry 1638 * @tag_type: type of tag 1639 * @group: group value for the work queue entry. 1640 */ 1641 static inline void cvmx_pow_tag_sw_full_nocheck(struct cvmx_wqe *wqp, uint32_t tag, 1642 enum cvmx_pow_tag_type tag_type, 1643 uint64_t group) 1644 { 1645 cvmx_addr_t ptr; 1646 cvmx_pow_tag_req_t tag_req; 1647 1648 if (CVMX_ENABLE_POW_CHECKS) { 1649 cvmx_pow_tag_req_t current_tag; 1650 __cvmx_pow_warn_if_pending_switch(__func__); 1651 current_tag = cvmx_pow_get_current_tag(); 1652 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL_NULL) 1653 pr_warn("%s called with NULL_NULL tag\n", __func__); 1654 if ((current_tag.s.type == tag_type) 1655 && (current_tag.s.tag == tag)) 1656 pr_warn("%s called to perform a tag switch to the same tag\n", 1657 __func__); 1658 if (tag_type == CVMX_POW_TAG_TYPE_NULL) 1659 pr_warn("%s called to perform a tag switch to NULL. Use cvmx_pow_tag_sw_null() instead\n", 1660 __func__); 1661 if (wqp != cvmx_phys_to_ptr(0x80)) 1662 if (wqp != cvmx_pow_get_current_wqp()) 1663 pr_warn("%s passed WQE(%p) doesn't match the address in the POW(%p)\n", 1664 __func__, wqp, 1665 cvmx_pow_get_current_wqp()); 1666 } 1667 1668 /* 1669 * Note that WQE in DRAM is not updated here, as the POW does 1670 * not read from DRAM once the WQE is in flight. See hardware 1671 * manual for complete details. It is the application's 1672 * responsibility to keep track of the current tag value if 1673 * that is important. 1674 */ 1675 1676 tag_req.u64 = 0; 1677 tag_req.s.op = CVMX_POW_TAG_OP_SWTAG_FULL; 1678 tag_req.s.tag = tag; 1679 tag_req.s.type = tag_type; 1680 tag_req.s.grp = group; 1681 1682 ptr.u64 = 0; 1683 ptr.sio.mem_region = CVMX_IO_SEG; 1684 ptr.sio.is_io = 1; 1685 ptr.sio.did = CVMX_OCT_DID_TAG_SWTAG; 1686 ptr.sio.offset = CAST64(wqp); 1687 1688 /* 1689 * once this store arrives at POW, it will attempt the switch 1690 * software must wait for the switch to complete separately. 1691 */ 1692 cvmx_write_io(ptr.u64, tag_req.u64); 1693 } 1694 1695 /** 1696 * Starts a tag switch to the provided tag value and tag type. 1697 * Completion for the tag switch must be checked for separately. This 1698 * function does NOT update the work queue entry in dram to match tag 1699 * value and type, so the application must keep track of these if they 1700 * are important to the application. This tag switch command must not 1701 * be used for switches to NULL, as the tag switch pending bit will be 1702 * set by the switch request, but never cleared by the hardware. 1703 * 1704 * This function must be used for tag switches from NULL. 1705 * 1706 * This function waits for any pending tag switches to complete 1707 * before requesting the tag switch. 1708 * 1709 * @wqp: pointer to work queue entry to submit. This entry is updated 1710 * to match the other parameters 1711 * @tag: tag value to be assigned to work queue entry 1712 * @tag_type: type of tag 1713 * @group: group value for the work queue entry. 1714 */ 1715 static inline void cvmx_pow_tag_sw_full(struct cvmx_wqe *wqp, uint32_t tag, 1716 enum cvmx_pow_tag_type tag_type, 1717 uint64_t group) 1718 { 1719 if (CVMX_ENABLE_POW_CHECKS) 1720 __cvmx_pow_warn_if_pending_switch(__func__); 1721 1722 /* 1723 * Ensure that there is not a pending tag switch, as a tag 1724 * switch cannot be started if a previous switch is still 1725 * pending. 1726 */ 1727 cvmx_pow_tag_sw_wait(); 1728 cvmx_pow_tag_sw_full_nocheck(wqp, tag, tag_type, group); 1729 } 1730 1731 /** 1732 * Switch to a NULL tag, which ends any ordering or 1733 * synchronization provided by the POW for the current 1734 * work queue entry. This operation completes immediately, 1735 * so completion should not be waited for. 1736 * This function does NOT wait for previous tag switches to complete, 1737 * so the caller must ensure that any previous tag switches have completed. 1738 */ 1739 static inline void cvmx_pow_tag_sw_null_nocheck(void) 1740 { 1741 cvmx_addr_t ptr; 1742 cvmx_pow_tag_req_t tag_req; 1743 1744 if (CVMX_ENABLE_POW_CHECKS) { 1745 cvmx_pow_tag_req_t current_tag; 1746 __cvmx_pow_warn_if_pending_switch(__func__); 1747 current_tag = cvmx_pow_get_current_tag(); 1748 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL_NULL) 1749 pr_warn("%s called with NULL_NULL tag\n", __func__); 1750 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL) 1751 pr_warn("%s called when we already have a NULL tag\n", 1752 __func__); 1753 } 1754 1755 tag_req.u64 = 0; 1756 tag_req.s.op = CVMX_POW_TAG_OP_SWTAG; 1757 tag_req.s.type = CVMX_POW_TAG_TYPE_NULL; 1758 1759 ptr.u64 = 0; 1760 ptr.sio.mem_region = CVMX_IO_SEG; 1761 ptr.sio.is_io = 1; 1762 ptr.sio.did = CVMX_OCT_DID_TAG_TAG1; 1763 1764 cvmx_write_io(ptr.u64, tag_req.u64); 1765 1766 /* switch to NULL completes immediately */ 1767 } 1768 1769 /** 1770 * Switch to a NULL tag, which ends any ordering or 1771 * synchronization provided by the POW for the current 1772 * work queue entry. This operation completes immediately, 1773 * so completion should not be waited for. 1774 * This function waits for any pending tag switches to complete 1775 * before requesting the switch to NULL. 1776 */ 1777 static inline void cvmx_pow_tag_sw_null(void) 1778 { 1779 if (CVMX_ENABLE_POW_CHECKS) 1780 __cvmx_pow_warn_if_pending_switch(__func__); 1781 1782 /* 1783 * Ensure that there is not a pending tag switch, as a tag 1784 * switch cannot be started if a previous switch is still 1785 * pending. 1786 */ 1787 cvmx_pow_tag_sw_wait(); 1788 cvmx_pow_tag_sw_null_nocheck(); 1789 1790 /* switch to NULL completes immediately */ 1791 } 1792 1793 /** 1794 * Submits work to an input queue. This function updates the work 1795 * queue entry in DRAM to match the arguments given. Note that the 1796 * tag provided is for the work queue entry submitted, and is 1797 * unrelated to the tag that the core currently holds. 1798 * 1799 * @wqp: pointer to work queue entry to submit. This entry is 1800 * updated to match the other parameters 1801 * @tag: tag value to be assigned to work queue entry 1802 * @tag_type: type of tag 1803 * @qos: Input queue to add to. 1804 * @grp: group value for the work queue entry. 1805 */ 1806 static inline void cvmx_pow_work_submit(struct cvmx_wqe *wqp, uint32_t tag, 1807 enum cvmx_pow_tag_type tag_type, 1808 uint64_t qos, uint64_t grp) 1809 { 1810 cvmx_addr_t ptr; 1811 cvmx_pow_tag_req_t tag_req; 1812 1813 wqp->word1.tag = tag; 1814 wqp->word1.tag_type = tag_type; 1815 1816 cvmx_wqe_set_qos(wqp, qos); 1817 cvmx_wqe_set_grp(wqp, grp); 1818 1819 tag_req.u64 = 0; 1820 tag_req.s.op = CVMX_POW_TAG_OP_ADDWQ; 1821 tag_req.s.type = tag_type; 1822 tag_req.s.tag = tag; 1823 tag_req.s.qos = qos; 1824 tag_req.s.grp = grp; 1825 1826 ptr.u64 = 0; 1827 ptr.sio.mem_region = CVMX_IO_SEG; 1828 ptr.sio.is_io = 1; 1829 ptr.sio.did = CVMX_OCT_DID_TAG_TAG1; 1830 ptr.sio.offset = cvmx_ptr_to_phys(wqp); 1831 1832 /* 1833 * SYNC write to memory before the work submit. This is 1834 * necessary as POW may read values from DRAM at this time. 1835 */ 1836 CVMX_SYNCWS; 1837 cvmx_write_io(ptr.u64, tag_req.u64); 1838 } 1839 1840 /** 1841 * This function sets the group mask for a core. The group mask 1842 * indicates which groups each core will accept work from. There are 1843 * 16 groups. 1844 * 1845 * @core_num: core to apply mask to 1846 * @mask: Group mask. There are 16 groups, so only bits 0-15 are valid, 1847 * representing groups 0-15. 1848 * Each 1 bit in the mask enables the core to accept work from 1849 * the corresponding group. 1850 */ 1851 static inline void cvmx_pow_set_group_mask(uint64_t core_num, uint64_t mask) 1852 { 1853 union cvmx_pow_pp_grp_mskx grp_msk; 1854 1855 grp_msk.u64 = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(core_num)); 1856 grp_msk.s.grp_msk = mask; 1857 cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(core_num), grp_msk.u64); 1858 } 1859 1860 /** 1861 * This function sets POW static priorities for a core. Each input queue has 1862 * an associated priority value. 1863 * 1864 * @core_num: core to apply priorities to 1865 * @priority: Vector of 8 priorities, one per POW Input Queue (0-7). 1866 * Highest priority is 0 and lowest is 7. A priority value 1867 * of 0xF instructs POW to skip the Input Queue when 1868 * scheduling to this specific core. 1869 * NOTE: priorities should not have gaps in values, meaning 1870 * {0,1,1,1,1,1,1,1} is a valid configuration while 1871 * {0,2,2,2,2,2,2,2} is not. 1872 */ 1873 static inline void cvmx_pow_set_priority(uint64_t core_num, 1874 const uint8_t priority[]) 1875 { 1876 /* POW priorities are supported on CN5xxx and later */ 1877 if (!OCTEON_IS_MODEL(OCTEON_CN3XXX)) { 1878 union cvmx_pow_pp_grp_mskx grp_msk; 1879 1880 grp_msk.u64 = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(core_num)); 1881 grp_msk.s.qos0_pri = priority[0]; 1882 grp_msk.s.qos1_pri = priority[1]; 1883 grp_msk.s.qos2_pri = priority[2]; 1884 grp_msk.s.qos3_pri = priority[3]; 1885 grp_msk.s.qos4_pri = priority[4]; 1886 grp_msk.s.qos5_pri = priority[5]; 1887 grp_msk.s.qos6_pri = priority[6]; 1888 grp_msk.s.qos7_pri = priority[7]; 1889 1890 /* Detect gaps between priorities and flag error */ 1891 { 1892 int i; 1893 uint32_t prio_mask = 0; 1894 1895 for (i = 0; i < 8; i++) 1896 if (priority[i] != 0xF) 1897 prio_mask |= 1 << priority[i]; 1898 1899 if (prio_mask ^ ((1 << cvmx_pop(prio_mask)) - 1)) { 1900 pr_err("POW static priorities should be " 1901 "contiguous (0x%llx)\n", 1902 (unsigned long long)prio_mask); 1903 return; 1904 } 1905 } 1906 1907 cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(core_num), grp_msk.u64); 1908 } 1909 } 1910 1911 /** 1912 * Performs a tag switch and then an immediate deschedule. This completes 1913 * immediately, so completion must not be waited for. This function does NOT 1914 * update the wqe in DRAM to match arguments. 1915 * 1916 * This function does NOT wait for any prior tag switches to complete, so the 1917 * calling code must do this. 1918 * 1919 * Note the following CAVEAT of the Octeon HW behavior when 1920 * re-scheduling DE-SCHEDULEd items whose (next) state is 1921 * ORDERED: 1922 * - If there are no switches pending at the time that the 1923 * HW executes the de-schedule, the HW will only re-schedule 1924 * the head of the FIFO associated with the given tag. This 1925 * means that in many respects, the HW treats this ORDERED 1926 * tag as an ATOMIC tag. Note that in the SWTAG_DESCH 1927 * case (to an ORDERED tag), the HW will do the switch 1928 * before the deschedule whenever it is possible to do 1929 * the switch immediately, so it may often look like 1930 * this case. 1931 * - If there is a pending switch to ORDERED at the time 1932 * the HW executes the de-schedule, the HW will perform 1933 * the switch at the time it re-schedules, and will be 1934 * able to reschedule any/all of the entries with the 1935 * same tag. 1936 * Due to this behavior, the RECOMMENDATION to software is 1937 * that they have a (next) state of ATOMIC when they 1938 * DE-SCHEDULE. If an ORDERED tag is what was really desired, 1939 * SW can choose to immediately switch to an ORDERED tag 1940 * after the work (that has an ATOMIC tag) is re-scheduled. 1941 * Note that since there are never any tag switches pending 1942 * when the HW re-schedules, this switch can be IMMEDIATE upon 1943 * the reception of the pointer during the re-schedule. 1944 * 1945 * @tag: New tag value 1946 * @tag_type: New tag type 1947 * @group: New group value 1948 * @no_sched: Control whether this work queue entry will be rescheduled. 1949 * - 1 : don't schedule this work 1950 * - 0 : allow this work to be scheduled. 1951 */ 1952 static inline void cvmx_pow_tag_sw_desched_nocheck( 1953 uint32_t tag, 1954 enum cvmx_pow_tag_type tag_type, 1955 uint64_t group, 1956 uint64_t no_sched) 1957 { 1958 cvmx_addr_t ptr; 1959 cvmx_pow_tag_req_t tag_req; 1960 1961 if (CVMX_ENABLE_POW_CHECKS) { 1962 cvmx_pow_tag_req_t current_tag; 1963 __cvmx_pow_warn_if_pending_switch(__func__); 1964 current_tag = cvmx_pow_get_current_tag(); 1965 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL_NULL) 1966 pr_warn("%s called with NULL_NULL tag\n", __func__); 1967 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL) 1968 pr_warn("%s called with NULL tag. Deschedule not allowed from NULL state\n", 1969 __func__); 1970 if ((current_tag.s.type != CVMX_POW_TAG_TYPE_ATOMIC) 1971 && (tag_type != CVMX_POW_TAG_TYPE_ATOMIC)) 1972 pr_warn("%s called where neither the before or after tag is ATOMIC\n", 1973 __func__); 1974 } 1975 1976 tag_req.u64 = 0; 1977 tag_req.s.op = CVMX_POW_TAG_OP_SWTAG_DESCH; 1978 tag_req.s.tag = tag; 1979 tag_req.s.type = tag_type; 1980 tag_req.s.grp = group; 1981 tag_req.s.no_sched = no_sched; 1982 1983 ptr.u64 = 0; 1984 ptr.sio.mem_region = CVMX_IO_SEG; 1985 ptr.sio.is_io = 1; 1986 ptr.sio.did = CVMX_OCT_DID_TAG_TAG3; 1987 /* 1988 * since TAG3 is used, this store will clear the local pending 1989 * switch bit. 1990 */ 1991 cvmx_write_io(ptr.u64, tag_req.u64); 1992 } 1993 1994 /** 1995 * Performs a tag switch and then an immediate deschedule. This completes 1996 * immediately, so completion must not be waited for. This function does NOT 1997 * update the wqe in DRAM to match arguments. 1998 * 1999 * This function waits for any prior tag switches to complete, so the 2000 * calling code may call this function with a pending tag switch. 2001 * 2002 * Note the following CAVEAT of the Octeon HW behavior when 2003 * re-scheduling DE-SCHEDULEd items whose (next) state is 2004 * ORDERED: 2005 * - If there are no switches pending at the time that the 2006 * HW executes the de-schedule, the HW will only re-schedule 2007 * the head of the FIFO associated with the given tag. This 2008 * means that in many respects, the HW treats this ORDERED 2009 * tag as an ATOMIC tag. Note that in the SWTAG_DESCH 2010 * case (to an ORDERED tag), the HW will do the switch 2011 * before the deschedule whenever it is possible to do 2012 * the switch immediately, so it may often look like 2013 * this case. 2014 * - If there is a pending switch to ORDERED at the time 2015 * the HW executes the de-schedule, the HW will perform 2016 * the switch at the time it re-schedules, and will be 2017 * able to reschedule any/all of the entries with the 2018 * same tag. 2019 * Due to this behavior, the RECOMMENDATION to software is 2020 * that they have a (next) state of ATOMIC when they 2021 * DE-SCHEDULE. If an ORDERED tag is what was really desired, 2022 * SW can choose to immediately switch to an ORDERED tag 2023 * after the work (that has an ATOMIC tag) is re-scheduled. 2024 * Note that since there are never any tag switches pending 2025 * when the HW re-schedules, this switch can be IMMEDIATE upon 2026 * the reception of the pointer during the re-schedule. 2027 * 2028 * @tag: New tag value 2029 * @tag_type: New tag type 2030 * @group: New group value 2031 * @no_sched: Control whether this work queue entry will be rescheduled. 2032 * - 1 : don't schedule this work 2033 * - 0 : allow this work to be scheduled. 2034 */ 2035 static inline void cvmx_pow_tag_sw_desched(uint32_t tag, 2036 enum cvmx_pow_tag_type tag_type, 2037 uint64_t group, uint64_t no_sched) 2038 { 2039 if (CVMX_ENABLE_POW_CHECKS) 2040 __cvmx_pow_warn_if_pending_switch(__func__); 2041 2042 /* Need to make sure any writes to the work queue entry are complete */ 2043 CVMX_SYNCWS; 2044 /* 2045 * Ensure that there is not a pending tag switch, as a tag 2046 * switch cannot be started if a previous switch is still 2047 * pending. 2048 */ 2049 cvmx_pow_tag_sw_wait(); 2050 cvmx_pow_tag_sw_desched_nocheck(tag, tag_type, group, no_sched); 2051 } 2052 2053 /** 2054 * Deschedules the current work queue entry. 2055 * 2056 * @no_sched: no schedule flag value to be set on the work queue 2057 * entry. If this is set the entry will not be 2058 * rescheduled. 2059 */ 2060 static inline void cvmx_pow_desched(uint64_t no_sched) 2061 { 2062 cvmx_addr_t ptr; 2063 cvmx_pow_tag_req_t tag_req; 2064 2065 if (CVMX_ENABLE_POW_CHECKS) { 2066 cvmx_pow_tag_req_t current_tag; 2067 __cvmx_pow_warn_if_pending_switch(__func__); 2068 current_tag = cvmx_pow_get_current_tag(); 2069 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL_NULL) 2070 pr_warn("%s called with NULL_NULL tag\n", __func__); 2071 if (current_tag.s.type == CVMX_POW_TAG_TYPE_NULL) 2072 pr_warn("%s called with NULL tag. Deschedule not expected from NULL state\n", 2073 __func__); 2074 } 2075 2076 /* Need to make sure any writes to the work queue entry are complete */ 2077 CVMX_SYNCWS; 2078 2079 tag_req.u64 = 0; 2080 tag_req.s.op = CVMX_POW_TAG_OP_DESCH; 2081 tag_req.s.no_sched = no_sched; 2082 2083 ptr.u64 = 0; 2084 ptr.sio.mem_region = CVMX_IO_SEG; 2085 ptr.sio.is_io = 1; 2086 ptr.sio.did = CVMX_OCT_DID_TAG_TAG3; 2087 /* 2088 * since TAG3 is used, this store will clear the local pending 2089 * switch bit. 2090 */ 2091 cvmx_write_io(ptr.u64, tag_req.u64); 2092 } 2093 2094 /**************************************************** 2095 * Define usage of bits within the 32 bit tag values. 2096 *****************************************************/ 2097 2098 /* 2099 * Number of bits of the tag used by software. The SW bits are always 2100 * a contiguous block of the high starting at bit 31. The hardware 2101 * bits are always the low bits. By default, the top 8 bits of the 2102 * tag are reserved for software, and the low 24 are set by the IPD 2103 * unit. 2104 */ 2105 #define CVMX_TAG_SW_BITS (8) 2106 #define CVMX_TAG_SW_SHIFT (32 - CVMX_TAG_SW_BITS) 2107 2108 /* Below is the list of values for the top 8 bits of the tag. */ 2109 /* 2110 * Tag values with top byte of this value are reserved for internal 2111 * executive uses. 2112 */ 2113 #define CVMX_TAG_SW_BITS_INTERNAL 0x1 2114 /* The executive divides the remaining 24 bits as follows: 2115 * - the upper 8 bits (bits 23 - 16 of the tag) define a subgroup 2116 * 2117 * - the lower 16 bits (bits 15 - 0 of the tag) define are the value 2118 * with the subgroup 2119 * 2120 * Note that this section describes the format of tags generated by 2121 * software - refer to the hardware documentation for a description of 2122 * the tags values generated by the packet input hardware. Subgroups 2123 * are defined here. 2124 */ 2125 /* Mask for the value portion of the tag */ 2126 #define CVMX_TAG_SUBGROUP_MASK 0xFFFF 2127 #define CVMX_TAG_SUBGROUP_SHIFT 16 2128 #define CVMX_TAG_SUBGROUP_PKO 0x1 2129 2130 /* End of executive tag subgroup definitions */ 2131 2132 /* 2133 * The remaining values software bit values 0x2 - 0xff are available 2134 * for application use. 2135 */ 2136 2137 /** 2138 * This function creates a 32 bit tag value from the two values provided. 2139 * 2140 * @sw_bits: The upper bits (number depends on configuration) are set 2141 * to this value. The remainder of bits are set by the 2142 * hw_bits parameter. 2143 * 2144 * @hw_bits: The lower bits (number depends on configuration) are set 2145 * to this value. The remainder of bits are set by the 2146 * sw_bits parameter. 2147 * 2148 * Returns 32 bit value of the combined hw and sw bits. 2149 */ 2150 static inline uint32_t cvmx_pow_tag_compose(uint64_t sw_bits, uint64_t hw_bits) 2151 { 2152 return ((sw_bits & cvmx_build_mask(CVMX_TAG_SW_BITS)) << 2153 CVMX_TAG_SW_SHIFT) | 2154 (hw_bits & cvmx_build_mask(32 - CVMX_TAG_SW_BITS)); 2155 } 2156 2157 /** 2158 * Extracts the bits allocated for software use from the tag 2159 * 2160 * @tag: 32 bit tag value 2161 * 2162 * Returns N bit software tag value, where N is configurable with the 2163 * CVMX_TAG_SW_BITS define 2164 */ 2165 static inline uint32_t cvmx_pow_tag_get_sw_bits(uint64_t tag) 2166 { 2167 return (tag >> (32 - CVMX_TAG_SW_BITS)) & 2168 cvmx_build_mask(CVMX_TAG_SW_BITS); 2169 } 2170 2171 /** 2172 * 2173 * Extracts the bits allocated for hardware use from the tag 2174 * 2175 * @tag: 32 bit tag value 2176 * 2177 * Returns (32 - N) bit software tag value, where N is configurable 2178 * with the CVMX_TAG_SW_BITS define 2179 */ 2180 static inline uint32_t cvmx_pow_tag_get_hw_bits(uint64_t tag) 2181 { 2182 return tag & cvmx_build_mask(32 - CVMX_TAG_SW_BITS); 2183 } 2184 2185 /** 2186 * Store the current POW internal state into the supplied 2187 * buffer. It is recommended that you pass a buffer of at least 2188 * 128KB. The format of the capture may change based on SDK 2189 * version and Octeon chip. 2190 * 2191 * @buffer: Buffer to store capture into 2192 * @buffer_size: 2193 * The size of the supplied buffer 2194 * 2195 * Returns Zero on success, negative on failure 2196 */ 2197 extern int cvmx_pow_capture(void *buffer, int buffer_size); 2198 2199 /** 2200 * Dump a POW capture to the console in a human readable format. 2201 * 2202 * @buffer: POW capture from cvmx_pow_capture() 2203 * @buffer_size: 2204 * Size of the buffer 2205 */ 2206 extern void cvmx_pow_display(void *buffer, int buffer_size); 2207 2208 /** 2209 * Return the number of POW entries supported by this chip 2210 * 2211 * Returns Number of POW entries 2212 */ 2213 extern int cvmx_pow_get_num_entries(void); 2214 2215 #endif /* __CVMX_POW_H__ */ 2216