1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* 3 * Copyright(c) 2015 - 2020 Intel Corporation. 4 * Copyright(c) 2021 Cornelis Networks. 5 */ 6 7 /* 8 * This file contains all of the code that is specific to the HFI chip 9 */ 10 11 #include <linux/pci.h> 12 #include <linux/delay.h> 13 #include <linux/interrupt.h> 14 #include <linux/module.h> 15 16 #include "hfi.h" 17 #include "trace.h" 18 #include "mad.h" 19 #include "pio.h" 20 #include "sdma.h" 21 #include "eprom.h" 22 #include "efivar.h" 23 #include "platform.h" 24 #include "aspm.h" 25 #include "affinity.h" 26 #include "debugfs.h" 27 #include "fault.h" 28 #include "netdev.h" 29 30 uint num_vls = HFI1_MAX_VLS_SUPPORTED; 31 module_param(num_vls, uint, S_IRUGO); 32 MODULE_PARM_DESC(num_vls, "Set number of Virtual Lanes to use (1-8)"); 33 34 /* 35 * Default time to aggregate two 10K packets from the idle state 36 * (timer not running). The timer starts at the end of the first packet, 37 * so only the time for one 10K packet and header plus a bit extra is needed. 38 * 10 * 1024 + 64 header byte = 10304 byte 39 * 10304 byte / 12.5 GB/s = 824.32ns 40 */ 41 uint rcv_intr_timeout = (824 + 16); /* 16 is for coalescing interrupt */ 42 module_param(rcv_intr_timeout, uint, S_IRUGO); 43 MODULE_PARM_DESC(rcv_intr_timeout, "Receive interrupt mitigation timeout in ns"); 44 45 uint rcv_intr_count = 16; /* same as qib */ 46 module_param(rcv_intr_count, uint, S_IRUGO); 47 MODULE_PARM_DESC(rcv_intr_count, "Receive interrupt mitigation count"); 48 49 ushort link_crc_mask = SUPPORTED_CRCS; 50 module_param(link_crc_mask, ushort, S_IRUGO); 51 MODULE_PARM_DESC(link_crc_mask, "CRCs to use on the link"); 52 53 uint loopback; 54 module_param_named(loopback, loopback, uint, S_IRUGO); 55 MODULE_PARM_DESC(loopback, "Put into loopback mode (1 = serdes, 3 = external cable"); 56 57 /* Other driver tunables */ 58 uint rcv_intr_dynamic = 1; /* enable dynamic mode for rcv int mitigation*/ 59 static ushort crc_14b_sideband = 1; 60 static uint use_flr = 1; 61 uint quick_linkup; /* skip LNI */ 62 63 struct flag_table { 64 u64 flag; /* the flag */ 65 char *str; /* description string */ 66 u16 extra; /* extra information */ 67 u16 unused0; 68 u32 unused1; 69 }; 70 71 /* str must be a string constant */ 72 #define FLAG_ENTRY(str, extra, flag) {flag, str, extra} 73 #define FLAG_ENTRY0(str, flag) {flag, str, 0} 74 75 /* Send Error Consequences */ 76 #define SEC_WRITE_DROPPED 0x1 77 #define SEC_PACKET_DROPPED 0x2 78 #define SEC_SC_HALTED 0x4 /* per-context only */ 79 #define SEC_SPC_FREEZE 0x8 /* per-HFI only */ 80 81 #define DEFAULT_KRCVQS 2 82 #define MIN_KERNEL_KCTXTS 2 83 #define FIRST_KERNEL_KCTXT 1 84 85 /* 86 * RSM instance allocation 87 * 0 - User Fecn Handling 88 * 1 - Vnic 89 * 2 - AIP 90 * 3 - Verbs 91 */ 92 #define RSM_INS_FECN 0 93 #define RSM_INS_VNIC 1 94 #define RSM_INS_AIP 2 95 #define RSM_INS_VERBS 3 96 97 /* Bit offset into the GUID which carries HFI id information */ 98 #define GUID_HFI_INDEX_SHIFT 39 99 100 /* extract the emulation revision */ 101 #define emulator_rev(dd) ((dd)->irev >> 8) 102 /* parallel and serial emulation versions are 3 and 4 respectively */ 103 #define is_emulator_p(dd) ((((dd)->irev) & 0xf) == 3) 104 #define is_emulator_s(dd) ((((dd)->irev) & 0xf) == 4) 105 106 /* RSM fields for Verbs */ 107 /* packet type */ 108 #define IB_PACKET_TYPE 2ull 109 #define QW_SHIFT 6ull 110 /* QPN[7..1] */ 111 #define QPN_WIDTH 7ull 112 113 /* LRH.BTH: QW 0, OFFSET 48 - for match */ 114 #define LRH_BTH_QW 0ull 115 #define LRH_BTH_BIT_OFFSET 48ull 116 #define LRH_BTH_OFFSET(off) ((LRH_BTH_QW << QW_SHIFT) | (off)) 117 #define LRH_BTH_MATCH_OFFSET LRH_BTH_OFFSET(LRH_BTH_BIT_OFFSET) 118 #define LRH_BTH_SELECT 119 #define LRH_BTH_MASK 3ull 120 #define LRH_BTH_VALUE 2ull 121 122 /* LRH.SC[3..0] QW 0, OFFSET 56 - for match */ 123 #define LRH_SC_QW 0ull 124 #define LRH_SC_BIT_OFFSET 56ull 125 #define LRH_SC_OFFSET(off) ((LRH_SC_QW << QW_SHIFT) | (off)) 126 #define LRH_SC_MATCH_OFFSET LRH_SC_OFFSET(LRH_SC_BIT_OFFSET) 127 #define LRH_SC_MASK 128ull 128 #define LRH_SC_VALUE 0ull 129 130 /* SC[n..0] QW 0, OFFSET 60 - for select */ 131 #define LRH_SC_SELECT_OFFSET ((LRH_SC_QW << QW_SHIFT) | (60ull)) 132 133 /* QPN[m+n:1] QW 1, OFFSET 1 */ 134 #define QPN_SELECT_OFFSET ((1ull << QW_SHIFT) | (1ull)) 135 136 /* RSM fields for AIP */ 137 /* LRH.BTH above is reused for this rule */ 138 139 /* BTH.DESTQP: QW 1, OFFSET 16 for match */ 140 #define BTH_DESTQP_QW 1ull 141 #define BTH_DESTQP_BIT_OFFSET 16ull 142 #define BTH_DESTQP_OFFSET(off) ((BTH_DESTQP_QW << QW_SHIFT) | (off)) 143 #define BTH_DESTQP_MATCH_OFFSET BTH_DESTQP_OFFSET(BTH_DESTQP_BIT_OFFSET) 144 #define BTH_DESTQP_MASK 0xFFull 145 #define BTH_DESTQP_VALUE 0x81ull 146 147 /* DETH.SQPN: QW 1 Offset 56 for select */ 148 /* We use 8 most significant Soure QPN bits as entropy fpr AIP */ 149 #define DETH_AIP_SQPN_QW 3ull 150 #define DETH_AIP_SQPN_BIT_OFFSET 56ull 151 #define DETH_AIP_SQPN_OFFSET(off) ((DETH_AIP_SQPN_QW << QW_SHIFT) | (off)) 152 #define DETH_AIP_SQPN_SELECT_OFFSET \ 153 DETH_AIP_SQPN_OFFSET(DETH_AIP_SQPN_BIT_OFFSET) 154 155 /* RSM fields for Vnic */ 156 /* L2_TYPE: QW 0, OFFSET 61 - for match */ 157 #define L2_TYPE_QW 0ull 158 #define L2_TYPE_BIT_OFFSET 61ull 159 #define L2_TYPE_OFFSET(off) ((L2_TYPE_QW << QW_SHIFT) | (off)) 160 #define L2_TYPE_MATCH_OFFSET L2_TYPE_OFFSET(L2_TYPE_BIT_OFFSET) 161 #define L2_TYPE_MASK 3ull 162 #define L2_16B_VALUE 2ull 163 164 /* L4_TYPE QW 1, OFFSET 0 - for match */ 165 #define L4_TYPE_QW 1ull 166 #define L4_TYPE_BIT_OFFSET 0ull 167 #define L4_TYPE_OFFSET(off) ((L4_TYPE_QW << QW_SHIFT) | (off)) 168 #define L4_TYPE_MATCH_OFFSET L4_TYPE_OFFSET(L4_TYPE_BIT_OFFSET) 169 #define L4_16B_TYPE_MASK 0xFFull 170 #define L4_16B_ETH_VALUE 0x78ull 171 172 /* 16B VESWID - for select */ 173 #define L4_16B_HDR_VESWID_OFFSET ((2 << QW_SHIFT) | (16ull)) 174 /* 16B ENTROPY - for select */ 175 #define L2_16B_ENTROPY_OFFSET ((1 << QW_SHIFT) | (32ull)) 176 177 /* defines to build power on SC2VL table */ 178 #define SC2VL_VAL( \ 179 num, \ 180 sc0, sc0val, \ 181 sc1, sc1val, \ 182 sc2, sc2val, \ 183 sc3, sc3val, \ 184 sc4, sc4val, \ 185 sc5, sc5val, \ 186 sc6, sc6val, \ 187 sc7, sc7val) \ 188 ( \ 189 ((u64)(sc0val) << SEND_SC2VLT##num##_SC##sc0##_SHIFT) | \ 190 ((u64)(sc1val) << SEND_SC2VLT##num##_SC##sc1##_SHIFT) | \ 191 ((u64)(sc2val) << SEND_SC2VLT##num##_SC##sc2##_SHIFT) | \ 192 ((u64)(sc3val) << SEND_SC2VLT##num##_SC##sc3##_SHIFT) | \ 193 ((u64)(sc4val) << SEND_SC2VLT##num##_SC##sc4##_SHIFT) | \ 194 ((u64)(sc5val) << SEND_SC2VLT##num##_SC##sc5##_SHIFT) | \ 195 ((u64)(sc6val) << SEND_SC2VLT##num##_SC##sc6##_SHIFT) | \ 196 ((u64)(sc7val) << SEND_SC2VLT##num##_SC##sc7##_SHIFT) \ 197 ) 198 199 #define DC_SC_VL_VAL( \ 200 range, \ 201 e0, e0val, \ 202 e1, e1val, \ 203 e2, e2val, \ 204 e3, e3val, \ 205 e4, e4val, \ 206 e5, e5val, \ 207 e6, e6val, \ 208 e7, e7val, \ 209 e8, e8val, \ 210 e9, e9val, \ 211 e10, e10val, \ 212 e11, e11val, \ 213 e12, e12val, \ 214 e13, e13val, \ 215 e14, e14val, \ 216 e15, e15val) \ 217 ( \ 218 ((u64)(e0val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e0##_SHIFT) | \ 219 ((u64)(e1val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e1##_SHIFT) | \ 220 ((u64)(e2val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e2##_SHIFT) | \ 221 ((u64)(e3val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e3##_SHIFT) | \ 222 ((u64)(e4val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e4##_SHIFT) | \ 223 ((u64)(e5val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e5##_SHIFT) | \ 224 ((u64)(e6val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e6##_SHIFT) | \ 225 ((u64)(e7val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e7##_SHIFT) | \ 226 ((u64)(e8val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e8##_SHIFT) | \ 227 ((u64)(e9val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e9##_SHIFT) | \ 228 ((u64)(e10val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e10##_SHIFT) | \ 229 ((u64)(e11val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e11##_SHIFT) | \ 230 ((u64)(e12val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e12##_SHIFT) | \ 231 ((u64)(e13val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e13##_SHIFT) | \ 232 ((u64)(e14val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e14##_SHIFT) | \ 233 ((u64)(e15val) << DCC_CFG_SC_VL_TABLE_##range##_ENTRY##e15##_SHIFT) \ 234 ) 235 236 /* all CceStatus sub-block freeze bits */ 237 #define ALL_FROZE (CCE_STATUS_SDMA_FROZE_SMASK \ 238 | CCE_STATUS_RXE_FROZE_SMASK \ 239 | CCE_STATUS_TXE_FROZE_SMASK \ 240 | CCE_STATUS_TXE_PIO_FROZE_SMASK) 241 /* all CceStatus sub-block TXE pause bits */ 242 #define ALL_TXE_PAUSE (CCE_STATUS_TXE_PIO_PAUSED_SMASK \ 243 | CCE_STATUS_TXE_PAUSED_SMASK \ 244 | CCE_STATUS_SDMA_PAUSED_SMASK) 245 /* all CceStatus sub-block RXE pause bits */ 246 #define ALL_RXE_PAUSE CCE_STATUS_RXE_PAUSED_SMASK 247 248 #define CNTR_MAX 0xFFFFFFFFFFFFFFFFULL 249 #define CNTR_32BIT_MAX 0x00000000FFFFFFFF 250 251 /* 252 * CCE Error flags. 253 */ 254 static const struct flag_table cce_err_status_flags[] = { 255 /* 0*/ FLAG_ENTRY0("CceCsrParityErr", 256 CCE_ERR_STATUS_CCE_CSR_PARITY_ERR_SMASK), 257 /* 1*/ FLAG_ENTRY0("CceCsrReadBadAddrErr", 258 CCE_ERR_STATUS_CCE_CSR_READ_BAD_ADDR_ERR_SMASK), 259 /* 2*/ FLAG_ENTRY0("CceCsrWriteBadAddrErr", 260 CCE_ERR_STATUS_CCE_CSR_WRITE_BAD_ADDR_ERR_SMASK), 261 /* 3*/ FLAG_ENTRY0("CceTrgtAsyncFifoParityErr", 262 CCE_ERR_STATUS_CCE_TRGT_ASYNC_FIFO_PARITY_ERR_SMASK), 263 /* 4*/ FLAG_ENTRY0("CceTrgtAccessErr", 264 CCE_ERR_STATUS_CCE_TRGT_ACCESS_ERR_SMASK), 265 /* 5*/ FLAG_ENTRY0("CceRspdDataParityErr", 266 CCE_ERR_STATUS_CCE_RSPD_DATA_PARITY_ERR_SMASK), 267 /* 6*/ FLAG_ENTRY0("CceCli0AsyncFifoParityErr", 268 CCE_ERR_STATUS_CCE_CLI0_ASYNC_FIFO_PARITY_ERR_SMASK), 269 /* 7*/ FLAG_ENTRY0("CceCsrCfgBusParityErr", 270 CCE_ERR_STATUS_CCE_CSR_CFG_BUS_PARITY_ERR_SMASK), 271 /* 8*/ FLAG_ENTRY0("CceCli2AsyncFifoParityErr", 272 CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK), 273 /* 9*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr", 274 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR_SMASK), 275 /*10*/ FLAG_ENTRY0("CceCli1AsyncFifoPioCrdtParityErr", 276 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR_SMASK), 277 /*11*/ FLAG_ENTRY0("CceCli1AsyncFifoRxdmaParityError", 278 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERROR_SMASK), 279 /*12*/ FLAG_ENTRY0("CceCli1AsyncFifoDbgParityError", 280 CCE_ERR_STATUS_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERROR_SMASK), 281 /*13*/ FLAG_ENTRY0("PcicRetryMemCorErr", 282 CCE_ERR_STATUS_PCIC_RETRY_MEM_COR_ERR_SMASK), 283 /*14*/ FLAG_ENTRY0("PcicRetryMemCorErr", 284 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_COR_ERR_SMASK), 285 /*15*/ FLAG_ENTRY0("PcicPostHdQCorErr", 286 CCE_ERR_STATUS_PCIC_POST_HD_QCOR_ERR_SMASK), 287 /*16*/ FLAG_ENTRY0("PcicPostHdQCorErr", 288 CCE_ERR_STATUS_PCIC_POST_DAT_QCOR_ERR_SMASK), 289 /*17*/ FLAG_ENTRY0("PcicPostHdQCorErr", 290 CCE_ERR_STATUS_PCIC_CPL_HD_QCOR_ERR_SMASK), 291 /*18*/ FLAG_ENTRY0("PcicCplDatQCorErr", 292 CCE_ERR_STATUS_PCIC_CPL_DAT_QCOR_ERR_SMASK), 293 /*19*/ FLAG_ENTRY0("PcicNPostHQParityErr", 294 CCE_ERR_STATUS_PCIC_NPOST_HQ_PARITY_ERR_SMASK), 295 /*20*/ FLAG_ENTRY0("PcicNPostDatQParityErr", 296 CCE_ERR_STATUS_PCIC_NPOST_DAT_QPARITY_ERR_SMASK), 297 /*21*/ FLAG_ENTRY0("PcicRetryMemUncErr", 298 CCE_ERR_STATUS_PCIC_RETRY_MEM_UNC_ERR_SMASK), 299 /*22*/ FLAG_ENTRY0("PcicRetrySotMemUncErr", 300 CCE_ERR_STATUS_PCIC_RETRY_SOT_MEM_UNC_ERR_SMASK), 301 /*23*/ FLAG_ENTRY0("PcicPostHdQUncErr", 302 CCE_ERR_STATUS_PCIC_POST_HD_QUNC_ERR_SMASK), 303 /*24*/ FLAG_ENTRY0("PcicPostDatQUncErr", 304 CCE_ERR_STATUS_PCIC_POST_DAT_QUNC_ERR_SMASK), 305 /*25*/ FLAG_ENTRY0("PcicCplHdQUncErr", 306 CCE_ERR_STATUS_PCIC_CPL_HD_QUNC_ERR_SMASK), 307 /*26*/ FLAG_ENTRY0("PcicCplDatQUncErr", 308 CCE_ERR_STATUS_PCIC_CPL_DAT_QUNC_ERR_SMASK), 309 /*27*/ FLAG_ENTRY0("PcicTransmitFrontParityErr", 310 CCE_ERR_STATUS_PCIC_TRANSMIT_FRONT_PARITY_ERR_SMASK), 311 /*28*/ FLAG_ENTRY0("PcicTransmitBackParityErr", 312 CCE_ERR_STATUS_PCIC_TRANSMIT_BACK_PARITY_ERR_SMASK), 313 /*29*/ FLAG_ENTRY0("PcicReceiveParityErr", 314 CCE_ERR_STATUS_PCIC_RECEIVE_PARITY_ERR_SMASK), 315 /*30*/ FLAG_ENTRY0("CceTrgtCplTimeoutErr", 316 CCE_ERR_STATUS_CCE_TRGT_CPL_TIMEOUT_ERR_SMASK), 317 /*31*/ FLAG_ENTRY0("LATriggered", 318 CCE_ERR_STATUS_LA_TRIGGERED_SMASK), 319 /*32*/ FLAG_ENTRY0("CceSegReadBadAddrErr", 320 CCE_ERR_STATUS_CCE_SEG_READ_BAD_ADDR_ERR_SMASK), 321 /*33*/ FLAG_ENTRY0("CceSegWriteBadAddrErr", 322 CCE_ERR_STATUS_CCE_SEG_WRITE_BAD_ADDR_ERR_SMASK), 323 /*34*/ FLAG_ENTRY0("CceRcplAsyncFifoParityErr", 324 CCE_ERR_STATUS_CCE_RCPL_ASYNC_FIFO_PARITY_ERR_SMASK), 325 /*35*/ FLAG_ENTRY0("CceRxdmaConvFifoParityErr", 326 CCE_ERR_STATUS_CCE_RXDMA_CONV_FIFO_PARITY_ERR_SMASK), 327 /*36*/ FLAG_ENTRY0("CceMsixTableCorErr", 328 CCE_ERR_STATUS_CCE_MSIX_TABLE_COR_ERR_SMASK), 329 /*37*/ FLAG_ENTRY0("CceMsixTableUncErr", 330 CCE_ERR_STATUS_CCE_MSIX_TABLE_UNC_ERR_SMASK), 331 /*38*/ FLAG_ENTRY0("CceIntMapCorErr", 332 CCE_ERR_STATUS_CCE_INT_MAP_COR_ERR_SMASK), 333 /*39*/ FLAG_ENTRY0("CceIntMapUncErr", 334 CCE_ERR_STATUS_CCE_INT_MAP_UNC_ERR_SMASK), 335 /*40*/ FLAG_ENTRY0("CceMsixCsrParityErr", 336 CCE_ERR_STATUS_CCE_MSIX_CSR_PARITY_ERR_SMASK), 337 /*41-63 reserved*/ 338 }; 339 340 /* 341 * Misc Error flags 342 */ 343 #define MES(text) MISC_ERR_STATUS_MISC_##text##_ERR_SMASK 344 static const struct flag_table misc_err_status_flags[] = { 345 /* 0*/ FLAG_ENTRY0("CSR_PARITY", MES(CSR_PARITY)), 346 /* 1*/ FLAG_ENTRY0("CSR_READ_BAD_ADDR", MES(CSR_READ_BAD_ADDR)), 347 /* 2*/ FLAG_ENTRY0("CSR_WRITE_BAD_ADDR", MES(CSR_WRITE_BAD_ADDR)), 348 /* 3*/ FLAG_ENTRY0("SBUS_WRITE_FAILED", MES(SBUS_WRITE_FAILED)), 349 /* 4*/ FLAG_ENTRY0("KEY_MISMATCH", MES(KEY_MISMATCH)), 350 /* 5*/ FLAG_ENTRY0("FW_AUTH_FAILED", MES(FW_AUTH_FAILED)), 351 /* 6*/ FLAG_ENTRY0("EFUSE_CSR_PARITY", MES(EFUSE_CSR_PARITY)), 352 /* 7*/ FLAG_ENTRY0("EFUSE_READ_BAD_ADDR", MES(EFUSE_READ_BAD_ADDR)), 353 /* 8*/ FLAG_ENTRY0("EFUSE_WRITE", MES(EFUSE_WRITE)), 354 /* 9*/ FLAG_ENTRY0("EFUSE_DONE_PARITY", MES(EFUSE_DONE_PARITY)), 355 /*10*/ FLAG_ENTRY0("INVALID_EEP_CMD", MES(INVALID_EEP_CMD)), 356 /*11*/ FLAG_ENTRY0("MBIST_FAIL", MES(MBIST_FAIL)), 357 /*12*/ FLAG_ENTRY0("PLL_LOCK_FAIL", MES(PLL_LOCK_FAIL)) 358 }; 359 360 /* 361 * TXE PIO Error flags and consequences 362 */ 363 static const struct flag_table pio_err_status_flags[] = { 364 /* 0*/ FLAG_ENTRY("PioWriteBadCtxt", 365 SEC_WRITE_DROPPED, 366 SEND_PIO_ERR_STATUS_PIO_WRITE_BAD_CTXT_ERR_SMASK), 367 /* 1*/ FLAG_ENTRY("PioWriteAddrParity", 368 SEC_SPC_FREEZE, 369 SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK), 370 /* 2*/ FLAG_ENTRY("PioCsrParity", 371 SEC_SPC_FREEZE, 372 SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK), 373 /* 3*/ FLAG_ENTRY("PioSbMemFifo0", 374 SEC_SPC_FREEZE, 375 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK), 376 /* 4*/ FLAG_ENTRY("PioSbMemFifo1", 377 SEC_SPC_FREEZE, 378 SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK), 379 /* 5*/ FLAG_ENTRY("PioPccFifoParity", 380 SEC_SPC_FREEZE, 381 SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK), 382 /* 6*/ FLAG_ENTRY("PioPecFifoParity", 383 SEC_SPC_FREEZE, 384 SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK), 385 /* 7*/ FLAG_ENTRY("PioSbrdctlCrrelParity", 386 SEC_SPC_FREEZE, 387 SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK), 388 /* 8*/ FLAG_ENTRY("PioSbrdctrlCrrelFifoParity", 389 SEC_SPC_FREEZE, 390 SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK), 391 /* 9*/ FLAG_ENTRY("PioPktEvictFifoParityErr", 392 SEC_SPC_FREEZE, 393 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK), 394 /*10*/ FLAG_ENTRY("PioSmPktResetParity", 395 SEC_SPC_FREEZE, 396 SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK), 397 /*11*/ FLAG_ENTRY("PioVlLenMemBank0Unc", 398 SEC_SPC_FREEZE, 399 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK), 400 /*12*/ FLAG_ENTRY("PioVlLenMemBank1Unc", 401 SEC_SPC_FREEZE, 402 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK), 403 /*13*/ FLAG_ENTRY("PioVlLenMemBank0Cor", 404 0, 405 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_COR_ERR_SMASK), 406 /*14*/ FLAG_ENTRY("PioVlLenMemBank1Cor", 407 0, 408 SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_COR_ERR_SMASK), 409 /*15*/ FLAG_ENTRY("PioCreditRetFifoParity", 410 SEC_SPC_FREEZE, 411 SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK), 412 /*16*/ FLAG_ENTRY("PioPpmcPblFifo", 413 SEC_SPC_FREEZE, 414 SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK), 415 /*17*/ FLAG_ENTRY("PioInitSmIn", 416 0, 417 SEND_PIO_ERR_STATUS_PIO_INIT_SM_IN_ERR_SMASK), 418 /*18*/ FLAG_ENTRY("PioPktEvictSmOrArbSm", 419 SEC_SPC_FREEZE, 420 SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK), 421 /*19*/ FLAG_ENTRY("PioHostAddrMemUnc", 422 SEC_SPC_FREEZE, 423 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK), 424 /*20*/ FLAG_ENTRY("PioHostAddrMemCor", 425 0, 426 SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_COR_ERR_SMASK), 427 /*21*/ FLAG_ENTRY("PioWriteDataParity", 428 SEC_SPC_FREEZE, 429 SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK), 430 /*22*/ FLAG_ENTRY("PioStateMachine", 431 SEC_SPC_FREEZE, 432 SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK), 433 /*23*/ FLAG_ENTRY("PioWriteQwValidParity", 434 SEC_WRITE_DROPPED | SEC_SPC_FREEZE, 435 SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK), 436 /*24*/ FLAG_ENTRY("PioBlockQwCountParity", 437 SEC_WRITE_DROPPED | SEC_SPC_FREEZE, 438 SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK), 439 /*25*/ FLAG_ENTRY("PioVlfVlLenParity", 440 SEC_SPC_FREEZE, 441 SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK), 442 /*26*/ FLAG_ENTRY("PioVlfSopParity", 443 SEC_SPC_FREEZE, 444 SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK), 445 /*27*/ FLAG_ENTRY("PioVlFifoParity", 446 SEC_SPC_FREEZE, 447 SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK), 448 /*28*/ FLAG_ENTRY("PioPpmcBqcMemParity", 449 SEC_SPC_FREEZE, 450 SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK), 451 /*29*/ FLAG_ENTRY("PioPpmcSopLen", 452 SEC_SPC_FREEZE, 453 SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK), 454 /*30-31 reserved*/ 455 /*32*/ FLAG_ENTRY("PioCurrentFreeCntParity", 456 SEC_SPC_FREEZE, 457 SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK), 458 /*33*/ FLAG_ENTRY("PioLastReturnedCntParity", 459 SEC_SPC_FREEZE, 460 SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK), 461 /*34*/ FLAG_ENTRY("PioPccSopHeadParity", 462 SEC_SPC_FREEZE, 463 SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK), 464 /*35*/ FLAG_ENTRY("PioPecSopHeadParityErr", 465 SEC_SPC_FREEZE, 466 SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK), 467 /*36-63 reserved*/ 468 }; 469 470 /* TXE PIO errors that cause an SPC freeze */ 471 #define ALL_PIO_FREEZE_ERR \ 472 (SEND_PIO_ERR_STATUS_PIO_WRITE_ADDR_PARITY_ERR_SMASK \ 473 | SEND_PIO_ERR_STATUS_PIO_CSR_PARITY_ERR_SMASK \ 474 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO0_ERR_SMASK \ 475 | SEND_PIO_ERR_STATUS_PIO_SB_MEM_FIFO1_ERR_SMASK \ 476 | SEND_PIO_ERR_STATUS_PIO_PCC_FIFO_PARITY_ERR_SMASK \ 477 | SEND_PIO_ERR_STATUS_PIO_PEC_FIFO_PARITY_ERR_SMASK \ 478 | SEND_PIO_ERR_STATUS_PIO_SBRDCTL_CRREL_PARITY_ERR_SMASK \ 479 | SEND_PIO_ERR_STATUS_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR_SMASK \ 480 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_FIFO_PARITY_ERR_SMASK \ 481 | SEND_PIO_ERR_STATUS_PIO_SM_PKT_RESET_PARITY_ERR_SMASK \ 482 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK0_UNC_ERR_SMASK \ 483 | SEND_PIO_ERR_STATUS_PIO_VL_LEN_MEM_BANK1_UNC_ERR_SMASK \ 484 | SEND_PIO_ERR_STATUS_PIO_CREDIT_RET_FIFO_PARITY_ERR_SMASK \ 485 | SEND_PIO_ERR_STATUS_PIO_PPMC_PBL_FIFO_ERR_SMASK \ 486 | SEND_PIO_ERR_STATUS_PIO_PKT_EVICT_SM_OR_ARB_SM_ERR_SMASK \ 487 | SEND_PIO_ERR_STATUS_PIO_HOST_ADDR_MEM_UNC_ERR_SMASK \ 488 | SEND_PIO_ERR_STATUS_PIO_WRITE_DATA_PARITY_ERR_SMASK \ 489 | SEND_PIO_ERR_STATUS_PIO_STATE_MACHINE_ERR_SMASK \ 490 | SEND_PIO_ERR_STATUS_PIO_WRITE_QW_VALID_PARITY_ERR_SMASK \ 491 | SEND_PIO_ERR_STATUS_PIO_BLOCK_QW_COUNT_PARITY_ERR_SMASK \ 492 | SEND_PIO_ERR_STATUS_PIO_VLF_VL_LEN_PARITY_ERR_SMASK \ 493 | SEND_PIO_ERR_STATUS_PIO_VLF_SOP_PARITY_ERR_SMASK \ 494 | SEND_PIO_ERR_STATUS_PIO_VL_FIFO_PARITY_ERR_SMASK \ 495 | SEND_PIO_ERR_STATUS_PIO_PPMC_BQC_MEM_PARITY_ERR_SMASK \ 496 | SEND_PIO_ERR_STATUS_PIO_PPMC_SOP_LEN_ERR_SMASK \ 497 | SEND_PIO_ERR_STATUS_PIO_CURRENT_FREE_CNT_PARITY_ERR_SMASK \ 498 | SEND_PIO_ERR_STATUS_PIO_LAST_RETURNED_CNT_PARITY_ERR_SMASK \ 499 | SEND_PIO_ERR_STATUS_PIO_PCC_SOP_HEAD_PARITY_ERR_SMASK \ 500 | SEND_PIO_ERR_STATUS_PIO_PEC_SOP_HEAD_PARITY_ERR_SMASK) 501 502 /* 503 * TXE SDMA Error flags 504 */ 505 static const struct flag_table sdma_err_status_flags[] = { 506 /* 0*/ FLAG_ENTRY0("SDmaRpyTagErr", 507 SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK), 508 /* 1*/ FLAG_ENTRY0("SDmaCsrParityErr", 509 SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK), 510 /* 2*/ FLAG_ENTRY0("SDmaPcieReqTrackingUncErr", 511 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK), 512 /* 3*/ FLAG_ENTRY0("SDmaPcieReqTrackingCorErr", 513 SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_COR_ERR_SMASK), 514 /*04-63 reserved*/ 515 }; 516 517 /* TXE SDMA errors that cause an SPC freeze */ 518 #define ALL_SDMA_FREEZE_ERR \ 519 (SEND_DMA_ERR_STATUS_SDMA_RPY_TAG_ERR_SMASK \ 520 | SEND_DMA_ERR_STATUS_SDMA_CSR_PARITY_ERR_SMASK \ 521 | SEND_DMA_ERR_STATUS_SDMA_PCIE_REQ_TRACKING_UNC_ERR_SMASK) 522 523 /* SendEgressErrInfo bits that correspond to a PortXmitDiscard counter */ 524 #define PORT_DISCARD_EGRESS_ERRS \ 525 (SEND_EGRESS_ERR_INFO_TOO_LONG_IB_PACKET_ERR_SMASK \ 526 | SEND_EGRESS_ERR_INFO_VL_MAPPING_ERR_SMASK \ 527 | SEND_EGRESS_ERR_INFO_VL_ERR_SMASK) 528 529 /* 530 * TXE Egress Error flags 531 */ 532 #define SEES(text) SEND_EGRESS_ERR_STATUS_##text##_ERR_SMASK 533 static const struct flag_table egress_err_status_flags[] = { 534 /* 0*/ FLAG_ENTRY0("TxPktIntegrityMemCorErr", SEES(TX_PKT_INTEGRITY_MEM_COR)), 535 /* 1*/ FLAG_ENTRY0("TxPktIntegrityMemUncErr", SEES(TX_PKT_INTEGRITY_MEM_UNC)), 536 /* 2 reserved */ 537 /* 3*/ FLAG_ENTRY0("TxEgressFifoUnderrunOrParityErr", 538 SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY)), 539 /* 4*/ FLAG_ENTRY0("TxLinkdownErr", SEES(TX_LINKDOWN)), 540 /* 5*/ FLAG_ENTRY0("TxIncorrectLinkStateErr", SEES(TX_INCORRECT_LINK_STATE)), 541 /* 6 reserved */ 542 /* 7*/ FLAG_ENTRY0("TxPioLaunchIntfParityErr", 543 SEES(TX_PIO_LAUNCH_INTF_PARITY)), 544 /* 8*/ FLAG_ENTRY0("TxSdmaLaunchIntfParityErr", 545 SEES(TX_SDMA_LAUNCH_INTF_PARITY)), 546 /* 9-10 reserved */ 547 /*11*/ FLAG_ENTRY0("TxSbrdCtlStateMachineParityErr", 548 SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY)), 549 /*12*/ FLAG_ENTRY0("TxIllegalVLErr", SEES(TX_ILLEGAL_VL)), 550 /*13*/ FLAG_ENTRY0("TxLaunchCsrParityErr", SEES(TX_LAUNCH_CSR_PARITY)), 551 /*14*/ FLAG_ENTRY0("TxSbrdCtlCsrParityErr", SEES(TX_SBRD_CTL_CSR_PARITY)), 552 /*15*/ FLAG_ENTRY0("TxConfigParityErr", SEES(TX_CONFIG_PARITY)), 553 /*16*/ FLAG_ENTRY0("TxSdma0DisallowedPacketErr", 554 SEES(TX_SDMA0_DISALLOWED_PACKET)), 555 /*17*/ FLAG_ENTRY0("TxSdma1DisallowedPacketErr", 556 SEES(TX_SDMA1_DISALLOWED_PACKET)), 557 /*18*/ FLAG_ENTRY0("TxSdma2DisallowedPacketErr", 558 SEES(TX_SDMA2_DISALLOWED_PACKET)), 559 /*19*/ FLAG_ENTRY0("TxSdma3DisallowedPacketErr", 560 SEES(TX_SDMA3_DISALLOWED_PACKET)), 561 /*20*/ FLAG_ENTRY0("TxSdma4DisallowedPacketErr", 562 SEES(TX_SDMA4_DISALLOWED_PACKET)), 563 /*21*/ FLAG_ENTRY0("TxSdma5DisallowedPacketErr", 564 SEES(TX_SDMA5_DISALLOWED_PACKET)), 565 /*22*/ FLAG_ENTRY0("TxSdma6DisallowedPacketErr", 566 SEES(TX_SDMA6_DISALLOWED_PACKET)), 567 /*23*/ FLAG_ENTRY0("TxSdma7DisallowedPacketErr", 568 SEES(TX_SDMA7_DISALLOWED_PACKET)), 569 /*24*/ FLAG_ENTRY0("TxSdma8DisallowedPacketErr", 570 SEES(TX_SDMA8_DISALLOWED_PACKET)), 571 /*25*/ FLAG_ENTRY0("TxSdma9DisallowedPacketErr", 572 SEES(TX_SDMA9_DISALLOWED_PACKET)), 573 /*26*/ FLAG_ENTRY0("TxSdma10DisallowedPacketErr", 574 SEES(TX_SDMA10_DISALLOWED_PACKET)), 575 /*27*/ FLAG_ENTRY0("TxSdma11DisallowedPacketErr", 576 SEES(TX_SDMA11_DISALLOWED_PACKET)), 577 /*28*/ FLAG_ENTRY0("TxSdma12DisallowedPacketErr", 578 SEES(TX_SDMA12_DISALLOWED_PACKET)), 579 /*29*/ FLAG_ENTRY0("TxSdma13DisallowedPacketErr", 580 SEES(TX_SDMA13_DISALLOWED_PACKET)), 581 /*30*/ FLAG_ENTRY0("TxSdma14DisallowedPacketErr", 582 SEES(TX_SDMA14_DISALLOWED_PACKET)), 583 /*31*/ FLAG_ENTRY0("TxSdma15DisallowedPacketErr", 584 SEES(TX_SDMA15_DISALLOWED_PACKET)), 585 /*32*/ FLAG_ENTRY0("TxLaunchFifo0UncOrParityErr", 586 SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY)), 587 /*33*/ FLAG_ENTRY0("TxLaunchFifo1UncOrParityErr", 588 SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY)), 589 /*34*/ FLAG_ENTRY0("TxLaunchFifo2UncOrParityErr", 590 SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY)), 591 /*35*/ FLAG_ENTRY0("TxLaunchFifo3UncOrParityErr", 592 SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY)), 593 /*36*/ FLAG_ENTRY0("TxLaunchFifo4UncOrParityErr", 594 SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY)), 595 /*37*/ FLAG_ENTRY0("TxLaunchFifo5UncOrParityErr", 596 SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY)), 597 /*38*/ FLAG_ENTRY0("TxLaunchFifo6UncOrParityErr", 598 SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY)), 599 /*39*/ FLAG_ENTRY0("TxLaunchFifo7UncOrParityErr", 600 SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY)), 601 /*40*/ FLAG_ENTRY0("TxLaunchFifo8UncOrParityErr", 602 SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY)), 603 /*41*/ FLAG_ENTRY0("TxCreditReturnParityErr", SEES(TX_CREDIT_RETURN_PARITY)), 604 /*42*/ FLAG_ENTRY0("TxSbHdrUncErr", SEES(TX_SB_HDR_UNC)), 605 /*43*/ FLAG_ENTRY0("TxReadSdmaMemoryUncErr", SEES(TX_READ_SDMA_MEMORY_UNC)), 606 /*44*/ FLAG_ENTRY0("TxReadPioMemoryUncErr", SEES(TX_READ_PIO_MEMORY_UNC)), 607 /*45*/ FLAG_ENTRY0("TxEgressFifoUncErr", SEES(TX_EGRESS_FIFO_UNC)), 608 /*46*/ FLAG_ENTRY0("TxHcrcInsertionErr", SEES(TX_HCRC_INSERTION)), 609 /*47*/ FLAG_ENTRY0("TxCreditReturnVLErr", SEES(TX_CREDIT_RETURN_VL)), 610 /*48*/ FLAG_ENTRY0("TxLaunchFifo0CorErr", SEES(TX_LAUNCH_FIFO0_COR)), 611 /*49*/ FLAG_ENTRY0("TxLaunchFifo1CorErr", SEES(TX_LAUNCH_FIFO1_COR)), 612 /*50*/ FLAG_ENTRY0("TxLaunchFifo2CorErr", SEES(TX_LAUNCH_FIFO2_COR)), 613 /*51*/ FLAG_ENTRY0("TxLaunchFifo3CorErr", SEES(TX_LAUNCH_FIFO3_COR)), 614 /*52*/ FLAG_ENTRY0("TxLaunchFifo4CorErr", SEES(TX_LAUNCH_FIFO4_COR)), 615 /*53*/ FLAG_ENTRY0("TxLaunchFifo5CorErr", SEES(TX_LAUNCH_FIFO5_COR)), 616 /*54*/ FLAG_ENTRY0("TxLaunchFifo6CorErr", SEES(TX_LAUNCH_FIFO6_COR)), 617 /*55*/ FLAG_ENTRY0("TxLaunchFifo7CorErr", SEES(TX_LAUNCH_FIFO7_COR)), 618 /*56*/ FLAG_ENTRY0("TxLaunchFifo8CorErr", SEES(TX_LAUNCH_FIFO8_COR)), 619 /*57*/ FLAG_ENTRY0("TxCreditOverrunErr", SEES(TX_CREDIT_OVERRUN)), 620 /*58*/ FLAG_ENTRY0("TxSbHdrCorErr", SEES(TX_SB_HDR_COR)), 621 /*59*/ FLAG_ENTRY0("TxReadSdmaMemoryCorErr", SEES(TX_READ_SDMA_MEMORY_COR)), 622 /*60*/ FLAG_ENTRY0("TxReadPioMemoryCorErr", SEES(TX_READ_PIO_MEMORY_COR)), 623 /*61*/ FLAG_ENTRY0("TxEgressFifoCorErr", SEES(TX_EGRESS_FIFO_COR)), 624 /*62*/ FLAG_ENTRY0("TxReadSdmaMemoryCsrUncErr", 625 SEES(TX_READ_SDMA_MEMORY_CSR_UNC)), 626 /*63*/ FLAG_ENTRY0("TxReadPioMemoryCsrUncErr", 627 SEES(TX_READ_PIO_MEMORY_CSR_UNC)), 628 }; 629 630 /* 631 * TXE Egress Error Info flags 632 */ 633 #define SEEI(text) SEND_EGRESS_ERR_INFO_##text##_ERR_SMASK 634 static const struct flag_table egress_err_info_flags[] = { 635 /* 0*/ FLAG_ENTRY0("Reserved", 0ull), 636 /* 1*/ FLAG_ENTRY0("VLErr", SEEI(VL)), 637 /* 2*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)), 638 /* 3*/ FLAG_ENTRY0("JobKeyErr", SEEI(JOB_KEY)), 639 /* 4*/ FLAG_ENTRY0("PartitionKeyErr", SEEI(PARTITION_KEY)), 640 /* 5*/ FLAG_ENTRY0("SLIDErr", SEEI(SLID)), 641 /* 6*/ FLAG_ENTRY0("OpcodeErr", SEEI(OPCODE)), 642 /* 7*/ FLAG_ENTRY0("VLMappingErr", SEEI(VL_MAPPING)), 643 /* 8*/ FLAG_ENTRY0("RawErr", SEEI(RAW)), 644 /* 9*/ FLAG_ENTRY0("RawIPv6Err", SEEI(RAW_IPV6)), 645 /*10*/ FLAG_ENTRY0("GRHErr", SEEI(GRH)), 646 /*11*/ FLAG_ENTRY0("BypassErr", SEEI(BYPASS)), 647 /*12*/ FLAG_ENTRY0("KDETHPacketsErr", SEEI(KDETH_PACKETS)), 648 /*13*/ FLAG_ENTRY0("NonKDETHPacketsErr", SEEI(NON_KDETH_PACKETS)), 649 /*14*/ FLAG_ENTRY0("TooSmallIBPacketsErr", SEEI(TOO_SMALL_IB_PACKETS)), 650 /*15*/ FLAG_ENTRY0("TooSmallBypassPacketsErr", SEEI(TOO_SMALL_BYPASS_PACKETS)), 651 /*16*/ FLAG_ENTRY0("PbcTestErr", SEEI(PBC_TEST)), 652 /*17*/ FLAG_ENTRY0("BadPktLenErr", SEEI(BAD_PKT_LEN)), 653 /*18*/ FLAG_ENTRY0("TooLongIBPacketErr", SEEI(TOO_LONG_IB_PACKET)), 654 /*19*/ FLAG_ENTRY0("TooLongBypassPacketsErr", SEEI(TOO_LONG_BYPASS_PACKETS)), 655 /*20*/ FLAG_ENTRY0("PbcStaticRateControlErr", SEEI(PBC_STATIC_RATE_CONTROL)), 656 /*21*/ FLAG_ENTRY0("BypassBadPktLenErr", SEEI(BAD_PKT_LEN)), 657 }; 658 659 /* TXE Egress errors that cause an SPC freeze */ 660 #define ALL_TXE_EGRESS_FREEZE_ERR \ 661 (SEES(TX_EGRESS_FIFO_UNDERRUN_OR_PARITY) \ 662 | SEES(TX_PIO_LAUNCH_INTF_PARITY) \ 663 | SEES(TX_SDMA_LAUNCH_INTF_PARITY) \ 664 | SEES(TX_SBRD_CTL_STATE_MACHINE_PARITY) \ 665 | SEES(TX_LAUNCH_CSR_PARITY) \ 666 | SEES(TX_SBRD_CTL_CSR_PARITY) \ 667 | SEES(TX_CONFIG_PARITY) \ 668 | SEES(TX_LAUNCH_FIFO0_UNC_OR_PARITY) \ 669 | SEES(TX_LAUNCH_FIFO1_UNC_OR_PARITY) \ 670 | SEES(TX_LAUNCH_FIFO2_UNC_OR_PARITY) \ 671 | SEES(TX_LAUNCH_FIFO3_UNC_OR_PARITY) \ 672 | SEES(TX_LAUNCH_FIFO4_UNC_OR_PARITY) \ 673 | SEES(TX_LAUNCH_FIFO5_UNC_OR_PARITY) \ 674 | SEES(TX_LAUNCH_FIFO6_UNC_OR_PARITY) \ 675 | SEES(TX_LAUNCH_FIFO7_UNC_OR_PARITY) \ 676 | SEES(TX_LAUNCH_FIFO8_UNC_OR_PARITY) \ 677 | SEES(TX_CREDIT_RETURN_PARITY)) 678 679 /* 680 * TXE Send error flags 681 */ 682 #define SES(name) SEND_ERR_STATUS_SEND_##name##_ERR_SMASK 683 static const struct flag_table send_err_status_flags[] = { 684 /* 0*/ FLAG_ENTRY0("SendCsrParityErr", SES(CSR_PARITY)), 685 /* 1*/ FLAG_ENTRY0("SendCsrReadBadAddrErr", SES(CSR_READ_BAD_ADDR)), 686 /* 2*/ FLAG_ENTRY0("SendCsrWriteBadAddrErr", SES(CSR_WRITE_BAD_ADDR)) 687 }; 688 689 /* 690 * TXE Send Context Error flags and consequences 691 */ 692 static const struct flag_table sc_err_status_flags[] = { 693 /* 0*/ FLAG_ENTRY("InconsistentSop", 694 SEC_PACKET_DROPPED | SEC_SC_HALTED, 695 SEND_CTXT_ERR_STATUS_PIO_INCONSISTENT_SOP_ERR_SMASK), 696 /* 1*/ FLAG_ENTRY("DisallowedPacket", 697 SEC_PACKET_DROPPED | SEC_SC_HALTED, 698 SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK), 699 /* 2*/ FLAG_ENTRY("WriteCrossesBoundary", 700 SEC_WRITE_DROPPED | SEC_SC_HALTED, 701 SEND_CTXT_ERR_STATUS_PIO_WRITE_CROSSES_BOUNDARY_ERR_SMASK), 702 /* 3*/ FLAG_ENTRY("WriteOverflow", 703 SEC_WRITE_DROPPED | SEC_SC_HALTED, 704 SEND_CTXT_ERR_STATUS_PIO_WRITE_OVERFLOW_ERR_SMASK), 705 /* 4*/ FLAG_ENTRY("WriteOutOfBounds", 706 SEC_WRITE_DROPPED | SEC_SC_HALTED, 707 SEND_CTXT_ERR_STATUS_PIO_WRITE_OUT_OF_BOUNDS_ERR_SMASK), 708 /* 5-63 reserved*/ 709 }; 710 711 /* 712 * RXE Receive Error flags 713 */ 714 #define RXES(name) RCV_ERR_STATUS_RX_##name##_ERR_SMASK 715 static const struct flag_table rxe_err_status_flags[] = { 716 /* 0*/ FLAG_ENTRY0("RxDmaCsrCorErr", RXES(DMA_CSR_COR)), 717 /* 1*/ FLAG_ENTRY0("RxDcIntfParityErr", RXES(DC_INTF_PARITY)), 718 /* 2*/ FLAG_ENTRY0("RxRcvHdrUncErr", RXES(RCV_HDR_UNC)), 719 /* 3*/ FLAG_ENTRY0("RxRcvHdrCorErr", RXES(RCV_HDR_COR)), 720 /* 4*/ FLAG_ENTRY0("RxRcvDataUncErr", RXES(RCV_DATA_UNC)), 721 /* 5*/ FLAG_ENTRY0("RxRcvDataCorErr", RXES(RCV_DATA_COR)), 722 /* 6*/ FLAG_ENTRY0("RxRcvQpMapTableUncErr", RXES(RCV_QP_MAP_TABLE_UNC)), 723 /* 7*/ FLAG_ENTRY0("RxRcvQpMapTableCorErr", RXES(RCV_QP_MAP_TABLE_COR)), 724 /* 8*/ FLAG_ENTRY0("RxRcvCsrParityErr", RXES(RCV_CSR_PARITY)), 725 /* 9*/ FLAG_ENTRY0("RxDcSopEopParityErr", RXES(DC_SOP_EOP_PARITY)), 726 /*10*/ FLAG_ENTRY0("RxDmaFlagUncErr", RXES(DMA_FLAG_UNC)), 727 /*11*/ FLAG_ENTRY0("RxDmaFlagCorErr", RXES(DMA_FLAG_COR)), 728 /*12*/ FLAG_ENTRY0("RxRcvFsmEncodingErr", RXES(RCV_FSM_ENCODING)), 729 /*13*/ FLAG_ENTRY0("RxRbufFreeListUncErr", RXES(RBUF_FREE_LIST_UNC)), 730 /*14*/ FLAG_ENTRY0("RxRbufFreeListCorErr", RXES(RBUF_FREE_LIST_COR)), 731 /*15*/ FLAG_ENTRY0("RxRbufLookupDesRegUncErr", RXES(RBUF_LOOKUP_DES_REG_UNC)), 732 /*16*/ FLAG_ENTRY0("RxRbufLookupDesRegUncCorErr", 733 RXES(RBUF_LOOKUP_DES_REG_UNC_COR)), 734 /*17*/ FLAG_ENTRY0("RxRbufLookupDesUncErr", RXES(RBUF_LOOKUP_DES_UNC)), 735 /*18*/ FLAG_ENTRY0("RxRbufLookupDesCorErr", RXES(RBUF_LOOKUP_DES_COR)), 736 /*19*/ FLAG_ENTRY0("RxRbufBlockListReadUncErr", 737 RXES(RBUF_BLOCK_LIST_READ_UNC)), 738 /*20*/ FLAG_ENTRY0("RxRbufBlockListReadCorErr", 739 RXES(RBUF_BLOCK_LIST_READ_COR)), 740 /*21*/ FLAG_ENTRY0("RxRbufCsrQHeadBufNumParityErr", 741 RXES(RBUF_CSR_QHEAD_BUF_NUM_PARITY)), 742 /*22*/ FLAG_ENTRY0("RxRbufCsrQEntCntParityErr", 743 RXES(RBUF_CSR_QENT_CNT_PARITY)), 744 /*23*/ FLAG_ENTRY0("RxRbufCsrQNextBufParityErr", 745 RXES(RBUF_CSR_QNEXT_BUF_PARITY)), 746 /*24*/ FLAG_ENTRY0("RxRbufCsrQVldBitParityErr", 747 RXES(RBUF_CSR_QVLD_BIT_PARITY)), 748 /*25*/ FLAG_ENTRY0("RxRbufCsrQHdPtrParityErr", RXES(RBUF_CSR_QHD_PTR_PARITY)), 749 /*26*/ FLAG_ENTRY0("RxRbufCsrQTlPtrParityErr", RXES(RBUF_CSR_QTL_PTR_PARITY)), 750 /*27*/ FLAG_ENTRY0("RxRbufCsrQNumOfPktParityErr", 751 RXES(RBUF_CSR_QNUM_OF_PKT_PARITY)), 752 /*28*/ FLAG_ENTRY0("RxRbufCsrQEOPDWParityErr", RXES(RBUF_CSR_QEOPDW_PARITY)), 753 /*29*/ FLAG_ENTRY0("RxRbufCtxIdParityErr", RXES(RBUF_CTX_ID_PARITY)), 754 /*30*/ FLAG_ENTRY0("RxRBufBadLookupErr", RXES(RBUF_BAD_LOOKUP)), 755 /*31*/ FLAG_ENTRY0("RxRbufFullErr", RXES(RBUF_FULL)), 756 /*32*/ FLAG_ENTRY0("RxRbufEmptyErr", RXES(RBUF_EMPTY)), 757 /*33*/ FLAG_ENTRY0("RxRbufFlRdAddrParityErr", RXES(RBUF_FL_RD_ADDR_PARITY)), 758 /*34*/ FLAG_ENTRY0("RxRbufFlWrAddrParityErr", RXES(RBUF_FL_WR_ADDR_PARITY)), 759 /*35*/ FLAG_ENTRY0("RxRbufFlInitdoneParityErr", 760 RXES(RBUF_FL_INITDONE_PARITY)), 761 /*36*/ FLAG_ENTRY0("RxRbufFlInitWrAddrParityErr", 762 RXES(RBUF_FL_INIT_WR_ADDR_PARITY)), 763 /*37*/ FLAG_ENTRY0("RxRbufNextFreeBufUncErr", RXES(RBUF_NEXT_FREE_BUF_UNC)), 764 /*38*/ FLAG_ENTRY0("RxRbufNextFreeBufCorErr", RXES(RBUF_NEXT_FREE_BUF_COR)), 765 /*39*/ FLAG_ENTRY0("RxLookupDesPart1UncErr", RXES(LOOKUP_DES_PART1_UNC)), 766 /*40*/ FLAG_ENTRY0("RxLookupDesPart1UncCorErr", 767 RXES(LOOKUP_DES_PART1_UNC_COR)), 768 /*41*/ FLAG_ENTRY0("RxLookupDesPart2ParityErr", 769 RXES(LOOKUP_DES_PART2_PARITY)), 770 /*42*/ FLAG_ENTRY0("RxLookupRcvArrayUncErr", RXES(LOOKUP_RCV_ARRAY_UNC)), 771 /*43*/ FLAG_ENTRY0("RxLookupRcvArrayCorErr", RXES(LOOKUP_RCV_ARRAY_COR)), 772 /*44*/ FLAG_ENTRY0("RxLookupCsrParityErr", RXES(LOOKUP_CSR_PARITY)), 773 /*45*/ FLAG_ENTRY0("RxHqIntrCsrParityErr", RXES(HQ_INTR_CSR_PARITY)), 774 /*46*/ FLAG_ENTRY0("RxHqIntrFsmErr", RXES(HQ_INTR_FSM)), 775 /*47*/ FLAG_ENTRY0("RxRbufDescPart1UncErr", RXES(RBUF_DESC_PART1_UNC)), 776 /*48*/ FLAG_ENTRY0("RxRbufDescPart1CorErr", RXES(RBUF_DESC_PART1_COR)), 777 /*49*/ FLAG_ENTRY0("RxRbufDescPart2UncErr", RXES(RBUF_DESC_PART2_UNC)), 778 /*50*/ FLAG_ENTRY0("RxRbufDescPart2CorErr", RXES(RBUF_DESC_PART2_COR)), 779 /*51*/ FLAG_ENTRY0("RxDmaHdrFifoRdUncErr", RXES(DMA_HDR_FIFO_RD_UNC)), 780 /*52*/ FLAG_ENTRY0("RxDmaHdrFifoRdCorErr", RXES(DMA_HDR_FIFO_RD_COR)), 781 /*53*/ FLAG_ENTRY0("RxDmaDataFifoRdUncErr", RXES(DMA_DATA_FIFO_RD_UNC)), 782 /*54*/ FLAG_ENTRY0("RxDmaDataFifoRdCorErr", RXES(DMA_DATA_FIFO_RD_COR)), 783 /*55*/ FLAG_ENTRY0("RxRbufDataUncErr", RXES(RBUF_DATA_UNC)), 784 /*56*/ FLAG_ENTRY0("RxRbufDataCorErr", RXES(RBUF_DATA_COR)), 785 /*57*/ FLAG_ENTRY0("RxDmaCsrParityErr", RXES(DMA_CSR_PARITY)), 786 /*58*/ FLAG_ENTRY0("RxDmaEqFsmEncodingErr", RXES(DMA_EQ_FSM_ENCODING)), 787 /*59*/ FLAG_ENTRY0("RxDmaDqFsmEncodingErr", RXES(DMA_DQ_FSM_ENCODING)), 788 /*60*/ FLAG_ENTRY0("RxDmaCsrUncErr", RXES(DMA_CSR_UNC)), 789 /*61*/ FLAG_ENTRY0("RxCsrReadBadAddrErr", RXES(CSR_READ_BAD_ADDR)), 790 /*62*/ FLAG_ENTRY0("RxCsrWriteBadAddrErr", RXES(CSR_WRITE_BAD_ADDR)), 791 /*63*/ FLAG_ENTRY0("RxCsrParityErr", RXES(CSR_PARITY)) 792 }; 793 794 /* RXE errors that will trigger an SPC freeze */ 795 #define ALL_RXE_FREEZE_ERR \ 796 (RCV_ERR_STATUS_RX_RCV_QP_MAP_TABLE_UNC_ERR_SMASK \ 797 | RCV_ERR_STATUS_RX_RCV_CSR_PARITY_ERR_SMASK \ 798 | RCV_ERR_STATUS_RX_DMA_FLAG_UNC_ERR_SMASK \ 799 | RCV_ERR_STATUS_RX_RCV_FSM_ENCODING_ERR_SMASK \ 800 | RCV_ERR_STATUS_RX_RBUF_FREE_LIST_UNC_ERR_SMASK \ 801 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_ERR_SMASK \ 802 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR_SMASK \ 803 | RCV_ERR_STATUS_RX_RBUF_LOOKUP_DES_UNC_ERR_SMASK \ 804 | RCV_ERR_STATUS_RX_RBUF_BLOCK_LIST_READ_UNC_ERR_SMASK \ 805 | RCV_ERR_STATUS_RX_RBUF_CSR_QHEAD_BUF_NUM_PARITY_ERR_SMASK \ 806 | RCV_ERR_STATUS_RX_RBUF_CSR_QENT_CNT_PARITY_ERR_SMASK \ 807 | RCV_ERR_STATUS_RX_RBUF_CSR_QNEXT_BUF_PARITY_ERR_SMASK \ 808 | RCV_ERR_STATUS_RX_RBUF_CSR_QVLD_BIT_PARITY_ERR_SMASK \ 809 | RCV_ERR_STATUS_RX_RBUF_CSR_QHD_PTR_PARITY_ERR_SMASK \ 810 | RCV_ERR_STATUS_RX_RBUF_CSR_QTL_PTR_PARITY_ERR_SMASK \ 811 | RCV_ERR_STATUS_RX_RBUF_CSR_QNUM_OF_PKT_PARITY_ERR_SMASK \ 812 | RCV_ERR_STATUS_RX_RBUF_CSR_QEOPDW_PARITY_ERR_SMASK \ 813 | RCV_ERR_STATUS_RX_RBUF_CTX_ID_PARITY_ERR_SMASK \ 814 | RCV_ERR_STATUS_RX_RBUF_BAD_LOOKUP_ERR_SMASK \ 815 | RCV_ERR_STATUS_RX_RBUF_FULL_ERR_SMASK \ 816 | RCV_ERR_STATUS_RX_RBUF_EMPTY_ERR_SMASK \ 817 | RCV_ERR_STATUS_RX_RBUF_FL_RD_ADDR_PARITY_ERR_SMASK \ 818 | RCV_ERR_STATUS_RX_RBUF_FL_WR_ADDR_PARITY_ERR_SMASK \ 819 | RCV_ERR_STATUS_RX_RBUF_FL_INITDONE_PARITY_ERR_SMASK \ 820 | RCV_ERR_STATUS_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR_SMASK \ 821 | RCV_ERR_STATUS_RX_RBUF_NEXT_FREE_BUF_UNC_ERR_SMASK \ 822 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_ERR_SMASK \ 823 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART1_UNC_COR_ERR_SMASK \ 824 | RCV_ERR_STATUS_RX_LOOKUP_DES_PART2_PARITY_ERR_SMASK \ 825 | RCV_ERR_STATUS_RX_LOOKUP_RCV_ARRAY_UNC_ERR_SMASK \ 826 | RCV_ERR_STATUS_RX_LOOKUP_CSR_PARITY_ERR_SMASK \ 827 | RCV_ERR_STATUS_RX_HQ_INTR_CSR_PARITY_ERR_SMASK \ 828 | RCV_ERR_STATUS_RX_HQ_INTR_FSM_ERR_SMASK \ 829 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_UNC_ERR_SMASK \ 830 | RCV_ERR_STATUS_RX_RBUF_DESC_PART1_COR_ERR_SMASK \ 831 | RCV_ERR_STATUS_RX_RBUF_DESC_PART2_UNC_ERR_SMASK \ 832 | RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK \ 833 | RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK \ 834 | RCV_ERR_STATUS_RX_RBUF_DATA_UNC_ERR_SMASK \ 835 | RCV_ERR_STATUS_RX_DMA_CSR_PARITY_ERR_SMASK \ 836 | RCV_ERR_STATUS_RX_DMA_EQ_FSM_ENCODING_ERR_SMASK \ 837 | RCV_ERR_STATUS_RX_DMA_DQ_FSM_ENCODING_ERR_SMASK \ 838 | RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK \ 839 | RCV_ERR_STATUS_RX_CSR_PARITY_ERR_SMASK) 840 841 #define RXE_FREEZE_ABORT_MASK \ 842 (RCV_ERR_STATUS_RX_DMA_CSR_UNC_ERR_SMASK | \ 843 RCV_ERR_STATUS_RX_DMA_HDR_FIFO_RD_UNC_ERR_SMASK | \ 844 RCV_ERR_STATUS_RX_DMA_DATA_FIFO_RD_UNC_ERR_SMASK) 845 846 /* 847 * DCC Error Flags 848 */ 849 #define DCCE(name) DCC_ERR_FLG_##name##_SMASK 850 static const struct flag_table dcc_err_flags[] = { 851 FLAG_ENTRY0("bad_l2_err", DCCE(BAD_L2_ERR)), 852 FLAG_ENTRY0("bad_sc_err", DCCE(BAD_SC_ERR)), 853 FLAG_ENTRY0("bad_mid_tail_err", DCCE(BAD_MID_TAIL_ERR)), 854 FLAG_ENTRY0("bad_preemption_err", DCCE(BAD_PREEMPTION_ERR)), 855 FLAG_ENTRY0("preemption_err", DCCE(PREEMPTION_ERR)), 856 FLAG_ENTRY0("preemptionvl15_err", DCCE(PREEMPTIONVL15_ERR)), 857 FLAG_ENTRY0("bad_vl_marker_err", DCCE(BAD_VL_MARKER_ERR)), 858 FLAG_ENTRY0("bad_dlid_target_err", DCCE(BAD_DLID_TARGET_ERR)), 859 FLAG_ENTRY0("bad_lver_err", DCCE(BAD_LVER_ERR)), 860 FLAG_ENTRY0("uncorrectable_err", DCCE(UNCORRECTABLE_ERR)), 861 FLAG_ENTRY0("bad_crdt_ack_err", DCCE(BAD_CRDT_ACK_ERR)), 862 FLAG_ENTRY0("unsup_pkt_type", DCCE(UNSUP_PKT_TYPE)), 863 FLAG_ENTRY0("bad_ctrl_flit_err", DCCE(BAD_CTRL_FLIT_ERR)), 864 FLAG_ENTRY0("event_cntr_parity_err", DCCE(EVENT_CNTR_PARITY_ERR)), 865 FLAG_ENTRY0("event_cntr_rollover_err", DCCE(EVENT_CNTR_ROLLOVER_ERR)), 866 FLAG_ENTRY0("link_err", DCCE(LINK_ERR)), 867 FLAG_ENTRY0("misc_cntr_rollover_err", DCCE(MISC_CNTR_ROLLOVER_ERR)), 868 FLAG_ENTRY0("bad_ctrl_dist_err", DCCE(BAD_CTRL_DIST_ERR)), 869 FLAG_ENTRY0("bad_tail_dist_err", DCCE(BAD_TAIL_DIST_ERR)), 870 FLAG_ENTRY0("bad_head_dist_err", DCCE(BAD_HEAD_DIST_ERR)), 871 FLAG_ENTRY0("nonvl15_state_err", DCCE(NONVL15_STATE_ERR)), 872 FLAG_ENTRY0("vl15_multi_err", DCCE(VL15_MULTI_ERR)), 873 FLAG_ENTRY0("bad_pkt_length_err", DCCE(BAD_PKT_LENGTH_ERR)), 874 FLAG_ENTRY0("unsup_vl_err", DCCE(UNSUP_VL_ERR)), 875 FLAG_ENTRY0("perm_nvl15_err", DCCE(PERM_NVL15_ERR)), 876 FLAG_ENTRY0("slid_zero_err", DCCE(SLID_ZERO_ERR)), 877 FLAG_ENTRY0("dlid_zero_err", DCCE(DLID_ZERO_ERR)), 878 FLAG_ENTRY0("length_mtu_err", DCCE(LENGTH_MTU_ERR)), 879 FLAG_ENTRY0("rx_early_drop_err", DCCE(RX_EARLY_DROP_ERR)), 880 FLAG_ENTRY0("late_short_err", DCCE(LATE_SHORT_ERR)), 881 FLAG_ENTRY0("late_long_err", DCCE(LATE_LONG_ERR)), 882 FLAG_ENTRY0("late_ebp_err", DCCE(LATE_EBP_ERR)), 883 FLAG_ENTRY0("fpe_tx_fifo_ovflw_err", DCCE(FPE_TX_FIFO_OVFLW_ERR)), 884 FLAG_ENTRY0("fpe_tx_fifo_unflw_err", DCCE(FPE_TX_FIFO_UNFLW_ERR)), 885 FLAG_ENTRY0("csr_access_blocked_host", DCCE(CSR_ACCESS_BLOCKED_HOST)), 886 FLAG_ENTRY0("csr_access_blocked_uc", DCCE(CSR_ACCESS_BLOCKED_UC)), 887 FLAG_ENTRY0("tx_ctrl_parity_err", DCCE(TX_CTRL_PARITY_ERR)), 888 FLAG_ENTRY0("tx_ctrl_parity_mbe_err", DCCE(TX_CTRL_PARITY_MBE_ERR)), 889 FLAG_ENTRY0("tx_sc_parity_err", DCCE(TX_SC_PARITY_ERR)), 890 FLAG_ENTRY0("rx_ctrl_parity_mbe_err", DCCE(RX_CTRL_PARITY_MBE_ERR)), 891 FLAG_ENTRY0("csr_parity_err", DCCE(CSR_PARITY_ERR)), 892 FLAG_ENTRY0("csr_inval_addr", DCCE(CSR_INVAL_ADDR)), 893 FLAG_ENTRY0("tx_byte_shft_parity_err", DCCE(TX_BYTE_SHFT_PARITY_ERR)), 894 FLAG_ENTRY0("rx_byte_shft_parity_err", DCCE(RX_BYTE_SHFT_PARITY_ERR)), 895 FLAG_ENTRY0("fmconfig_err", DCCE(FMCONFIG_ERR)), 896 FLAG_ENTRY0("rcvport_err", DCCE(RCVPORT_ERR)), 897 }; 898 899 /* 900 * LCB error flags 901 */ 902 #define LCBE(name) DC_LCB_ERR_FLG_##name##_SMASK 903 static const struct flag_table lcb_err_flags[] = { 904 /* 0*/ FLAG_ENTRY0("CSR_PARITY_ERR", LCBE(CSR_PARITY_ERR)), 905 /* 1*/ FLAG_ENTRY0("INVALID_CSR_ADDR", LCBE(INVALID_CSR_ADDR)), 906 /* 2*/ FLAG_ENTRY0("RST_FOR_FAILED_DESKEW", LCBE(RST_FOR_FAILED_DESKEW)), 907 /* 3*/ FLAG_ENTRY0("ALL_LNS_FAILED_REINIT_TEST", 908 LCBE(ALL_LNS_FAILED_REINIT_TEST)), 909 /* 4*/ FLAG_ENTRY0("LOST_REINIT_STALL_OR_TOS", LCBE(LOST_REINIT_STALL_OR_TOS)), 910 /* 5*/ FLAG_ENTRY0("TX_LESS_THAN_FOUR_LNS", LCBE(TX_LESS_THAN_FOUR_LNS)), 911 /* 6*/ FLAG_ENTRY0("RX_LESS_THAN_FOUR_LNS", LCBE(RX_LESS_THAN_FOUR_LNS)), 912 /* 7*/ FLAG_ENTRY0("SEQ_CRC_ERR", LCBE(SEQ_CRC_ERR)), 913 /* 8*/ FLAG_ENTRY0("REINIT_FROM_PEER", LCBE(REINIT_FROM_PEER)), 914 /* 9*/ FLAG_ENTRY0("REINIT_FOR_LN_DEGRADE", LCBE(REINIT_FOR_LN_DEGRADE)), 915 /*10*/ FLAG_ENTRY0("CRC_ERR_CNT_HIT_LIMIT", LCBE(CRC_ERR_CNT_HIT_LIMIT)), 916 /*11*/ FLAG_ENTRY0("RCLK_STOPPED", LCBE(RCLK_STOPPED)), 917 /*12*/ FLAG_ENTRY0("UNEXPECTED_REPLAY_MARKER", LCBE(UNEXPECTED_REPLAY_MARKER)), 918 /*13*/ FLAG_ENTRY0("UNEXPECTED_ROUND_TRIP_MARKER", 919 LCBE(UNEXPECTED_ROUND_TRIP_MARKER)), 920 /*14*/ FLAG_ENTRY0("ILLEGAL_NULL_LTP", LCBE(ILLEGAL_NULL_LTP)), 921 /*15*/ FLAG_ENTRY0("ILLEGAL_FLIT_ENCODING", LCBE(ILLEGAL_FLIT_ENCODING)), 922 /*16*/ FLAG_ENTRY0("FLIT_INPUT_BUF_OFLW", LCBE(FLIT_INPUT_BUF_OFLW)), 923 /*17*/ FLAG_ENTRY0("VL_ACK_INPUT_BUF_OFLW", LCBE(VL_ACK_INPUT_BUF_OFLW)), 924 /*18*/ FLAG_ENTRY0("VL_ACK_INPUT_PARITY_ERR", LCBE(VL_ACK_INPUT_PARITY_ERR)), 925 /*19*/ FLAG_ENTRY0("VL_ACK_INPUT_WRONG_CRC_MODE", 926 LCBE(VL_ACK_INPUT_WRONG_CRC_MODE)), 927 /*20*/ FLAG_ENTRY0("FLIT_INPUT_BUF_MBE", LCBE(FLIT_INPUT_BUF_MBE)), 928 /*21*/ FLAG_ENTRY0("FLIT_INPUT_BUF_SBE", LCBE(FLIT_INPUT_BUF_SBE)), 929 /*22*/ FLAG_ENTRY0("REPLAY_BUF_MBE", LCBE(REPLAY_BUF_MBE)), 930 /*23*/ FLAG_ENTRY0("REPLAY_BUF_SBE", LCBE(REPLAY_BUF_SBE)), 931 /*24*/ FLAG_ENTRY0("CREDIT_RETURN_FLIT_MBE", LCBE(CREDIT_RETURN_FLIT_MBE)), 932 /*25*/ FLAG_ENTRY0("RST_FOR_LINK_TIMEOUT", LCBE(RST_FOR_LINK_TIMEOUT)), 933 /*26*/ FLAG_ENTRY0("RST_FOR_INCOMPLT_RND_TRIP", 934 LCBE(RST_FOR_INCOMPLT_RND_TRIP)), 935 /*27*/ FLAG_ENTRY0("HOLD_REINIT", LCBE(HOLD_REINIT)), 936 /*28*/ FLAG_ENTRY0("NEG_EDGE_LINK_TRANSFER_ACTIVE", 937 LCBE(NEG_EDGE_LINK_TRANSFER_ACTIVE)), 938 /*29*/ FLAG_ENTRY0("REDUNDANT_FLIT_PARITY_ERR", 939 LCBE(REDUNDANT_FLIT_PARITY_ERR)) 940 }; 941 942 /* 943 * DC8051 Error Flags 944 */ 945 #define D8E(name) DC_DC8051_ERR_FLG_##name##_SMASK 946 static const struct flag_table dc8051_err_flags[] = { 947 FLAG_ENTRY0("SET_BY_8051", D8E(SET_BY_8051)), 948 FLAG_ENTRY0("LOST_8051_HEART_BEAT", D8E(LOST_8051_HEART_BEAT)), 949 FLAG_ENTRY0("CRAM_MBE", D8E(CRAM_MBE)), 950 FLAG_ENTRY0("CRAM_SBE", D8E(CRAM_SBE)), 951 FLAG_ENTRY0("DRAM_MBE", D8E(DRAM_MBE)), 952 FLAG_ENTRY0("DRAM_SBE", D8E(DRAM_SBE)), 953 FLAG_ENTRY0("IRAM_MBE", D8E(IRAM_MBE)), 954 FLAG_ENTRY0("IRAM_SBE", D8E(IRAM_SBE)), 955 FLAG_ENTRY0("UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES", 956 D8E(UNMATCHED_SECURE_MSG_ACROSS_BCC_LANES)), 957 FLAG_ENTRY0("INVALID_CSR_ADDR", D8E(INVALID_CSR_ADDR)), 958 }; 959 960 /* 961 * DC8051 Information Error flags 962 * 963 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.ERROR field. 964 */ 965 static const struct flag_table dc8051_info_err_flags[] = { 966 FLAG_ENTRY0("Spico ROM check failed", SPICO_ROM_FAILED), 967 FLAG_ENTRY0("Unknown frame received", UNKNOWN_FRAME), 968 FLAG_ENTRY0("Target BER not met", TARGET_BER_NOT_MET), 969 FLAG_ENTRY0("Serdes internal loopback failure", 970 FAILED_SERDES_INTERNAL_LOOPBACK), 971 FLAG_ENTRY0("Failed SerDes init", FAILED_SERDES_INIT), 972 FLAG_ENTRY0("Failed LNI(Polling)", FAILED_LNI_POLLING), 973 FLAG_ENTRY0("Failed LNI(Debounce)", FAILED_LNI_DEBOUNCE), 974 FLAG_ENTRY0("Failed LNI(EstbComm)", FAILED_LNI_ESTBCOMM), 975 FLAG_ENTRY0("Failed LNI(OptEq)", FAILED_LNI_OPTEQ), 976 FLAG_ENTRY0("Failed LNI(VerifyCap_1)", FAILED_LNI_VERIFY_CAP1), 977 FLAG_ENTRY0("Failed LNI(VerifyCap_2)", FAILED_LNI_VERIFY_CAP2), 978 FLAG_ENTRY0("Failed LNI(ConfigLT)", FAILED_LNI_CONFIGLT), 979 FLAG_ENTRY0("Host Handshake Timeout", HOST_HANDSHAKE_TIMEOUT), 980 FLAG_ENTRY0("External Device Request Timeout", 981 EXTERNAL_DEVICE_REQ_TIMEOUT), 982 }; 983 984 /* 985 * DC8051 Information Host Information flags 986 * 987 * Flags in DC8051_DBG_ERR_INFO_SET_BY_8051.HOST_MSG field. 988 */ 989 static const struct flag_table dc8051_info_host_msg_flags[] = { 990 FLAG_ENTRY0("Host request done", 0x0001), 991 FLAG_ENTRY0("BC PWR_MGM message", 0x0002), 992 FLAG_ENTRY0("BC SMA message", 0x0004), 993 FLAG_ENTRY0("BC Unknown message (BCC)", 0x0008), 994 FLAG_ENTRY0("BC Unknown message (LCB)", 0x0010), 995 FLAG_ENTRY0("External device config request", 0x0020), 996 FLAG_ENTRY0("VerifyCap all frames received", 0x0040), 997 FLAG_ENTRY0("LinkUp achieved", 0x0080), 998 FLAG_ENTRY0("Link going down", 0x0100), 999 FLAG_ENTRY0("Link width downgraded", 0x0200), 1000 }; 1001 1002 static u32 encoded_size(u32 size); 1003 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate); 1004 static int set_physical_link_state(struct hfi1_devdata *dd, u64 state); 1005 static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management, 1006 u8 *continuous); 1007 static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z, 1008 u8 *vcu, u16 *vl15buf, u8 *crc_sizes); 1009 static void read_vc_remote_link_width(struct hfi1_devdata *dd, 1010 u8 *remote_tx_rate, u16 *link_widths); 1011 static void read_vc_local_link_mode(struct hfi1_devdata *dd, u8 *misc_bits, 1012 u8 *flag_bits, u16 *link_widths); 1013 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id, 1014 u8 *device_rev); 1015 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx); 1016 static int read_tx_settings(struct hfi1_devdata *dd, u8 *enable_lane_tx, 1017 u8 *tx_polarity_inversion, 1018 u8 *rx_polarity_inversion, u8 *max_rate); 1019 static void handle_sdma_eng_err(struct hfi1_devdata *dd, 1020 unsigned int context, u64 err_status); 1021 static void handle_qsfp_int(struct hfi1_devdata *dd, u32 source, u64 reg); 1022 static void handle_dcc_err(struct hfi1_devdata *dd, 1023 unsigned int context, u64 err_status); 1024 static void handle_lcb_err(struct hfi1_devdata *dd, 1025 unsigned int context, u64 err_status); 1026 static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg); 1027 static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1028 static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1029 static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1030 static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1031 static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1032 static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1033 static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg); 1034 static void set_partition_keys(struct hfi1_pportdata *ppd); 1035 static const char *link_state_name(u32 state); 1036 static const char *link_state_reason_name(struct hfi1_pportdata *ppd, 1037 u32 state); 1038 static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data, 1039 u64 *out_data); 1040 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data); 1041 static int thermal_init(struct hfi1_devdata *dd); 1042 1043 static void update_statusp(struct hfi1_pportdata *ppd, u32 state); 1044 static int wait_phys_link_offline_substates(struct hfi1_pportdata *ppd, 1045 int msecs); 1046 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state, 1047 int msecs); 1048 static void log_state_transition(struct hfi1_pportdata *ppd, u32 state); 1049 static void log_physical_state(struct hfi1_pportdata *ppd, u32 state); 1050 static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state, 1051 int msecs); 1052 static int wait_phys_link_out_of_offline(struct hfi1_pportdata *ppd, 1053 int msecs); 1054 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc); 1055 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr); 1056 static void handle_temp_err(struct hfi1_devdata *dd); 1057 static void dc_shutdown(struct hfi1_devdata *dd); 1058 static void dc_start(struct hfi1_devdata *dd); 1059 static int qos_rmt_entries(unsigned int n_krcv_queues, unsigned int *mp, 1060 unsigned int *np); 1061 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd); 1062 static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms); 1063 static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index); 1064 static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width); 1065 1066 /* 1067 * Error interrupt table entry. This is used as input to the interrupt 1068 * "clear down" routine used for all second tier error interrupt register. 1069 * Second tier interrupt registers have a single bit representing them 1070 * in the top-level CceIntStatus. 1071 */ 1072 struct err_reg_info { 1073 u32 status; /* status CSR offset */ 1074 u32 clear; /* clear CSR offset */ 1075 u32 mask; /* mask CSR offset */ 1076 void (*handler)(struct hfi1_devdata *dd, u32 source, u64 reg); 1077 const char *desc; 1078 }; 1079 1080 #define NUM_MISC_ERRS (IS_GENERAL_ERR_END + 1 - IS_GENERAL_ERR_START) 1081 #define NUM_DC_ERRS (IS_DC_END + 1 - IS_DC_START) 1082 #define NUM_VARIOUS (IS_VARIOUS_END + 1 - IS_VARIOUS_START) 1083 1084 /* 1085 * Helpers for building HFI and DC error interrupt table entries. Different 1086 * helpers are needed because of inconsistent register names. 1087 */ 1088 #define EE(reg, handler, desc) \ 1089 { reg##_STATUS, reg##_CLEAR, reg##_MASK, \ 1090 handler, desc } 1091 #define DC_EE1(reg, handler, desc) \ 1092 { reg##_FLG, reg##_FLG_CLR, reg##_FLG_EN, handler, desc } 1093 #define DC_EE2(reg, handler, desc) \ 1094 { reg##_FLG, reg##_CLR, reg##_EN, handler, desc } 1095 1096 /* 1097 * Table of the "misc" grouping of error interrupts. Each entry refers to 1098 * another register containing more information. 1099 */ 1100 static const struct err_reg_info misc_errs[NUM_MISC_ERRS] = { 1101 /* 0*/ EE(CCE_ERR, handle_cce_err, "CceErr"), 1102 /* 1*/ EE(RCV_ERR, handle_rxe_err, "RxeErr"), 1103 /* 2*/ EE(MISC_ERR, handle_misc_err, "MiscErr"), 1104 /* 3*/ { 0, 0, 0, NULL }, /* reserved */ 1105 /* 4*/ EE(SEND_PIO_ERR, handle_pio_err, "PioErr"), 1106 /* 5*/ EE(SEND_DMA_ERR, handle_sdma_err, "SDmaErr"), 1107 /* 6*/ EE(SEND_EGRESS_ERR, handle_egress_err, "EgressErr"), 1108 /* 7*/ EE(SEND_ERR, handle_txe_err, "TxeErr") 1109 /* the rest are reserved */ 1110 }; 1111 1112 /* 1113 * Index into the Various section of the interrupt sources 1114 * corresponding to the Critical Temperature interrupt. 1115 */ 1116 #define TCRIT_INT_SOURCE 4 1117 1118 /* 1119 * SDMA error interrupt entry - refers to another register containing more 1120 * information. 1121 */ 1122 static const struct err_reg_info sdma_eng_err = 1123 EE(SEND_DMA_ENG_ERR, handle_sdma_eng_err, "SDmaEngErr"); 1124 1125 static const struct err_reg_info various_err[NUM_VARIOUS] = { 1126 /* 0*/ { 0, 0, 0, NULL }, /* PbcInt */ 1127 /* 1*/ { 0, 0, 0, NULL }, /* GpioAssertInt */ 1128 /* 2*/ EE(ASIC_QSFP1, handle_qsfp_int, "QSFP1"), 1129 /* 3*/ EE(ASIC_QSFP2, handle_qsfp_int, "QSFP2"), 1130 /* 4*/ { 0, 0, 0, NULL }, /* TCritInt */ 1131 /* rest are reserved */ 1132 }; 1133 1134 /* 1135 * The DC encoding of mtu_cap for 10K MTU in the DCC_CFG_PORT_CONFIG 1136 * register can not be derived from the MTU value because 10K is not 1137 * a power of 2. Therefore, we need a constant. Everything else can 1138 * be calculated. 1139 */ 1140 #define DCC_CFG_PORT_MTU_CAP_10240 7 1141 1142 /* 1143 * Table of the DC grouping of error interrupts. Each entry refers to 1144 * another register containing more information. 1145 */ 1146 static const struct err_reg_info dc_errs[NUM_DC_ERRS] = { 1147 /* 0*/ DC_EE1(DCC_ERR, handle_dcc_err, "DCC Err"), 1148 /* 1*/ DC_EE2(DC_LCB_ERR, handle_lcb_err, "LCB Err"), 1149 /* 2*/ DC_EE2(DC_DC8051_ERR, handle_8051_interrupt, "DC8051 Interrupt"), 1150 /* 3*/ /* dc_lbm_int - special, see is_dc_int() */ 1151 /* the rest are reserved */ 1152 }; 1153 1154 struct cntr_entry { 1155 /* 1156 * counter name 1157 */ 1158 char *name; 1159 1160 /* 1161 * csr to read for name (if applicable) 1162 */ 1163 u64 csr; 1164 1165 /* 1166 * offset into dd or ppd to store the counter's value 1167 */ 1168 int offset; 1169 1170 /* 1171 * flags 1172 */ 1173 u8 flags; 1174 1175 /* 1176 * accessor for stat element, context either dd or ppd 1177 */ 1178 u64 (*rw_cntr)(const struct cntr_entry *, void *context, int vl, 1179 int mode, u64 data); 1180 }; 1181 1182 #define C_RCV_HDR_OVF_FIRST C_RCV_HDR_OVF_0 1183 #define C_RCV_HDR_OVF_LAST C_RCV_HDR_OVF_159 1184 1185 #define CNTR_ELEM(name, csr, offset, flags, accessor) \ 1186 { \ 1187 name, \ 1188 csr, \ 1189 offset, \ 1190 flags, \ 1191 accessor \ 1192 } 1193 1194 /* 32bit RXE */ 1195 #define RXE32_PORT_CNTR_ELEM(name, counter, flags) \ 1196 CNTR_ELEM(#name, \ 1197 (counter * 8 + RCV_COUNTER_ARRAY32), \ 1198 0, flags | CNTR_32BIT, \ 1199 port_access_u32_csr) 1200 1201 #define RXE32_DEV_CNTR_ELEM(name, counter, flags) \ 1202 CNTR_ELEM(#name, \ 1203 (counter * 8 + RCV_COUNTER_ARRAY32), \ 1204 0, flags | CNTR_32BIT, \ 1205 dev_access_u32_csr) 1206 1207 /* 64bit RXE */ 1208 #define RXE64_PORT_CNTR_ELEM(name, counter, flags) \ 1209 CNTR_ELEM(#name, \ 1210 (counter * 8 + RCV_COUNTER_ARRAY64), \ 1211 0, flags, \ 1212 port_access_u64_csr) 1213 1214 #define RXE64_DEV_CNTR_ELEM(name, counter, flags) \ 1215 CNTR_ELEM(#name, \ 1216 (counter * 8 + RCV_COUNTER_ARRAY64), \ 1217 0, flags, \ 1218 dev_access_u64_csr) 1219 1220 #define OVR_LBL(ctx) C_RCV_HDR_OVF_ ## ctx 1221 #define OVR_ELM(ctx) \ 1222 CNTR_ELEM("RcvHdrOvr" #ctx, \ 1223 (RCV_HDR_OVFL_CNT + ctx * 0x100), \ 1224 0, CNTR_NORMAL, port_access_u64_csr) 1225 1226 /* 32bit TXE */ 1227 #define TXE32_PORT_CNTR_ELEM(name, counter, flags) \ 1228 CNTR_ELEM(#name, \ 1229 (counter * 8 + SEND_COUNTER_ARRAY32), \ 1230 0, flags | CNTR_32BIT, \ 1231 port_access_u32_csr) 1232 1233 /* 64bit TXE */ 1234 #define TXE64_PORT_CNTR_ELEM(name, counter, flags) \ 1235 CNTR_ELEM(#name, \ 1236 (counter * 8 + SEND_COUNTER_ARRAY64), \ 1237 0, flags, \ 1238 port_access_u64_csr) 1239 1240 # define TX64_DEV_CNTR_ELEM(name, counter, flags) \ 1241 CNTR_ELEM(#name,\ 1242 counter * 8 + SEND_COUNTER_ARRAY64, \ 1243 0, \ 1244 flags, \ 1245 dev_access_u64_csr) 1246 1247 /* CCE */ 1248 #define CCE_PERF_DEV_CNTR_ELEM(name, counter, flags) \ 1249 CNTR_ELEM(#name, \ 1250 (counter * 8 + CCE_COUNTER_ARRAY32), \ 1251 0, flags | CNTR_32BIT, \ 1252 dev_access_u32_csr) 1253 1254 #define CCE_INT_DEV_CNTR_ELEM(name, counter, flags) \ 1255 CNTR_ELEM(#name, \ 1256 (counter * 8 + CCE_INT_COUNTER_ARRAY32), \ 1257 0, flags | CNTR_32BIT, \ 1258 dev_access_u32_csr) 1259 1260 /* DC */ 1261 #define DC_PERF_CNTR(name, counter, flags) \ 1262 CNTR_ELEM(#name, \ 1263 counter, \ 1264 0, \ 1265 flags, \ 1266 dev_access_u64_csr) 1267 1268 #define DC_PERF_CNTR_LCB(name, counter, flags) \ 1269 CNTR_ELEM(#name, \ 1270 counter, \ 1271 0, \ 1272 flags, \ 1273 dc_access_lcb_cntr) 1274 1275 /* ibp counters */ 1276 #define SW_IBP_CNTR(name, cntr) \ 1277 CNTR_ELEM(#name, \ 1278 0, \ 1279 0, \ 1280 CNTR_SYNTH, \ 1281 access_ibp_##cntr) 1282 1283 /** 1284 * hfi1_addr_from_offset - return addr for readq/writeq 1285 * @dd: the dd device 1286 * @offset: the offset of the CSR within bar0 1287 * 1288 * This routine selects the appropriate base address 1289 * based on the indicated offset. 1290 */ 1291 static inline void __iomem *hfi1_addr_from_offset( 1292 const struct hfi1_devdata *dd, 1293 u32 offset) 1294 { 1295 if (offset >= dd->base2_start) 1296 return dd->kregbase2 + (offset - dd->base2_start); 1297 return dd->kregbase1 + offset; 1298 } 1299 1300 /** 1301 * read_csr - read CSR at the indicated offset 1302 * @dd: the dd device 1303 * @offset: the offset of the CSR within bar0 1304 * 1305 * Return: the value read or all FF's if there 1306 * is no mapping 1307 */ 1308 u64 read_csr(const struct hfi1_devdata *dd, u32 offset) 1309 { 1310 if (dd->flags & HFI1_PRESENT) 1311 return readq(hfi1_addr_from_offset(dd, offset)); 1312 return -1; 1313 } 1314 1315 /** 1316 * write_csr - write CSR at the indicated offset 1317 * @dd: the dd device 1318 * @offset: the offset of the CSR within bar0 1319 * @value: value to write 1320 */ 1321 void write_csr(const struct hfi1_devdata *dd, u32 offset, u64 value) 1322 { 1323 if (dd->flags & HFI1_PRESENT) { 1324 void __iomem *base = hfi1_addr_from_offset(dd, offset); 1325 1326 /* avoid write to RcvArray */ 1327 if (WARN_ON(offset >= RCV_ARRAY && offset < dd->base2_start)) 1328 return; 1329 writeq(value, base); 1330 } 1331 } 1332 1333 /** 1334 * get_csr_addr - return te iomem address for offset 1335 * @dd: the dd device 1336 * @offset: the offset of the CSR within bar0 1337 * 1338 * Return: The iomem address to use in subsequent 1339 * writeq/readq operations. 1340 */ 1341 void __iomem *get_csr_addr( 1342 const struct hfi1_devdata *dd, 1343 u32 offset) 1344 { 1345 if (dd->flags & HFI1_PRESENT) 1346 return hfi1_addr_from_offset(dd, offset); 1347 return NULL; 1348 } 1349 1350 static inline u64 read_write_csr(const struct hfi1_devdata *dd, u32 csr, 1351 int mode, u64 value) 1352 { 1353 u64 ret; 1354 1355 if (mode == CNTR_MODE_R) { 1356 ret = read_csr(dd, csr); 1357 } else if (mode == CNTR_MODE_W) { 1358 write_csr(dd, csr, value); 1359 ret = value; 1360 } else { 1361 dd_dev_err(dd, "Invalid cntr register access mode"); 1362 return 0; 1363 } 1364 1365 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, ret, mode); 1366 return ret; 1367 } 1368 1369 /* Dev Access */ 1370 static u64 dev_access_u32_csr(const struct cntr_entry *entry, 1371 void *context, int vl, int mode, u64 data) 1372 { 1373 struct hfi1_devdata *dd = context; 1374 u64 csr = entry->csr; 1375 1376 if (entry->flags & CNTR_SDMA) { 1377 if (vl == CNTR_INVALID_VL) 1378 return 0; 1379 csr += 0x100 * vl; 1380 } else { 1381 if (vl != CNTR_INVALID_VL) 1382 return 0; 1383 } 1384 return read_write_csr(dd, csr, mode, data); 1385 } 1386 1387 static u64 access_sde_err_cnt(const struct cntr_entry *entry, 1388 void *context, int idx, int mode, u64 data) 1389 { 1390 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1391 1392 if (dd->per_sdma && idx < dd->num_sdma) 1393 return dd->per_sdma[idx].err_cnt; 1394 return 0; 1395 } 1396 1397 static u64 access_sde_int_cnt(const struct cntr_entry *entry, 1398 void *context, int idx, int mode, u64 data) 1399 { 1400 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1401 1402 if (dd->per_sdma && idx < dd->num_sdma) 1403 return dd->per_sdma[idx].sdma_int_cnt; 1404 return 0; 1405 } 1406 1407 static u64 access_sde_idle_int_cnt(const struct cntr_entry *entry, 1408 void *context, int idx, int mode, u64 data) 1409 { 1410 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1411 1412 if (dd->per_sdma && idx < dd->num_sdma) 1413 return dd->per_sdma[idx].idle_int_cnt; 1414 return 0; 1415 } 1416 1417 static u64 access_sde_progress_int_cnt(const struct cntr_entry *entry, 1418 void *context, int idx, int mode, 1419 u64 data) 1420 { 1421 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1422 1423 if (dd->per_sdma && idx < dd->num_sdma) 1424 return dd->per_sdma[idx].progress_int_cnt; 1425 return 0; 1426 } 1427 1428 static u64 dev_access_u64_csr(const struct cntr_entry *entry, void *context, 1429 int vl, int mode, u64 data) 1430 { 1431 struct hfi1_devdata *dd = context; 1432 1433 u64 val = 0; 1434 u64 csr = entry->csr; 1435 1436 if (entry->flags & CNTR_VL) { 1437 if (vl == CNTR_INVALID_VL) 1438 return 0; 1439 csr += 8 * vl; 1440 } else { 1441 if (vl != CNTR_INVALID_VL) 1442 return 0; 1443 } 1444 1445 val = read_write_csr(dd, csr, mode, data); 1446 return val; 1447 } 1448 1449 static u64 dc_access_lcb_cntr(const struct cntr_entry *entry, void *context, 1450 int vl, int mode, u64 data) 1451 { 1452 struct hfi1_devdata *dd = context; 1453 u32 csr = entry->csr; 1454 int ret = 0; 1455 1456 if (vl != CNTR_INVALID_VL) 1457 return 0; 1458 if (mode == CNTR_MODE_R) 1459 ret = read_lcb_csr(dd, csr, &data); 1460 else if (mode == CNTR_MODE_W) 1461 ret = write_lcb_csr(dd, csr, data); 1462 1463 if (ret) { 1464 if (!(dd->flags & HFI1_SHUTDOWN)) 1465 dd_dev_err(dd, "Could not acquire LCB for counter 0x%x", csr); 1466 return 0; 1467 } 1468 1469 hfi1_cdbg(CNTR, "csr 0x%x val 0x%llx mode %d", csr, data, mode); 1470 return data; 1471 } 1472 1473 /* Port Access */ 1474 static u64 port_access_u32_csr(const struct cntr_entry *entry, void *context, 1475 int vl, int mode, u64 data) 1476 { 1477 struct hfi1_pportdata *ppd = context; 1478 1479 if (vl != CNTR_INVALID_VL) 1480 return 0; 1481 return read_write_csr(ppd->dd, entry->csr, mode, data); 1482 } 1483 1484 static u64 port_access_u64_csr(const struct cntr_entry *entry, 1485 void *context, int vl, int mode, u64 data) 1486 { 1487 struct hfi1_pportdata *ppd = context; 1488 u64 val; 1489 u64 csr = entry->csr; 1490 1491 if (entry->flags & CNTR_VL) { 1492 if (vl == CNTR_INVALID_VL) 1493 return 0; 1494 csr += 8 * vl; 1495 } else { 1496 if (vl != CNTR_INVALID_VL) 1497 return 0; 1498 } 1499 val = read_write_csr(ppd->dd, csr, mode, data); 1500 return val; 1501 } 1502 1503 /* Software defined */ 1504 static inline u64 read_write_sw(struct hfi1_devdata *dd, u64 *cntr, int mode, 1505 u64 data) 1506 { 1507 u64 ret; 1508 1509 if (mode == CNTR_MODE_R) { 1510 ret = *cntr; 1511 } else if (mode == CNTR_MODE_W) { 1512 *cntr = data; 1513 ret = data; 1514 } else { 1515 dd_dev_err(dd, "Invalid cntr sw access mode"); 1516 return 0; 1517 } 1518 1519 hfi1_cdbg(CNTR, "val 0x%llx mode %d", ret, mode); 1520 1521 return ret; 1522 } 1523 1524 static u64 access_sw_link_dn_cnt(const struct cntr_entry *entry, void *context, 1525 int vl, int mode, u64 data) 1526 { 1527 struct hfi1_pportdata *ppd = context; 1528 1529 if (vl != CNTR_INVALID_VL) 1530 return 0; 1531 return read_write_sw(ppd->dd, &ppd->link_downed, mode, data); 1532 } 1533 1534 static u64 access_sw_link_up_cnt(const struct cntr_entry *entry, void *context, 1535 int vl, int mode, u64 data) 1536 { 1537 struct hfi1_pportdata *ppd = context; 1538 1539 if (vl != CNTR_INVALID_VL) 1540 return 0; 1541 return read_write_sw(ppd->dd, &ppd->link_up, mode, data); 1542 } 1543 1544 static u64 access_sw_unknown_frame_cnt(const struct cntr_entry *entry, 1545 void *context, int vl, int mode, 1546 u64 data) 1547 { 1548 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; 1549 1550 if (vl != CNTR_INVALID_VL) 1551 return 0; 1552 return read_write_sw(ppd->dd, &ppd->unknown_frame_count, mode, data); 1553 } 1554 1555 static u64 access_sw_xmit_discards(const struct cntr_entry *entry, 1556 void *context, int vl, int mode, u64 data) 1557 { 1558 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; 1559 u64 zero = 0; 1560 u64 *counter; 1561 1562 if (vl == CNTR_INVALID_VL) 1563 counter = &ppd->port_xmit_discards; 1564 else if (vl >= 0 && vl < C_VL_COUNT) 1565 counter = &ppd->port_xmit_discards_vl[vl]; 1566 else 1567 counter = &zero; 1568 1569 return read_write_sw(ppd->dd, counter, mode, data); 1570 } 1571 1572 static u64 access_xmit_constraint_errs(const struct cntr_entry *entry, 1573 void *context, int vl, int mode, 1574 u64 data) 1575 { 1576 struct hfi1_pportdata *ppd = context; 1577 1578 if (vl != CNTR_INVALID_VL) 1579 return 0; 1580 1581 return read_write_sw(ppd->dd, &ppd->port_xmit_constraint_errors, 1582 mode, data); 1583 } 1584 1585 static u64 access_rcv_constraint_errs(const struct cntr_entry *entry, 1586 void *context, int vl, int mode, u64 data) 1587 { 1588 struct hfi1_pportdata *ppd = context; 1589 1590 if (vl != CNTR_INVALID_VL) 1591 return 0; 1592 1593 return read_write_sw(ppd->dd, &ppd->port_rcv_constraint_errors, 1594 mode, data); 1595 } 1596 1597 u64 get_all_cpu_total(u64 __percpu *cntr) 1598 { 1599 int cpu; 1600 u64 counter = 0; 1601 1602 for_each_possible_cpu(cpu) 1603 counter += *per_cpu_ptr(cntr, cpu); 1604 return counter; 1605 } 1606 1607 static u64 read_write_cpu(struct hfi1_devdata *dd, u64 *z_val, 1608 u64 __percpu *cntr, 1609 int vl, int mode, u64 data) 1610 { 1611 u64 ret = 0; 1612 1613 if (vl != CNTR_INVALID_VL) 1614 return 0; 1615 1616 if (mode == CNTR_MODE_R) { 1617 ret = get_all_cpu_total(cntr) - *z_val; 1618 } else if (mode == CNTR_MODE_W) { 1619 /* A write can only zero the counter */ 1620 if (data == 0) 1621 *z_val = get_all_cpu_total(cntr); 1622 else 1623 dd_dev_err(dd, "Per CPU cntrs can only be zeroed"); 1624 } else { 1625 dd_dev_err(dd, "Invalid cntr sw cpu access mode"); 1626 return 0; 1627 } 1628 1629 return ret; 1630 } 1631 1632 static u64 access_sw_cpu_intr(const struct cntr_entry *entry, 1633 void *context, int vl, int mode, u64 data) 1634 { 1635 struct hfi1_devdata *dd = context; 1636 1637 return read_write_cpu(dd, &dd->z_int_counter, dd->int_counter, vl, 1638 mode, data); 1639 } 1640 1641 static u64 access_sw_cpu_rcv_limit(const struct cntr_entry *entry, 1642 void *context, int vl, int mode, u64 data) 1643 { 1644 struct hfi1_devdata *dd = context; 1645 1646 return read_write_cpu(dd, &dd->z_rcv_limit, dd->rcv_limit, vl, 1647 mode, data); 1648 } 1649 1650 static u64 access_sw_pio_wait(const struct cntr_entry *entry, 1651 void *context, int vl, int mode, u64 data) 1652 { 1653 struct hfi1_devdata *dd = context; 1654 1655 return dd->verbs_dev.n_piowait; 1656 } 1657 1658 static u64 access_sw_pio_drain(const struct cntr_entry *entry, 1659 void *context, int vl, int mode, u64 data) 1660 { 1661 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1662 1663 return dd->verbs_dev.n_piodrain; 1664 } 1665 1666 static u64 access_sw_ctx0_seq_drop(const struct cntr_entry *entry, 1667 void *context, int vl, int mode, u64 data) 1668 { 1669 struct hfi1_devdata *dd = context; 1670 1671 return dd->ctx0_seq_drop; 1672 } 1673 1674 static u64 access_sw_vtx_wait(const struct cntr_entry *entry, 1675 void *context, int vl, int mode, u64 data) 1676 { 1677 struct hfi1_devdata *dd = context; 1678 1679 return dd->verbs_dev.n_txwait; 1680 } 1681 1682 static u64 access_sw_kmem_wait(const struct cntr_entry *entry, 1683 void *context, int vl, int mode, u64 data) 1684 { 1685 struct hfi1_devdata *dd = context; 1686 1687 return dd->verbs_dev.n_kmem_wait; 1688 } 1689 1690 static u64 access_sw_send_schedule(const struct cntr_entry *entry, 1691 void *context, int vl, int mode, u64 data) 1692 { 1693 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1694 1695 return read_write_cpu(dd, &dd->z_send_schedule, dd->send_schedule, vl, 1696 mode, data); 1697 } 1698 1699 /* Software counters for the error status bits within MISC_ERR_STATUS */ 1700 static u64 access_misc_pll_lock_fail_err_cnt(const struct cntr_entry *entry, 1701 void *context, int vl, int mode, 1702 u64 data) 1703 { 1704 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1705 1706 return dd->misc_err_status_cnt[12]; 1707 } 1708 1709 static u64 access_misc_mbist_fail_err_cnt(const struct cntr_entry *entry, 1710 void *context, int vl, int mode, 1711 u64 data) 1712 { 1713 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1714 1715 return dd->misc_err_status_cnt[11]; 1716 } 1717 1718 static u64 access_misc_invalid_eep_cmd_err_cnt(const struct cntr_entry *entry, 1719 void *context, int vl, int mode, 1720 u64 data) 1721 { 1722 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1723 1724 return dd->misc_err_status_cnt[10]; 1725 } 1726 1727 static u64 access_misc_efuse_done_parity_err_cnt(const struct cntr_entry *entry, 1728 void *context, int vl, 1729 int mode, u64 data) 1730 { 1731 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1732 1733 return dd->misc_err_status_cnt[9]; 1734 } 1735 1736 static u64 access_misc_efuse_write_err_cnt(const struct cntr_entry *entry, 1737 void *context, int vl, int mode, 1738 u64 data) 1739 { 1740 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1741 1742 return dd->misc_err_status_cnt[8]; 1743 } 1744 1745 static u64 access_misc_efuse_read_bad_addr_err_cnt( 1746 const struct cntr_entry *entry, 1747 void *context, int vl, int mode, u64 data) 1748 { 1749 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1750 1751 return dd->misc_err_status_cnt[7]; 1752 } 1753 1754 static u64 access_misc_efuse_csr_parity_err_cnt(const struct cntr_entry *entry, 1755 void *context, int vl, 1756 int mode, u64 data) 1757 { 1758 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1759 1760 return dd->misc_err_status_cnt[6]; 1761 } 1762 1763 static u64 access_misc_fw_auth_failed_err_cnt(const struct cntr_entry *entry, 1764 void *context, int vl, int mode, 1765 u64 data) 1766 { 1767 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1768 1769 return dd->misc_err_status_cnt[5]; 1770 } 1771 1772 static u64 access_misc_key_mismatch_err_cnt(const struct cntr_entry *entry, 1773 void *context, int vl, int mode, 1774 u64 data) 1775 { 1776 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1777 1778 return dd->misc_err_status_cnt[4]; 1779 } 1780 1781 static u64 access_misc_sbus_write_failed_err_cnt(const struct cntr_entry *entry, 1782 void *context, int vl, 1783 int mode, u64 data) 1784 { 1785 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1786 1787 return dd->misc_err_status_cnt[3]; 1788 } 1789 1790 static u64 access_misc_csr_write_bad_addr_err_cnt( 1791 const struct cntr_entry *entry, 1792 void *context, int vl, int mode, u64 data) 1793 { 1794 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1795 1796 return dd->misc_err_status_cnt[2]; 1797 } 1798 1799 static u64 access_misc_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, 1800 void *context, int vl, 1801 int mode, u64 data) 1802 { 1803 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1804 1805 return dd->misc_err_status_cnt[1]; 1806 } 1807 1808 static u64 access_misc_csr_parity_err_cnt(const struct cntr_entry *entry, 1809 void *context, int vl, int mode, 1810 u64 data) 1811 { 1812 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1813 1814 return dd->misc_err_status_cnt[0]; 1815 } 1816 1817 /* 1818 * Software counter for the aggregate of 1819 * individual CceErrStatus counters 1820 */ 1821 static u64 access_sw_cce_err_status_aggregated_cnt( 1822 const struct cntr_entry *entry, 1823 void *context, int vl, int mode, u64 data) 1824 { 1825 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1826 1827 return dd->sw_cce_err_status_aggregate; 1828 } 1829 1830 /* 1831 * Software counters corresponding to each of the 1832 * error status bits within CceErrStatus 1833 */ 1834 static u64 access_cce_msix_csr_parity_err_cnt(const struct cntr_entry *entry, 1835 void *context, int vl, int mode, 1836 u64 data) 1837 { 1838 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1839 1840 return dd->cce_err_status_cnt[40]; 1841 } 1842 1843 static u64 access_cce_int_map_unc_err_cnt(const struct cntr_entry *entry, 1844 void *context, int vl, int mode, 1845 u64 data) 1846 { 1847 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1848 1849 return dd->cce_err_status_cnt[39]; 1850 } 1851 1852 static u64 access_cce_int_map_cor_err_cnt(const struct cntr_entry *entry, 1853 void *context, int vl, int mode, 1854 u64 data) 1855 { 1856 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1857 1858 return dd->cce_err_status_cnt[38]; 1859 } 1860 1861 static u64 access_cce_msix_table_unc_err_cnt(const struct cntr_entry *entry, 1862 void *context, int vl, int mode, 1863 u64 data) 1864 { 1865 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1866 1867 return dd->cce_err_status_cnt[37]; 1868 } 1869 1870 static u64 access_cce_msix_table_cor_err_cnt(const struct cntr_entry *entry, 1871 void *context, int vl, int mode, 1872 u64 data) 1873 { 1874 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1875 1876 return dd->cce_err_status_cnt[36]; 1877 } 1878 1879 static u64 access_cce_rxdma_conv_fifo_parity_err_cnt( 1880 const struct cntr_entry *entry, 1881 void *context, int vl, int mode, u64 data) 1882 { 1883 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1884 1885 return dd->cce_err_status_cnt[35]; 1886 } 1887 1888 static u64 access_cce_rcpl_async_fifo_parity_err_cnt( 1889 const struct cntr_entry *entry, 1890 void *context, int vl, int mode, u64 data) 1891 { 1892 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1893 1894 return dd->cce_err_status_cnt[34]; 1895 } 1896 1897 static u64 access_cce_seg_write_bad_addr_err_cnt(const struct cntr_entry *entry, 1898 void *context, int vl, 1899 int mode, u64 data) 1900 { 1901 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1902 1903 return dd->cce_err_status_cnt[33]; 1904 } 1905 1906 static u64 access_cce_seg_read_bad_addr_err_cnt(const struct cntr_entry *entry, 1907 void *context, int vl, int mode, 1908 u64 data) 1909 { 1910 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1911 1912 return dd->cce_err_status_cnt[32]; 1913 } 1914 1915 static u64 access_la_triggered_cnt(const struct cntr_entry *entry, 1916 void *context, int vl, int mode, u64 data) 1917 { 1918 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1919 1920 return dd->cce_err_status_cnt[31]; 1921 } 1922 1923 static u64 access_cce_trgt_cpl_timeout_err_cnt(const struct cntr_entry *entry, 1924 void *context, int vl, int mode, 1925 u64 data) 1926 { 1927 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1928 1929 return dd->cce_err_status_cnt[30]; 1930 } 1931 1932 static u64 access_pcic_receive_parity_err_cnt(const struct cntr_entry *entry, 1933 void *context, int vl, int mode, 1934 u64 data) 1935 { 1936 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1937 1938 return dd->cce_err_status_cnt[29]; 1939 } 1940 1941 static u64 access_pcic_transmit_back_parity_err_cnt( 1942 const struct cntr_entry *entry, 1943 void *context, int vl, int mode, u64 data) 1944 { 1945 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1946 1947 return dd->cce_err_status_cnt[28]; 1948 } 1949 1950 static u64 access_pcic_transmit_front_parity_err_cnt( 1951 const struct cntr_entry *entry, 1952 void *context, int vl, int mode, u64 data) 1953 { 1954 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1955 1956 return dd->cce_err_status_cnt[27]; 1957 } 1958 1959 static u64 access_pcic_cpl_dat_q_unc_err_cnt(const struct cntr_entry *entry, 1960 void *context, int vl, int mode, 1961 u64 data) 1962 { 1963 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1964 1965 return dd->cce_err_status_cnt[26]; 1966 } 1967 1968 static u64 access_pcic_cpl_hd_q_unc_err_cnt(const struct cntr_entry *entry, 1969 void *context, int vl, int mode, 1970 u64 data) 1971 { 1972 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1973 1974 return dd->cce_err_status_cnt[25]; 1975 } 1976 1977 static u64 access_pcic_post_dat_q_unc_err_cnt(const struct cntr_entry *entry, 1978 void *context, int vl, int mode, 1979 u64 data) 1980 { 1981 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1982 1983 return dd->cce_err_status_cnt[24]; 1984 } 1985 1986 static u64 access_pcic_post_hd_q_unc_err_cnt(const struct cntr_entry *entry, 1987 void *context, int vl, int mode, 1988 u64 data) 1989 { 1990 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 1991 1992 return dd->cce_err_status_cnt[23]; 1993 } 1994 1995 static u64 access_pcic_retry_sot_mem_unc_err_cnt(const struct cntr_entry *entry, 1996 void *context, int vl, 1997 int mode, u64 data) 1998 { 1999 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2000 2001 return dd->cce_err_status_cnt[22]; 2002 } 2003 2004 static u64 access_pcic_retry_mem_unc_err(const struct cntr_entry *entry, 2005 void *context, int vl, int mode, 2006 u64 data) 2007 { 2008 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2009 2010 return dd->cce_err_status_cnt[21]; 2011 } 2012 2013 static u64 access_pcic_n_post_dat_q_parity_err_cnt( 2014 const struct cntr_entry *entry, 2015 void *context, int vl, int mode, u64 data) 2016 { 2017 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2018 2019 return dd->cce_err_status_cnt[20]; 2020 } 2021 2022 static u64 access_pcic_n_post_h_q_parity_err_cnt(const struct cntr_entry *entry, 2023 void *context, int vl, 2024 int mode, u64 data) 2025 { 2026 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2027 2028 return dd->cce_err_status_cnt[19]; 2029 } 2030 2031 static u64 access_pcic_cpl_dat_q_cor_err_cnt(const struct cntr_entry *entry, 2032 void *context, int vl, int mode, 2033 u64 data) 2034 { 2035 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2036 2037 return dd->cce_err_status_cnt[18]; 2038 } 2039 2040 static u64 access_pcic_cpl_hd_q_cor_err_cnt(const struct cntr_entry *entry, 2041 void *context, int vl, int mode, 2042 u64 data) 2043 { 2044 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2045 2046 return dd->cce_err_status_cnt[17]; 2047 } 2048 2049 static u64 access_pcic_post_dat_q_cor_err_cnt(const struct cntr_entry *entry, 2050 void *context, int vl, int mode, 2051 u64 data) 2052 { 2053 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2054 2055 return dd->cce_err_status_cnt[16]; 2056 } 2057 2058 static u64 access_pcic_post_hd_q_cor_err_cnt(const struct cntr_entry *entry, 2059 void *context, int vl, int mode, 2060 u64 data) 2061 { 2062 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2063 2064 return dd->cce_err_status_cnt[15]; 2065 } 2066 2067 static u64 access_pcic_retry_sot_mem_cor_err_cnt(const struct cntr_entry *entry, 2068 void *context, int vl, 2069 int mode, u64 data) 2070 { 2071 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2072 2073 return dd->cce_err_status_cnt[14]; 2074 } 2075 2076 static u64 access_pcic_retry_mem_cor_err_cnt(const struct cntr_entry *entry, 2077 void *context, int vl, int mode, 2078 u64 data) 2079 { 2080 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2081 2082 return dd->cce_err_status_cnt[13]; 2083 } 2084 2085 static u64 access_cce_cli1_async_fifo_dbg_parity_err_cnt( 2086 const struct cntr_entry *entry, 2087 void *context, int vl, int mode, u64 data) 2088 { 2089 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2090 2091 return dd->cce_err_status_cnt[12]; 2092 } 2093 2094 static u64 access_cce_cli1_async_fifo_rxdma_parity_err_cnt( 2095 const struct cntr_entry *entry, 2096 void *context, int vl, int mode, u64 data) 2097 { 2098 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2099 2100 return dd->cce_err_status_cnt[11]; 2101 } 2102 2103 static u64 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt( 2104 const struct cntr_entry *entry, 2105 void *context, int vl, int mode, u64 data) 2106 { 2107 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2108 2109 return dd->cce_err_status_cnt[10]; 2110 } 2111 2112 static u64 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt( 2113 const struct cntr_entry *entry, 2114 void *context, int vl, int mode, u64 data) 2115 { 2116 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2117 2118 return dd->cce_err_status_cnt[9]; 2119 } 2120 2121 static u64 access_cce_cli2_async_fifo_parity_err_cnt( 2122 const struct cntr_entry *entry, 2123 void *context, int vl, int mode, u64 data) 2124 { 2125 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2126 2127 return dd->cce_err_status_cnt[8]; 2128 } 2129 2130 static u64 access_cce_csr_cfg_bus_parity_err_cnt(const struct cntr_entry *entry, 2131 void *context, int vl, 2132 int mode, u64 data) 2133 { 2134 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2135 2136 return dd->cce_err_status_cnt[7]; 2137 } 2138 2139 static u64 access_cce_cli0_async_fifo_parity_err_cnt( 2140 const struct cntr_entry *entry, 2141 void *context, int vl, int mode, u64 data) 2142 { 2143 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2144 2145 return dd->cce_err_status_cnt[6]; 2146 } 2147 2148 static u64 access_cce_rspd_data_parity_err_cnt(const struct cntr_entry *entry, 2149 void *context, int vl, int mode, 2150 u64 data) 2151 { 2152 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2153 2154 return dd->cce_err_status_cnt[5]; 2155 } 2156 2157 static u64 access_cce_trgt_access_err_cnt(const struct cntr_entry *entry, 2158 void *context, int vl, int mode, 2159 u64 data) 2160 { 2161 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2162 2163 return dd->cce_err_status_cnt[4]; 2164 } 2165 2166 static u64 access_cce_trgt_async_fifo_parity_err_cnt( 2167 const struct cntr_entry *entry, 2168 void *context, int vl, int mode, u64 data) 2169 { 2170 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2171 2172 return dd->cce_err_status_cnt[3]; 2173 } 2174 2175 static u64 access_cce_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry, 2176 void *context, int vl, 2177 int mode, u64 data) 2178 { 2179 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2180 2181 return dd->cce_err_status_cnt[2]; 2182 } 2183 2184 static u64 access_cce_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, 2185 void *context, int vl, 2186 int mode, u64 data) 2187 { 2188 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2189 2190 return dd->cce_err_status_cnt[1]; 2191 } 2192 2193 static u64 access_ccs_csr_parity_err_cnt(const struct cntr_entry *entry, 2194 void *context, int vl, int mode, 2195 u64 data) 2196 { 2197 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2198 2199 return dd->cce_err_status_cnt[0]; 2200 } 2201 2202 /* 2203 * Software counters corresponding to each of the 2204 * error status bits within RcvErrStatus 2205 */ 2206 static u64 access_rx_csr_parity_err_cnt(const struct cntr_entry *entry, 2207 void *context, int vl, int mode, 2208 u64 data) 2209 { 2210 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2211 2212 return dd->rcv_err_status_cnt[63]; 2213 } 2214 2215 static u64 access_rx_csr_write_bad_addr_err_cnt(const struct cntr_entry *entry, 2216 void *context, int vl, 2217 int mode, u64 data) 2218 { 2219 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2220 2221 return dd->rcv_err_status_cnt[62]; 2222 } 2223 2224 static u64 access_rx_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, 2225 void *context, int vl, int mode, 2226 u64 data) 2227 { 2228 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2229 2230 return dd->rcv_err_status_cnt[61]; 2231 } 2232 2233 static u64 access_rx_dma_csr_unc_err_cnt(const struct cntr_entry *entry, 2234 void *context, int vl, int mode, 2235 u64 data) 2236 { 2237 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2238 2239 return dd->rcv_err_status_cnt[60]; 2240 } 2241 2242 static u64 access_rx_dma_dq_fsm_encoding_err_cnt(const struct cntr_entry *entry, 2243 void *context, int vl, 2244 int mode, u64 data) 2245 { 2246 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2247 2248 return dd->rcv_err_status_cnt[59]; 2249 } 2250 2251 static u64 access_rx_dma_eq_fsm_encoding_err_cnt(const struct cntr_entry *entry, 2252 void *context, int vl, 2253 int mode, u64 data) 2254 { 2255 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2256 2257 return dd->rcv_err_status_cnt[58]; 2258 } 2259 2260 static u64 access_rx_dma_csr_parity_err_cnt(const struct cntr_entry *entry, 2261 void *context, int vl, int mode, 2262 u64 data) 2263 { 2264 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2265 2266 return dd->rcv_err_status_cnt[57]; 2267 } 2268 2269 static u64 access_rx_rbuf_data_cor_err_cnt(const struct cntr_entry *entry, 2270 void *context, int vl, int mode, 2271 u64 data) 2272 { 2273 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2274 2275 return dd->rcv_err_status_cnt[56]; 2276 } 2277 2278 static u64 access_rx_rbuf_data_unc_err_cnt(const struct cntr_entry *entry, 2279 void *context, int vl, int mode, 2280 u64 data) 2281 { 2282 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2283 2284 return dd->rcv_err_status_cnt[55]; 2285 } 2286 2287 static u64 access_rx_dma_data_fifo_rd_cor_err_cnt( 2288 const struct cntr_entry *entry, 2289 void *context, int vl, int mode, u64 data) 2290 { 2291 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2292 2293 return dd->rcv_err_status_cnt[54]; 2294 } 2295 2296 static u64 access_rx_dma_data_fifo_rd_unc_err_cnt( 2297 const struct cntr_entry *entry, 2298 void *context, int vl, int mode, u64 data) 2299 { 2300 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2301 2302 return dd->rcv_err_status_cnt[53]; 2303 } 2304 2305 static u64 access_rx_dma_hdr_fifo_rd_cor_err_cnt(const struct cntr_entry *entry, 2306 void *context, int vl, 2307 int mode, u64 data) 2308 { 2309 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2310 2311 return dd->rcv_err_status_cnt[52]; 2312 } 2313 2314 static u64 access_rx_dma_hdr_fifo_rd_unc_err_cnt(const struct cntr_entry *entry, 2315 void *context, int vl, 2316 int mode, u64 data) 2317 { 2318 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2319 2320 return dd->rcv_err_status_cnt[51]; 2321 } 2322 2323 static u64 access_rx_rbuf_desc_part2_cor_err_cnt(const struct cntr_entry *entry, 2324 void *context, int vl, 2325 int mode, u64 data) 2326 { 2327 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2328 2329 return dd->rcv_err_status_cnt[50]; 2330 } 2331 2332 static u64 access_rx_rbuf_desc_part2_unc_err_cnt(const struct cntr_entry *entry, 2333 void *context, int vl, 2334 int mode, u64 data) 2335 { 2336 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2337 2338 return dd->rcv_err_status_cnt[49]; 2339 } 2340 2341 static u64 access_rx_rbuf_desc_part1_cor_err_cnt(const struct cntr_entry *entry, 2342 void *context, int vl, 2343 int mode, u64 data) 2344 { 2345 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2346 2347 return dd->rcv_err_status_cnt[48]; 2348 } 2349 2350 static u64 access_rx_rbuf_desc_part1_unc_err_cnt(const struct cntr_entry *entry, 2351 void *context, int vl, 2352 int mode, u64 data) 2353 { 2354 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2355 2356 return dd->rcv_err_status_cnt[47]; 2357 } 2358 2359 static u64 access_rx_hq_intr_fsm_err_cnt(const struct cntr_entry *entry, 2360 void *context, int vl, int mode, 2361 u64 data) 2362 { 2363 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2364 2365 return dd->rcv_err_status_cnt[46]; 2366 } 2367 2368 static u64 access_rx_hq_intr_csr_parity_err_cnt( 2369 const struct cntr_entry *entry, 2370 void *context, int vl, int mode, u64 data) 2371 { 2372 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2373 2374 return dd->rcv_err_status_cnt[45]; 2375 } 2376 2377 static u64 access_rx_lookup_csr_parity_err_cnt( 2378 const struct cntr_entry *entry, 2379 void *context, int vl, int mode, u64 data) 2380 { 2381 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2382 2383 return dd->rcv_err_status_cnt[44]; 2384 } 2385 2386 static u64 access_rx_lookup_rcv_array_cor_err_cnt( 2387 const struct cntr_entry *entry, 2388 void *context, int vl, int mode, u64 data) 2389 { 2390 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2391 2392 return dd->rcv_err_status_cnt[43]; 2393 } 2394 2395 static u64 access_rx_lookup_rcv_array_unc_err_cnt( 2396 const struct cntr_entry *entry, 2397 void *context, int vl, int mode, u64 data) 2398 { 2399 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2400 2401 return dd->rcv_err_status_cnt[42]; 2402 } 2403 2404 static u64 access_rx_lookup_des_part2_parity_err_cnt( 2405 const struct cntr_entry *entry, 2406 void *context, int vl, int mode, u64 data) 2407 { 2408 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2409 2410 return dd->rcv_err_status_cnt[41]; 2411 } 2412 2413 static u64 access_rx_lookup_des_part1_unc_cor_err_cnt( 2414 const struct cntr_entry *entry, 2415 void *context, int vl, int mode, u64 data) 2416 { 2417 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2418 2419 return dd->rcv_err_status_cnt[40]; 2420 } 2421 2422 static u64 access_rx_lookup_des_part1_unc_err_cnt( 2423 const struct cntr_entry *entry, 2424 void *context, int vl, int mode, u64 data) 2425 { 2426 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2427 2428 return dd->rcv_err_status_cnt[39]; 2429 } 2430 2431 static u64 access_rx_rbuf_next_free_buf_cor_err_cnt( 2432 const struct cntr_entry *entry, 2433 void *context, int vl, int mode, u64 data) 2434 { 2435 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2436 2437 return dd->rcv_err_status_cnt[38]; 2438 } 2439 2440 static u64 access_rx_rbuf_next_free_buf_unc_err_cnt( 2441 const struct cntr_entry *entry, 2442 void *context, int vl, int mode, u64 data) 2443 { 2444 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2445 2446 return dd->rcv_err_status_cnt[37]; 2447 } 2448 2449 static u64 access_rbuf_fl_init_wr_addr_parity_err_cnt( 2450 const struct cntr_entry *entry, 2451 void *context, int vl, int mode, u64 data) 2452 { 2453 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2454 2455 return dd->rcv_err_status_cnt[36]; 2456 } 2457 2458 static u64 access_rx_rbuf_fl_initdone_parity_err_cnt( 2459 const struct cntr_entry *entry, 2460 void *context, int vl, int mode, u64 data) 2461 { 2462 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2463 2464 return dd->rcv_err_status_cnt[35]; 2465 } 2466 2467 static u64 access_rx_rbuf_fl_write_addr_parity_err_cnt( 2468 const struct cntr_entry *entry, 2469 void *context, int vl, int mode, u64 data) 2470 { 2471 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2472 2473 return dd->rcv_err_status_cnt[34]; 2474 } 2475 2476 static u64 access_rx_rbuf_fl_rd_addr_parity_err_cnt( 2477 const struct cntr_entry *entry, 2478 void *context, int vl, int mode, u64 data) 2479 { 2480 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2481 2482 return dd->rcv_err_status_cnt[33]; 2483 } 2484 2485 static u64 access_rx_rbuf_empty_err_cnt(const struct cntr_entry *entry, 2486 void *context, int vl, int mode, 2487 u64 data) 2488 { 2489 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2490 2491 return dd->rcv_err_status_cnt[32]; 2492 } 2493 2494 static u64 access_rx_rbuf_full_err_cnt(const struct cntr_entry *entry, 2495 void *context, int vl, int mode, 2496 u64 data) 2497 { 2498 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2499 2500 return dd->rcv_err_status_cnt[31]; 2501 } 2502 2503 static u64 access_rbuf_bad_lookup_err_cnt(const struct cntr_entry *entry, 2504 void *context, int vl, int mode, 2505 u64 data) 2506 { 2507 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2508 2509 return dd->rcv_err_status_cnt[30]; 2510 } 2511 2512 static u64 access_rbuf_ctx_id_parity_err_cnt(const struct cntr_entry *entry, 2513 void *context, int vl, int mode, 2514 u64 data) 2515 { 2516 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2517 2518 return dd->rcv_err_status_cnt[29]; 2519 } 2520 2521 static u64 access_rbuf_csr_qeopdw_parity_err_cnt(const struct cntr_entry *entry, 2522 void *context, int vl, 2523 int mode, u64 data) 2524 { 2525 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2526 2527 return dd->rcv_err_status_cnt[28]; 2528 } 2529 2530 static u64 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt( 2531 const struct cntr_entry *entry, 2532 void *context, int vl, int mode, u64 data) 2533 { 2534 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2535 2536 return dd->rcv_err_status_cnt[27]; 2537 } 2538 2539 static u64 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt( 2540 const struct cntr_entry *entry, 2541 void *context, int vl, int mode, u64 data) 2542 { 2543 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2544 2545 return dd->rcv_err_status_cnt[26]; 2546 } 2547 2548 static u64 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt( 2549 const struct cntr_entry *entry, 2550 void *context, int vl, int mode, u64 data) 2551 { 2552 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2553 2554 return dd->rcv_err_status_cnt[25]; 2555 } 2556 2557 static u64 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt( 2558 const struct cntr_entry *entry, 2559 void *context, int vl, int mode, u64 data) 2560 { 2561 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2562 2563 return dd->rcv_err_status_cnt[24]; 2564 } 2565 2566 static u64 access_rx_rbuf_csr_q_next_buf_parity_err_cnt( 2567 const struct cntr_entry *entry, 2568 void *context, int vl, int mode, u64 data) 2569 { 2570 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2571 2572 return dd->rcv_err_status_cnt[23]; 2573 } 2574 2575 static u64 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt( 2576 const struct cntr_entry *entry, 2577 void *context, int vl, int mode, u64 data) 2578 { 2579 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2580 2581 return dd->rcv_err_status_cnt[22]; 2582 } 2583 2584 static u64 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt( 2585 const struct cntr_entry *entry, 2586 void *context, int vl, int mode, u64 data) 2587 { 2588 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2589 2590 return dd->rcv_err_status_cnt[21]; 2591 } 2592 2593 static u64 access_rx_rbuf_block_list_read_cor_err_cnt( 2594 const struct cntr_entry *entry, 2595 void *context, int vl, int mode, u64 data) 2596 { 2597 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2598 2599 return dd->rcv_err_status_cnt[20]; 2600 } 2601 2602 static u64 access_rx_rbuf_block_list_read_unc_err_cnt( 2603 const struct cntr_entry *entry, 2604 void *context, int vl, int mode, u64 data) 2605 { 2606 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2607 2608 return dd->rcv_err_status_cnt[19]; 2609 } 2610 2611 static u64 access_rx_rbuf_lookup_des_cor_err_cnt(const struct cntr_entry *entry, 2612 void *context, int vl, 2613 int mode, u64 data) 2614 { 2615 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2616 2617 return dd->rcv_err_status_cnt[18]; 2618 } 2619 2620 static u64 access_rx_rbuf_lookup_des_unc_err_cnt(const struct cntr_entry *entry, 2621 void *context, int vl, 2622 int mode, u64 data) 2623 { 2624 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2625 2626 return dd->rcv_err_status_cnt[17]; 2627 } 2628 2629 static u64 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt( 2630 const struct cntr_entry *entry, 2631 void *context, int vl, int mode, u64 data) 2632 { 2633 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2634 2635 return dd->rcv_err_status_cnt[16]; 2636 } 2637 2638 static u64 access_rx_rbuf_lookup_des_reg_unc_err_cnt( 2639 const struct cntr_entry *entry, 2640 void *context, int vl, int mode, u64 data) 2641 { 2642 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2643 2644 return dd->rcv_err_status_cnt[15]; 2645 } 2646 2647 static u64 access_rx_rbuf_free_list_cor_err_cnt(const struct cntr_entry *entry, 2648 void *context, int vl, 2649 int mode, u64 data) 2650 { 2651 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2652 2653 return dd->rcv_err_status_cnt[14]; 2654 } 2655 2656 static u64 access_rx_rbuf_free_list_unc_err_cnt(const struct cntr_entry *entry, 2657 void *context, int vl, 2658 int mode, u64 data) 2659 { 2660 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2661 2662 return dd->rcv_err_status_cnt[13]; 2663 } 2664 2665 static u64 access_rx_rcv_fsm_encoding_err_cnt(const struct cntr_entry *entry, 2666 void *context, int vl, int mode, 2667 u64 data) 2668 { 2669 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2670 2671 return dd->rcv_err_status_cnt[12]; 2672 } 2673 2674 static u64 access_rx_dma_flag_cor_err_cnt(const struct cntr_entry *entry, 2675 void *context, int vl, int mode, 2676 u64 data) 2677 { 2678 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2679 2680 return dd->rcv_err_status_cnt[11]; 2681 } 2682 2683 static u64 access_rx_dma_flag_unc_err_cnt(const struct cntr_entry *entry, 2684 void *context, int vl, int mode, 2685 u64 data) 2686 { 2687 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2688 2689 return dd->rcv_err_status_cnt[10]; 2690 } 2691 2692 static u64 access_rx_dc_sop_eop_parity_err_cnt(const struct cntr_entry *entry, 2693 void *context, int vl, int mode, 2694 u64 data) 2695 { 2696 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2697 2698 return dd->rcv_err_status_cnt[9]; 2699 } 2700 2701 static u64 access_rx_rcv_csr_parity_err_cnt(const struct cntr_entry *entry, 2702 void *context, int vl, int mode, 2703 u64 data) 2704 { 2705 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2706 2707 return dd->rcv_err_status_cnt[8]; 2708 } 2709 2710 static u64 access_rx_rcv_qp_map_table_cor_err_cnt( 2711 const struct cntr_entry *entry, 2712 void *context, int vl, int mode, u64 data) 2713 { 2714 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2715 2716 return dd->rcv_err_status_cnt[7]; 2717 } 2718 2719 static u64 access_rx_rcv_qp_map_table_unc_err_cnt( 2720 const struct cntr_entry *entry, 2721 void *context, int vl, int mode, u64 data) 2722 { 2723 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2724 2725 return dd->rcv_err_status_cnt[6]; 2726 } 2727 2728 static u64 access_rx_rcv_data_cor_err_cnt(const struct cntr_entry *entry, 2729 void *context, int vl, int mode, 2730 u64 data) 2731 { 2732 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2733 2734 return dd->rcv_err_status_cnt[5]; 2735 } 2736 2737 static u64 access_rx_rcv_data_unc_err_cnt(const struct cntr_entry *entry, 2738 void *context, int vl, int mode, 2739 u64 data) 2740 { 2741 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2742 2743 return dd->rcv_err_status_cnt[4]; 2744 } 2745 2746 static u64 access_rx_rcv_hdr_cor_err_cnt(const struct cntr_entry *entry, 2747 void *context, int vl, int mode, 2748 u64 data) 2749 { 2750 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2751 2752 return dd->rcv_err_status_cnt[3]; 2753 } 2754 2755 static u64 access_rx_rcv_hdr_unc_err_cnt(const struct cntr_entry *entry, 2756 void *context, int vl, int mode, 2757 u64 data) 2758 { 2759 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2760 2761 return dd->rcv_err_status_cnt[2]; 2762 } 2763 2764 static u64 access_rx_dc_intf_parity_err_cnt(const struct cntr_entry *entry, 2765 void *context, int vl, int mode, 2766 u64 data) 2767 { 2768 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2769 2770 return dd->rcv_err_status_cnt[1]; 2771 } 2772 2773 static u64 access_rx_dma_csr_cor_err_cnt(const struct cntr_entry *entry, 2774 void *context, int vl, int mode, 2775 u64 data) 2776 { 2777 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2778 2779 return dd->rcv_err_status_cnt[0]; 2780 } 2781 2782 /* 2783 * Software counters corresponding to each of the 2784 * error status bits within SendPioErrStatus 2785 */ 2786 static u64 access_pio_pec_sop_head_parity_err_cnt( 2787 const struct cntr_entry *entry, 2788 void *context, int vl, int mode, u64 data) 2789 { 2790 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2791 2792 return dd->send_pio_err_status_cnt[35]; 2793 } 2794 2795 static u64 access_pio_pcc_sop_head_parity_err_cnt( 2796 const struct cntr_entry *entry, 2797 void *context, int vl, int mode, u64 data) 2798 { 2799 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2800 2801 return dd->send_pio_err_status_cnt[34]; 2802 } 2803 2804 static u64 access_pio_last_returned_cnt_parity_err_cnt( 2805 const struct cntr_entry *entry, 2806 void *context, int vl, int mode, u64 data) 2807 { 2808 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2809 2810 return dd->send_pio_err_status_cnt[33]; 2811 } 2812 2813 static u64 access_pio_current_free_cnt_parity_err_cnt( 2814 const struct cntr_entry *entry, 2815 void *context, int vl, int mode, u64 data) 2816 { 2817 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2818 2819 return dd->send_pio_err_status_cnt[32]; 2820 } 2821 2822 static u64 access_pio_reserved_31_err_cnt(const struct cntr_entry *entry, 2823 void *context, int vl, int mode, 2824 u64 data) 2825 { 2826 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2827 2828 return dd->send_pio_err_status_cnt[31]; 2829 } 2830 2831 static u64 access_pio_reserved_30_err_cnt(const struct cntr_entry *entry, 2832 void *context, int vl, int mode, 2833 u64 data) 2834 { 2835 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2836 2837 return dd->send_pio_err_status_cnt[30]; 2838 } 2839 2840 static u64 access_pio_ppmc_sop_len_err_cnt(const struct cntr_entry *entry, 2841 void *context, int vl, int mode, 2842 u64 data) 2843 { 2844 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2845 2846 return dd->send_pio_err_status_cnt[29]; 2847 } 2848 2849 static u64 access_pio_ppmc_bqc_mem_parity_err_cnt( 2850 const struct cntr_entry *entry, 2851 void *context, int vl, int mode, u64 data) 2852 { 2853 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2854 2855 return dd->send_pio_err_status_cnt[28]; 2856 } 2857 2858 static u64 access_pio_vl_fifo_parity_err_cnt(const struct cntr_entry *entry, 2859 void *context, int vl, int mode, 2860 u64 data) 2861 { 2862 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2863 2864 return dd->send_pio_err_status_cnt[27]; 2865 } 2866 2867 static u64 access_pio_vlf_sop_parity_err_cnt(const struct cntr_entry *entry, 2868 void *context, int vl, int mode, 2869 u64 data) 2870 { 2871 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2872 2873 return dd->send_pio_err_status_cnt[26]; 2874 } 2875 2876 static u64 access_pio_vlf_v1_len_parity_err_cnt(const struct cntr_entry *entry, 2877 void *context, int vl, 2878 int mode, u64 data) 2879 { 2880 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2881 2882 return dd->send_pio_err_status_cnt[25]; 2883 } 2884 2885 static u64 access_pio_block_qw_count_parity_err_cnt( 2886 const struct cntr_entry *entry, 2887 void *context, int vl, int mode, u64 data) 2888 { 2889 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2890 2891 return dd->send_pio_err_status_cnt[24]; 2892 } 2893 2894 static u64 access_pio_write_qw_valid_parity_err_cnt( 2895 const struct cntr_entry *entry, 2896 void *context, int vl, int mode, u64 data) 2897 { 2898 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2899 2900 return dd->send_pio_err_status_cnt[23]; 2901 } 2902 2903 static u64 access_pio_state_machine_err_cnt(const struct cntr_entry *entry, 2904 void *context, int vl, int mode, 2905 u64 data) 2906 { 2907 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2908 2909 return dd->send_pio_err_status_cnt[22]; 2910 } 2911 2912 static u64 access_pio_write_data_parity_err_cnt(const struct cntr_entry *entry, 2913 void *context, int vl, 2914 int mode, u64 data) 2915 { 2916 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2917 2918 return dd->send_pio_err_status_cnt[21]; 2919 } 2920 2921 static u64 access_pio_host_addr_mem_cor_err_cnt(const struct cntr_entry *entry, 2922 void *context, int vl, 2923 int mode, u64 data) 2924 { 2925 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2926 2927 return dd->send_pio_err_status_cnt[20]; 2928 } 2929 2930 static u64 access_pio_host_addr_mem_unc_err_cnt(const struct cntr_entry *entry, 2931 void *context, int vl, 2932 int mode, u64 data) 2933 { 2934 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2935 2936 return dd->send_pio_err_status_cnt[19]; 2937 } 2938 2939 static u64 access_pio_pkt_evict_sm_or_arb_sm_err_cnt( 2940 const struct cntr_entry *entry, 2941 void *context, int vl, int mode, u64 data) 2942 { 2943 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2944 2945 return dd->send_pio_err_status_cnt[18]; 2946 } 2947 2948 static u64 access_pio_init_sm_in_err_cnt(const struct cntr_entry *entry, 2949 void *context, int vl, int mode, 2950 u64 data) 2951 { 2952 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2953 2954 return dd->send_pio_err_status_cnt[17]; 2955 } 2956 2957 static u64 access_pio_ppmc_pbl_fifo_err_cnt(const struct cntr_entry *entry, 2958 void *context, int vl, int mode, 2959 u64 data) 2960 { 2961 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2962 2963 return dd->send_pio_err_status_cnt[16]; 2964 } 2965 2966 static u64 access_pio_credit_ret_fifo_parity_err_cnt( 2967 const struct cntr_entry *entry, 2968 void *context, int vl, int mode, u64 data) 2969 { 2970 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2971 2972 return dd->send_pio_err_status_cnt[15]; 2973 } 2974 2975 static u64 access_pio_v1_len_mem_bank1_cor_err_cnt( 2976 const struct cntr_entry *entry, 2977 void *context, int vl, int mode, u64 data) 2978 { 2979 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2980 2981 return dd->send_pio_err_status_cnt[14]; 2982 } 2983 2984 static u64 access_pio_v1_len_mem_bank0_cor_err_cnt( 2985 const struct cntr_entry *entry, 2986 void *context, int vl, int mode, u64 data) 2987 { 2988 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2989 2990 return dd->send_pio_err_status_cnt[13]; 2991 } 2992 2993 static u64 access_pio_v1_len_mem_bank1_unc_err_cnt( 2994 const struct cntr_entry *entry, 2995 void *context, int vl, int mode, u64 data) 2996 { 2997 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 2998 2999 return dd->send_pio_err_status_cnt[12]; 3000 } 3001 3002 static u64 access_pio_v1_len_mem_bank0_unc_err_cnt( 3003 const struct cntr_entry *entry, 3004 void *context, int vl, int mode, u64 data) 3005 { 3006 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3007 3008 return dd->send_pio_err_status_cnt[11]; 3009 } 3010 3011 static u64 access_pio_sm_pkt_reset_parity_err_cnt( 3012 const struct cntr_entry *entry, 3013 void *context, int vl, int mode, u64 data) 3014 { 3015 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3016 3017 return dd->send_pio_err_status_cnt[10]; 3018 } 3019 3020 static u64 access_pio_pkt_evict_fifo_parity_err_cnt( 3021 const struct cntr_entry *entry, 3022 void *context, int vl, int mode, u64 data) 3023 { 3024 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3025 3026 return dd->send_pio_err_status_cnt[9]; 3027 } 3028 3029 static u64 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt( 3030 const struct cntr_entry *entry, 3031 void *context, int vl, int mode, u64 data) 3032 { 3033 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3034 3035 return dd->send_pio_err_status_cnt[8]; 3036 } 3037 3038 static u64 access_pio_sbrdctl_crrel_parity_err_cnt( 3039 const struct cntr_entry *entry, 3040 void *context, int vl, int mode, u64 data) 3041 { 3042 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3043 3044 return dd->send_pio_err_status_cnt[7]; 3045 } 3046 3047 static u64 access_pio_pec_fifo_parity_err_cnt(const struct cntr_entry *entry, 3048 void *context, int vl, int mode, 3049 u64 data) 3050 { 3051 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3052 3053 return dd->send_pio_err_status_cnt[6]; 3054 } 3055 3056 static u64 access_pio_pcc_fifo_parity_err_cnt(const struct cntr_entry *entry, 3057 void *context, int vl, int mode, 3058 u64 data) 3059 { 3060 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3061 3062 return dd->send_pio_err_status_cnt[5]; 3063 } 3064 3065 static u64 access_pio_sb_mem_fifo1_err_cnt(const struct cntr_entry *entry, 3066 void *context, int vl, int mode, 3067 u64 data) 3068 { 3069 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3070 3071 return dd->send_pio_err_status_cnt[4]; 3072 } 3073 3074 static u64 access_pio_sb_mem_fifo0_err_cnt(const struct cntr_entry *entry, 3075 void *context, int vl, int mode, 3076 u64 data) 3077 { 3078 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3079 3080 return dd->send_pio_err_status_cnt[3]; 3081 } 3082 3083 static u64 access_pio_csr_parity_err_cnt(const struct cntr_entry *entry, 3084 void *context, int vl, int mode, 3085 u64 data) 3086 { 3087 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3088 3089 return dd->send_pio_err_status_cnt[2]; 3090 } 3091 3092 static u64 access_pio_write_addr_parity_err_cnt(const struct cntr_entry *entry, 3093 void *context, int vl, 3094 int mode, u64 data) 3095 { 3096 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3097 3098 return dd->send_pio_err_status_cnt[1]; 3099 } 3100 3101 static u64 access_pio_write_bad_ctxt_err_cnt(const struct cntr_entry *entry, 3102 void *context, int vl, int mode, 3103 u64 data) 3104 { 3105 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3106 3107 return dd->send_pio_err_status_cnt[0]; 3108 } 3109 3110 /* 3111 * Software counters corresponding to each of the 3112 * error status bits within SendDmaErrStatus 3113 */ 3114 static u64 access_sdma_pcie_req_tracking_cor_err_cnt( 3115 const struct cntr_entry *entry, 3116 void *context, int vl, int mode, u64 data) 3117 { 3118 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3119 3120 return dd->send_dma_err_status_cnt[3]; 3121 } 3122 3123 static u64 access_sdma_pcie_req_tracking_unc_err_cnt( 3124 const struct cntr_entry *entry, 3125 void *context, int vl, int mode, u64 data) 3126 { 3127 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3128 3129 return dd->send_dma_err_status_cnt[2]; 3130 } 3131 3132 static u64 access_sdma_csr_parity_err_cnt(const struct cntr_entry *entry, 3133 void *context, int vl, int mode, 3134 u64 data) 3135 { 3136 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3137 3138 return dd->send_dma_err_status_cnt[1]; 3139 } 3140 3141 static u64 access_sdma_rpy_tag_err_cnt(const struct cntr_entry *entry, 3142 void *context, int vl, int mode, 3143 u64 data) 3144 { 3145 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3146 3147 return dd->send_dma_err_status_cnt[0]; 3148 } 3149 3150 /* 3151 * Software counters corresponding to each of the 3152 * error status bits within SendEgressErrStatus 3153 */ 3154 static u64 access_tx_read_pio_memory_csr_unc_err_cnt( 3155 const struct cntr_entry *entry, 3156 void *context, int vl, int mode, u64 data) 3157 { 3158 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3159 3160 return dd->send_egress_err_status_cnt[63]; 3161 } 3162 3163 static u64 access_tx_read_sdma_memory_csr_err_cnt( 3164 const struct cntr_entry *entry, 3165 void *context, int vl, int mode, u64 data) 3166 { 3167 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3168 3169 return dd->send_egress_err_status_cnt[62]; 3170 } 3171 3172 static u64 access_tx_egress_fifo_cor_err_cnt(const struct cntr_entry *entry, 3173 void *context, int vl, int mode, 3174 u64 data) 3175 { 3176 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3177 3178 return dd->send_egress_err_status_cnt[61]; 3179 } 3180 3181 static u64 access_tx_read_pio_memory_cor_err_cnt(const struct cntr_entry *entry, 3182 void *context, int vl, 3183 int mode, u64 data) 3184 { 3185 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3186 3187 return dd->send_egress_err_status_cnt[60]; 3188 } 3189 3190 static u64 access_tx_read_sdma_memory_cor_err_cnt( 3191 const struct cntr_entry *entry, 3192 void *context, int vl, int mode, u64 data) 3193 { 3194 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3195 3196 return dd->send_egress_err_status_cnt[59]; 3197 } 3198 3199 static u64 access_tx_sb_hdr_cor_err_cnt(const struct cntr_entry *entry, 3200 void *context, int vl, int mode, 3201 u64 data) 3202 { 3203 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3204 3205 return dd->send_egress_err_status_cnt[58]; 3206 } 3207 3208 static u64 access_tx_credit_overrun_err_cnt(const struct cntr_entry *entry, 3209 void *context, int vl, int mode, 3210 u64 data) 3211 { 3212 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3213 3214 return dd->send_egress_err_status_cnt[57]; 3215 } 3216 3217 static u64 access_tx_launch_fifo8_cor_err_cnt(const struct cntr_entry *entry, 3218 void *context, int vl, int mode, 3219 u64 data) 3220 { 3221 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3222 3223 return dd->send_egress_err_status_cnt[56]; 3224 } 3225 3226 static u64 access_tx_launch_fifo7_cor_err_cnt(const struct cntr_entry *entry, 3227 void *context, int vl, int mode, 3228 u64 data) 3229 { 3230 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3231 3232 return dd->send_egress_err_status_cnt[55]; 3233 } 3234 3235 static u64 access_tx_launch_fifo6_cor_err_cnt(const struct cntr_entry *entry, 3236 void *context, int vl, int mode, 3237 u64 data) 3238 { 3239 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3240 3241 return dd->send_egress_err_status_cnt[54]; 3242 } 3243 3244 static u64 access_tx_launch_fifo5_cor_err_cnt(const struct cntr_entry *entry, 3245 void *context, int vl, int mode, 3246 u64 data) 3247 { 3248 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3249 3250 return dd->send_egress_err_status_cnt[53]; 3251 } 3252 3253 static u64 access_tx_launch_fifo4_cor_err_cnt(const struct cntr_entry *entry, 3254 void *context, int vl, int mode, 3255 u64 data) 3256 { 3257 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3258 3259 return dd->send_egress_err_status_cnt[52]; 3260 } 3261 3262 static u64 access_tx_launch_fifo3_cor_err_cnt(const struct cntr_entry *entry, 3263 void *context, int vl, int mode, 3264 u64 data) 3265 { 3266 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3267 3268 return dd->send_egress_err_status_cnt[51]; 3269 } 3270 3271 static u64 access_tx_launch_fifo2_cor_err_cnt(const struct cntr_entry *entry, 3272 void *context, int vl, int mode, 3273 u64 data) 3274 { 3275 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3276 3277 return dd->send_egress_err_status_cnt[50]; 3278 } 3279 3280 static u64 access_tx_launch_fifo1_cor_err_cnt(const struct cntr_entry *entry, 3281 void *context, int vl, int mode, 3282 u64 data) 3283 { 3284 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3285 3286 return dd->send_egress_err_status_cnt[49]; 3287 } 3288 3289 static u64 access_tx_launch_fifo0_cor_err_cnt(const struct cntr_entry *entry, 3290 void *context, int vl, int mode, 3291 u64 data) 3292 { 3293 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3294 3295 return dd->send_egress_err_status_cnt[48]; 3296 } 3297 3298 static u64 access_tx_credit_return_vl_err_cnt(const struct cntr_entry *entry, 3299 void *context, int vl, int mode, 3300 u64 data) 3301 { 3302 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3303 3304 return dd->send_egress_err_status_cnt[47]; 3305 } 3306 3307 static u64 access_tx_hcrc_insertion_err_cnt(const struct cntr_entry *entry, 3308 void *context, int vl, int mode, 3309 u64 data) 3310 { 3311 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3312 3313 return dd->send_egress_err_status_cnt[46]; 3314 } 3315 3316 static u64 access_tx_egress_fifo_unc_err_cnt(const struct cntr_entry *entry, 3317 void *context, int vl, int mode, 3318 u64 data) 3319 { 3320 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3321 3322 return dd->send_egress_err_status_cnt[45]; 3323 } 3324 3325 static u64 access_tx_read_pio_memory_unc_err_cnt(const struct cntr_entry *entry, 3326 void *context, int vl, 3327 int mode, u64 data) 3328 { 3329 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3330 3331 return dd->send_egress_err_status_cnt[44]; 3332 } 3333 3334 static u64 access_tx_read_sdma_memory_unc_err_cnt( 3335 const struct cntr_entry *entry, 3336 void *context, int vl, int mode, u64 data) 3337 { 3338 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3339 3340 return dd->send_egress_err_status_cnt[43]; 3341 } 3342 3343 static u64 access_tx_sb_hdr_unc_err_cnt(const struct cntr_entry *entry, 3344 void *context, int vl, int mode, 3345 u64 data) 3346 { 3347 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3348 3349 return dd->send_egress_err_status_cnt[42]; 3350 } 3351 3352 static u64 access_tx_credit_return_partiy_err_cnt( 3353 const struct cntr_entry *entry, 3354 void *context, int vl, int mode, u64 data) 3355 { 3356 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3357 3358 return dd->send_egress_err_status_cnt[41]; 3359 } 3360 3361 static u64 access_tx_launch_fifo8_unc_or_parity_err_cnt( 3362 const struct cntr_entry *entry, 3363 void *context, int vl, int mode, u64 data) 3364 { 3365 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3366 3367 return dd->send_egress_err_status_cnt[40]; 3368 } 3369 3370 static u64 access_tx_launch_fifo7_unc_or_parity_err_cnt( 3371 const struct cntr_entry *entry, 3372 void *context, int vl, int mode, u64 data) 3373 { 3374 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3375 3376 return dd->send_egress_err_status_cnt[39]; 3377 } 3378 3379 static u64 access_tx_launch_fifo6_unc_or_parity_err_cnt( 3380 const struct cntr_entry *entry, 3381 void *context, int vl, int mode, u64 data) 3382 { 3383 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3384 3385 return dd->send_egress_err_status_cnt[38]; 3386 } 3387 3388 static u64 access_tx_launch_fifo5_unc_or_parity_err_cnt( 3389 const struct cntr_entry *entry, 3390 void *context, int vl, int mode, u64 data) 3391 { 3392 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3393 3394 return dd->send_egress_err_status_cnt[37]; 3395 } 3396 3397 static u64 access_tx_launch_fifo4_unc_or_parity_err_cnt( 3398 const struct cntr_entry *entry, 3399 void *context, int vl, int mode, u64 data) 3400 { 3401 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3402 3403 return dd->send_egress_err_status_cnt[36]; 3404 } 3405 3406 static u64 access_tx_launch_fifo3_unc_or_parity_err_cnt( 3407 const struct cntr_entry *entry, 3408 void *context, int vl, int mode, u64 data) 3409 { 3410 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3411 3412 return dd->send_egress_err_status_cnt[35]; 3413 } 3414 3415 static u64 access_tx_launch_fifo2_unc_or_parity_err_cnt( 3416 const struct cntr_entry *entry, 3417 void *context, int vl, int mode, u64 data) 3418 { 3419 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3420 3421 return dd->send_egress_err_status_cnt[34]; 3422 } 3423 3424 static u64 access_tx_launch_fifo1_unc_or_parity_err_cnt( 3425 const struct cntr_entry *entry, 3426 void *context, int vl, int mode, u64 data) 3427 { 3428 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3429 3430 return dd->send_egress_err_status_cnt[33]; 3431 } 3432 3433 static u64 access_tx_launch_fifo0_unc_or_parity_err_cnt( 3434 const struct cntr_entry *entry, 3435 void *context, int vl, int mode, u64 data) 3436 { 3437 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3438 3439 return dd->send_egress_err_status_cnt[32]; 3440 } 3441 3442 static u64 access_tx_sdma15_disallowed_packet_err_cnt( 3443 const struct cntr_entry *entry, 3444 void *context, int vl, int mode, u64 data) 3445 { 3446 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3447 3448 return dd->send_egress_err_status_cnt[31]; 3449 } 3450 3451 static u64 access_tx_sdma14_disallowed_packet_err_cnt( 3452 const struct cntr_entry *entry, 3453 void *context, int vl, int mode, u64 data) 3454 { 3455 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3456 3457 return dd->send_egress_err_status_cnt[30]; 3458 } 3459 3460 static u64 access_tx_sdma13_disallowed_packet_err_cnt( 3461 const struct cntr_entry *entry, 3462 void *context, int vl, int mode, u64 data) 3463 { 3464 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3465 3466 return dd->send_egress_err_status_cnt[29]; 3467 } 3468 3469 static u64 access_tx_sdma12_disallowed_packet_err_cnt( 3470 const struct cntr_entry *entry, 3471 void *context, int vl, int mode, u64 data) 3472 { 3473 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3474 3475 return dd->send_egress_err_status_cnt[28]; 3476 } 3477 3478 static u64 access_tx_sdma11_disallowed_packet_err_cnt( 3479 const struct cntr_entry *entry, 3480 void *context, int vl, int mode, u64 data) 3481 { 3482 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3483 3484 return dd->send_egress_err_status_cnt[27]; 3485 } 3486 3487 static u64 access_tx_sdma10_disallowed_packet_err_cnt( 3488 const struct cntr_entry *entry, 3489 void *context, int vl, int mode, u64 data) 3490 { 3491 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3492 3493 return dd->send_egress_err_status_cnt[26]; 3494 } 3495 3496 static u64 access_tx_sdma9_disallowed_packet_err_cnt( 3497 const struct cntr_entry *entry, 3498 void *context, int vl, int mode, u64 data) 3499 { 3500 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3501 3502 return dd->send_egress_err_status_cnt[25]; 3503 } 3504 3505 static u64 access_tx_sdma8_disallowed_packet_err_cnt( 3506 const struct cntr_entry *entry, 3507 void *context, int vl, int mode, u64 data) 3508 { 3509 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3510 3511 return dd->send_egress_err_status_cnt[24]; 3512 } 3513 3514 static u64 access_tx_sdma7_disallowed_packet_err_cnt( 3515 const struct cntr_entry *entry, 3516 void *context, int vl, int mode, u64 data) 3517 { 3518 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3519 3520 return dd->send_egress_err_status_cnt[23]; 3521 } 3522 3523 static u64 access_tx_sdma6_disallowed_packet_err_cnt( 3524 const struct cntr_entry *entry, 3525 void *context, int vl, int mode, u64 data) 3526 { 3527 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3528 3529 return dd->send_egress_err_status_cnt[22]; 3530 } 3531 3532 static u64 access_tx_sdma5_disallowed_packet_err_cnt( 3533 const struct cntr_entry *entry, 3534 void *context, int vl, int mode, u64 data) 3535 { 3536 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3537 3538 return dd->send_egress_err_status_cnt[21]; 3539 } 3540 3541 static u64 access_tx_sdma4_disallowed_packet_err_cnt( 3542 const struct cntr_entry *entry, 3543 void *context, int vl, int mode, u64 data) 3544 { 3545 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3546 3547 return dd->send_egress_err_status_cnt[20]; 3548 } 3549 3550 static u64 access_tx_sdma3_disallowed_packet_err_cnt( 3551 const struct cntr_entry *entry, 3552 void *context, int vl, int mode, u64 data) 3553 { 3554 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3555 3556 return dd->send_egress_err_status_cnt[19]; 3557 } 3558 3559 static u64 access_tx_sdma2_disallowed_packet_err_cnt( 3560 const struct cntr_entry *entry, 3561 void *context, int vl, int mode, u64 data) 3562 { 3563 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3564 3565 return dd->send_egress_err_status_cnt[18]; 3566 } 3567 3568 static u64 access_tx_sdma1_disallowed_packet_err_cnt( 3569 const struct cntr_entry *entry, 3570 void *context, int vl, int mode, u64 data) 3571 { 3572 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3573 3574 return dd->send_egress_err_status_cnt[17]; 3575 } 3576 3577 static u64 access_tx_sdma0_disallowed_packet_err_cnt( 3578 const struct cntr_entry *entry, 3579 void *context, int vl, int mode, u64 data) 3580 { 3581 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3582 3583 return dd->send_egress_err_status_cnt[16]; 3584 } 3585 3586 static u64 access_tx_config_parity_err_cnt(const struct cntr_entry *entry, 3587 void *context, int vl, int mode, 3588 u64 data) 3589 { 3590 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3591 3592 return dd->send_egress_err_status_cnt[15]; 3593 } 3594 3595 static u64 access_tx_sbrd_ctl_csr_parity_err_cnt(const struct cntr_entry *entry, 3596 void *context, int vl, 3597 int mode, u64 data) 3598 { 3599 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3600 3601 return dd->send_egress_err_status_cnt[14]; 3602 } 3603 3604 static u64 access_tx_launch_csr_parity_err_cnt(const struct cntr_entry *entry, 3605 void *context, int vl, int mode, 3606 u64 data) 3607 { 3608 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3609 3610 return dd->send_egress_err_status_cnt[13]; 3611 } 3612 3613 static u64 access_tx_illegal_vl_err_cnt(const struct cntr_entry *entry, 3614 void *context, int vl, int mode, 3615 u64 data) 3616 { 3617 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3618 3619 return dd->send_egress_err_status_cnt[12]; 3620 } 3621 3622 static u64 access_tx_sbrd_ctl_state_machine_parity_err_cnt( 3623 const struct cntr_entry *entry, 3624 void *context, int vl, int mode, u64 data) 3625 { 3626 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3627 3628 return dd->send_egress_err_status_cnt[11]; 3629 } 3630 3631 static u64 access_egress_reserved_10_err_cnt(const struct cntr_entry *entry, 3632 void *context, int vl, int mode, 3633 u64 data) 3634 { 3635 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3636 3637 return dd->send_egress_err_status_cnt[10]; 3638 } 3639 3640 static u64 access_egress_reserved_9_err_cnt(const struct cntr_entry *entry, 3641 void *context, int vl, int mode, 3642 u64 data) 3643 { 3644 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3645 3646 return dd->send_egress_err_status_cnt[9]; 3647 } 3648 3649 static u64 access_tx_sdma_launch_intf_parity_err_cnt( 3650 const struct cntr_entry *entry, 3651 void *context, int vl, int mode, u64 data) 3652 { 3653 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3654 3655 return dd->send_egress_err_status_cnt[8]; 3656 } 3657 3658 static u64 access_tx_pio_launch_intf_parity_err_cnt( 3659 const struct cntr_entry *entry, 3660 void *context, int vl, int mode, u64 data) 3661 { 3662 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3663 3664 return dd->send_egress_err_status_cnt[7]; 3665 } 3666 3667 static u64 access_egress_reserved_6_err_cnt(const struct cntr_entry *entry, 3668 void *context, int vl, int mode, 3669 u64 data) 3670 { 3671 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3672 3673 return dd->send_egress_err_status_cnt[6]; 3674 } 3675 3676 static u64 access_tx_incorrect_link_state_err_cnt( 3677 const struct cntr_entry *entry, 3678 void *context, int vl, int mode, u64 data) 3679 { 3680 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3681 3682 return dd->send_egress_err_status_cnt[5]; 3683 } 3684 3685 static u64 access_tx_linkdown_err_cnt(const struct cntr_entry *entry, 3686 void *context, int vl, int mode, 3687 u64 data) 3688 { 3689 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3690 3691 return dd->send_egress_err_status_cnt[4]; 3692 } 3693 3694 static u64 access_tx_egress_fifi_underrun_or_parity_err_cnt( 3695 const struct cntr_entry *entry, 3696 void *context, int vl, int mode, u64 data) 3697 { 3698 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3699 3700 return dd->send_egress_err_status_cnt[3]; 3701 } 3702 3703 static u64 access_egress_reserved_2_err_cnt(const struct cntr_entry *entry, 3704 void *context, int vl, int mode, 3705 u64 data) 3706 { 3707 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3708 3709 return dd->send_egress_err_status_cnt[2]; 3710 } 3711 3712 static u64 access_tx_pkt_integrity_mem_unc_err_cnt( 3713 const struct cntr_entry *entry, 3714 void *context, int vl, int mode, u64 data) 3715 { 3716 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3717 3718 return dd->send_egress_err_status_cnt[1]; 3719 } 3720 3721 static u64 access_tx_pkt_integrity_mem_cor_err_cnt( 3722 const struct cntr_entry *entry, 3723 void *context, int vl, int mode, u64 data) 3724 { 3725 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3726 3727 return dd->send_egress_err_status_cnt[0]; 3728 } 3729 3730 /* 3731 * Software counters corresponding to each of the 3732 * error status bits within SendErrStatus 3733 */ 3734 static u64 access_send_csr_write_bad_addr_err_cnt( 3735 const struct cntr_entry *entry, 3736 void *context, int vl, int mode, u64 data) 3737 { 3738 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3739 3740 return dd->send_err_status_cnt[2]; 3741 } 3742 3743 static u64 access_send_csr_read_bad_addr_err_cnt(const struct cntr_entry *entry, 3744 void *context, int vl, 3745 int mode, u64 data) 3746 { 3747 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3748 3749 return dd->send_err_status_cnt[1]; 3750 } 3751 3752 static u64 access_send_csr_parity_cnt(const struct cntr_entry *entry, 3753 void *context, int vl, int mode, 3754 u64 data) 3755 { 3756 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3757 3758 return dd->send_err_status_cnt[0]; 3759 } 3760 3761 /* 3762 * Software counters corresponding to each of the 3763 * error status bits within SendCtxtErrStatus 3764 */ 3765 static u64 access_pio_write_out_of_bounds_err_cnt( 3766 const struct cntr_entry *entry, 3767 void *context, int vl, int mode, u64 data) 3768 { 3769 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3770 3771 return dd->sw_ctxt_err_status_cnt[4]; 3772 } 3773 3774 static u64 access_pio_write_overflow_err_cnt(const struct cntr_entry *entry, 3775 void *context, int vl, int mode, 3776 u64 data) 3777 { 3778 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3779 3780 return dd->sw_ctxt_err_status_cnt[3]; 3781 } 3782 3783 static u64 access_pio_write_crosses_boundary_err_cnt( 3784 const struct cntr_entry *entry, 3785 void *context, int vl, int mode, u64 data) 3786 { 3787 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3788 3789 return dd->sw_ctxt_err_status_cnt[2]; 3790 } 3791 3792 static u64 access_pio_disallowed_packet_err_cnt(const struct cntr_entry *entry, 3793 void *context, int vl, 3794 int mode, u64 data) 3795 { 3796 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3797 3798 return dd->sw_ctxt_err_status_cnt[1]; 3799 } 3800 3801 static u64 access_pio_inconsistent_sop_err_cnt(const struct cntr_entry *entry, 3802 void *context, int vl, int mode, 3803 u64 data) 3804 { 3805 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3806 3807 return dd->sw_ctxt_err_status_cnt[0]; 3808 } 3809 3810 /* 3811 * Software counters corresponding to each of the 3812 * error status bits within SendDmaEngErrStatus 3813 */ 3814 static u64 access_sdma_header_request_fifo_cor_err_cnt( 3815 const struct cntr_entry *entry, 3816 void *context, int vl, int mode, u64 data) 3817 { 3818 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3819 3820 return dd->sw_send_dma_eng_err_status_cnt[23]; 3821 } 3822 3823 static u64 access_sdma_header_storage_cor_err_cnt( 3824 const struct cntr_entry *entry, 3825 void *context, int vl, int mode, u64 data) 3826 { 3827 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3828 3829 return dd->sw_send_dma_eng_err_status_cnt[22]; 3830 } 3831 3832 static u64 access_sdma_packet_tracking_cor_err_cnt( 3833 const struct cntr_entry *entry, 3834 void *context, int vl, int mode, u64 data) 3835 { 3836 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3837 3838 return dd->sw_send_dma_eng_err_status_cnt[21]; 3839 } 3840 3841 static u64 access_sdma_assembly_cor_err_cnt(const struct cntr_entry *entry, 3842 void *context, int vl, int mode, 3843 u64 data) 3844 { 3845 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3846 3847 return dd->sw_send_dma_eng_err_status_cnt[20]; 3848 } 3849 3850 static u64 access_sdma_desc_table_cor_err_cnt(const struct cntr_entry *entry, 3851 void *context, int vl, int mode, 3852 u64 data) 3853 { 3854 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3855 3856 return dd->sw_send_dma_eng_err_status_cnt[19]; 3857 } 3858 3859 static u64 access_sdma_header_request_fifo_unc_err_cnt( 3860 const struct cntr_entry *entry, 3861 void *context, int vl, int mode, u64 data) 3862 { 3863 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3864 3865 return dd->sw_send_dma_eng_err_status_cnt[18]; 3866 } 3867 3868 static u64 access_sdma_header_storage_unc_err_cnt( 3869 const struct cntr_entry *entry, 3870 void *context, int vl, int mode, u64 data) 3871 { 3872 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3873 3874 return dd->sw_send_dma_eng_err_status_cnt[17]; 3875 } 3876 3877 static u64 access_sdma_packet_tracking_unc_err_cnt( 3878 const struct cntr_entry *entry, 3879 void *context, int vl, int mode, u64 data) 3880 { 3881 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3882 3883 return dd->sw_send_dma_eng_err_status_cnt[16]; 3884 } 3885 3886 static u64 access_sdma_assembly_unc_err_cnt(const struct cntr_entry *entry, 3887 void *context, int vl, int mode, 3888 u64 data) 3889 { 3890 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3891 3892 return dd->sw_send_dma_eng_err_status_cnt[15]; 3893 } 3894 3895 static u64 access_sdma_desc_table_unc_err_cnt(const struct cntr_entry *entry, 3896 void *context, int vl, int mode, 3897 u64 data) 3898 { 3899 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3900 3901 return dd->sw_send_dma_eng_err_status_cnt[14]; 3902 } 3903 3904 static u64 access_sdma_timeout_err_cnt(const struct cntr_entry *entry, 3905 void *context, int vl, int mode, 3906 u64 data) 3907 { 3908 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3909 3910 return dd->sw_send_dma_eng_err_status_cnt[13]; 3911 } 3912 3913 static u64 access_sdma_header_length_err_cnt(const struct cntr_entry *entry, 3914 void *context, int vl, int mode, 3915 u64 data) 3916 { 3917 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3918 3919 return dd->sw_send_dma_eng_err_status_cnt[12]; 3920 } 3921 3922 static u64 access_sdma_header_address_err_cnt(const struct cntr_entry *entry, 3923 void *context, int vl, int mode, 3924 u64 data) 3925 { 3926 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3927 3928 return dd->sw_send_dma_eng_err_status_cnt[11]; 3929 } 3930 3931 static u64 access_sdma_header_select_err_cnt(const struct cntr_entry *entry, 3932 void *context, int vl, int mode, 3933 u64 data) 3934 { 3935 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3936 3937 return dd->sw_send_dma_eng_err_status_cnt[10]; 3938 } 3939 3940 static u64 access_sdma_reserved_9_err_cnt(const struct cntr_entry *entry, 3941 void *context, int vl, int mode, 3942 u64 data) 3943 { 3944 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3945 3946 return dd->sw_send_dma_eng_err_status_cnt[9]; 3947 } 3948 3949 static u64 access_sdma_packet_desc_overflow_err_cnt( 3950 const struct cntr_entry *entry, 3951 void *context, int vl, int mode, u64 data) 3952 { 3953 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3954 3955 return dd->sw_send_dma_eng_err_status_cnt[8]; 3956 } 3957 3958 static u64 access_sdma_length_mismatch_err_cnt(const struct cntr_entry *entry, 3959 void *context, int vl, 3960 int mode, u64 data) 3961 { 3962 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3963 3964 return dd->sw_send_dma_eng_err_status_cnt[7]; 3965 } 3966 3967 static u64 access_sdma_halt_err_cnt(const struct cntr_entry *entry, 3968 void *context, int vl, int mode, u64 data) 3969 { 3970 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3971 3972 return dd->sw_send_dma_eng_err_status_cnt[6]; 3973 } 3974 3975 static u64 access_sdma_mem_read_err_cnt(const struct cntr_entry *entry, 3976 void *context, int vl, int mode, 3977 u64 data) 3978 { 3979 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3980 3981 return dd->sw_send_dma_eng_err_status_cnt[5]; 3982 } 3983 3984 static u64 access_sdma_first_desc_err_cnt(const struct cntr_entry *entry, 3985 void *context, int vl, int mode, 3986 u64 data) 3987 { 3988 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3989 3990 return dd->sw_send_dma_eng_err_status_cnt[4]; 3991 } 3992 3993 static u64 access_sdma_tail_out_of_bounds_err_cnt( 3994 const struct cntr_entry *entry, 3995 void *context, int vl, int mode, u64 data) 3996 { 3997 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 3998 3999 return dd->sw_send_dma_eng_err_status_cnt[3]; 4000 } 4001 4002 static u64 access_sdma_too_long_err_cnt(const struct cntr_entry *entry, 4003 void *context, int vl, int mode, 4004 u64 data) 4005 { 4006 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 4007 4008 return dd->sw_send_dma_eng_err_status_cnt[2]; 4009 } 4010 4011 static u64 access_sdma_gen_mismatch_err_cnt(const struct cntr_entry *entry, 4012 void *context, int vl, int mode, 4013 u64 data) 4014 { 4015 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 4016 4017 return dd->sw_send_dma_eng_err_status_cnt[1]; 4018 } 4019 4020 static u64 access_sdma_wrong_dw_err_cnt(const struct cntr_entry *entry, 4021 void *context, int vl, int mode, 4022 u64 data) 4023 { 4024 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 4025 4026 return dd->sw_send_dma_eng_err_status_cnt[0]; 4027 } 4028 4029 static u64 access_dc_rcv_err_cnt(const struct cntr_entry *entry, 4030 void *context, int vl, int mode, 4031 u64 data) 4032 { 4033 struct hfi1_devdata *dd = (struct hfi1_devdata *)context; 4034 4035 u64 val = 0; 4036 u64 csr = entry->csr; 4037 4038 val = read_write_csr(dd, csr, mode, data); 4039 if (mode == CNTR_MODE_R) { 4040 val = val > CNTR_MAX - dd->sw_rcv_bypass_packet_errors ? 4041 CNTR_MAX : val + dd->sw_rcv_bypass_packet_errors; 4042 } else if (mode == CNTR_MODE_W) { 4043 dd->sw_rcv_bypass_packet_errors = 0; 4044 } else { 4045 dd_dev_err(dd, "Invalid cntr register access mode"); 4046 return 0; 4047 } 4048 return val; 4049 } 4050 4051 #define def_access_sw_cpu(cntr) \ 4052 static u64 access_sw_cpu_##cntr(const struct cntr_entry *entry, \ 4053 void *context, int vl, int mode, u64 data) \ 4054 { \ 4055 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \ 4056 return read_write_cpu(ppd->dd, &ppd->ibport_data.rvp.z_ ##cntr, \ 4057 ppd->ibport_data.rvp.cntr, vl, \ 4058 mode, data); \ 4059 } 4060 4061 def_access_sw_cpu(rc_acks); 4062 def_access_sw_cpu(rc_qacks); 4063 def_access_sw_cpu(rc_delayed_comp); 4064 4065 #define def_access_ibp_counter(cntr) \ 4066 static u64 access_ibp_##cntr(const struct cntr_entry *entry, \ 4067 void *context, int vl, int mode, u64 data) \ 4068 { \ 4069 struct hfi1_pportdata *ppd = (struct hfi1_pportdata *)context; \ 4070 \ 4071 if (vl != CNTR_INVALID_VL) \ 4072 return 0; \ 4073 \ 4074 return read_write_sw(ppd->dd, &ppd->ibport_data.rvp.n_ ##cntr, \ 4075 mode, data); \ 4076 } 4077 4078 def_access_ibp_counter(loop_pkts); 4079 def_access_ibp_counter(rc_resends); 4080 def_access_ibp_counter(rnr_naks); 4081 def_access_ibp_counter(other_naks); 4082 def_access_ibp_counter(rc_timeouts); 4083 def_access_ibp_counter(pkt_drops); 4084 def_access_ibp_counter(dmawait); 4085 def_access_ibp_counter(rc_seqnak); 4086 def_access_ibp_counter(rc_dupreq); 4087 def_access_ibp_counter(rdma_seq); 4088 def_access_ibp_counter(unaligned); 4089 def_access_ibp_counter(seq_naks); 4090 def_access_ibp_counter(rc_crwaits); 4091 4092 static struct cntr_entry dev_cntrs[DEV_CNTR_LAST] = { 4093 [C_RCV_OVF] = RXE32_DEV_CNTR_ELEM(RcvOverflow, RCV_BUF_OVFL_CNT, CNTR_SYNTH), 4094 [C_RX_LEN_ERR] = RXE32_DEV_CNTR_ELEM(RxLenErr, RCV_LENGTH_ERR_CNT, CNTR_SYNTH), 4095 [C_RX_SHORT_ERR] = RXE32_DEV_CNTR_ELEM(RxShrErr, RCV_SHORT_ERR_CNT, CNTR_SYNTH), 4096 [C_RX_ICRC_ERR] = RXE32_DEV_CNTR_ELEM(RxICrcErr, RCV_ICRC_ERR_CNT, CNTR_SYNTH), 4097 [C_RX_EBP] = RXE32_DEV_CNTR_ELEM(RxEbpCnt, RCV_EBP_CNT, CNTR_SYNTH), 4098 [C_RX_TID_FULL] = RXE32_DEV_CNTR_ELEM(RxTIDFullEr, RCV_TID_FULL_ERR_CNT, 4099 CNTR_NORMAL), 4100 [C_RX_TID_INVALID] = RXE32_DEV_CNTR_ELEM(RxTIDInvalid, RCV_TID_VALID_ERR_CNT, 4101 CNTR_NORMAL), 4102 [C_RX_TID_FLGMS] = RXE32_DEV_CNTR_ELEM(RxTidFLGMs, 4103 RCV_TID_FLOW_GEN_MISMATCH_CNT, 4104 CNTR_NORMAL), 4105 [C_RX_CTX_EGRS] = RXE32_DEV_CNTR_ELEM(RxCtxEgrS, RCV_CONTEXT_EGR_STALL, 4106 CNTR_NORMAL), 4107 [C_RCV_TID_FLSMS] = RXE32_DEV_CNTR_ELEM(RxTidFLSMs, 4108 RCV_TID_FLOW_SEQ_MISMATCH_CNT, CNTR_NORMAL), 4109 [C_CCE_PCI_CR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciCrSt, 4110 CCE_PCIE_POSTED_CRDT_STALL_CNT, CNTR_NORMAL), 4111 [C_CCE_PCI_TR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePciTrSt, CCE_PCIE_TRGT_STALL_CNT, 4112 CNTR_NORMAL), 4113 [C_CCE_PIO_WR_ST] = CCE_PERF_DEV_CNTR_ELEM(CcePioWrSt, CCE_PIO_WR_STALL_CNT, 4114 CNTR_NORMAL), 4115 [C_CCE_ERR_INT] = CCE_INT_DEV_CNTR_ELEM(CceErrInt, CCE_ERR_INT_CNT, 4116 CNTR_NORMAL), 4117 [C_CCE_SDMA_INT] = CCE_INT_DEV_CNTR_ELEM(CceSdmaInt, CCE_SDMA_INT_CNT, 4118 CNTR_NORMAL), 4119 [C_CCE_MISC_INT] = CCE_INT_DEV_CNTR_ELEM(CceMiscInt, CCE_MISC_INT_CNT, 4120 CNTR_NORMAL), 4121 [C_CCE_RCV_AV_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvAvInt, CCE_RCV_AVAIL_INT_CNT, 4122 CNTR_NORMAL), 4123 [C_CCE_RCV_URG_INT] = CCE_INT_DEV_CNTR_ELEM(CceRcvUrgInt, 4124 CCE_RCV_URGENT_INT_CNT, CNTR_NORMAL), 4125 [C_CCE_SEND_CR_INT] = CCE_INT_DEV_CNTR_ELEM(CceSndCrInt, 4126 CCE_SEND_CREDIT_INT_CNT, CNTR_NORMAL), 4127 [C_DC_UNC_ERR] = DC_PERF_CNTR(DcUnctblErr, DCC_ERR_UNCORRECTABLE_CNT, 4128 CNTR_SYNTH), 4129 [C_DC_RCV_ERR] = CNTR_ELEM("DcRecvErr", DCC_ERR_PORTRCV_ERR_CNT, 0, CNTR_SYNTH, 4130 access_dc_rcv_err_cnt), 4131 [C_DC_FM_CFG_ERR] = DC_PERF_CNTR(DcFmCfgErr, DCC_ERR_FMCONFIG_ERR_CNT, 4132 CNTR_SYNTH), 4133 [C_DC_RMT_PHY_ERR] = DC_PERF_CNTR(DcRmtPhyErr, DCC_ERR_RCVREMOTE_PHY_ERR_CNT, 4134 CNTR_SYNTH), 4135 [C_DC_DROPPED_PKT] = DC_PERF_CNTR(DcDroppedPkt, DCC_ERR_DROPPED_PKT_CNT, 4136 CNTR_SYNTH), 4137 [C_DC_MC_XMIT_PKTS] = DC_PERF_CNTR(DcMcXmitPkts, 4138 DCC_PRF_PORT_XMIT_MULTICAST_CNT, CNTR_SYNTH), 4139 [C_DC_MC_RCV_PKTS] = DC_PERF_CNTR(DcMcRcvPkts, 4140 DCC_PRF_PORT_RCV_MULTICAST_PKT_CNT, 4141 CNTR_SYNTH), 4142 [C_DC_XMIT_CERR] = DC_PERF_CNTR(DcXmitCorr, 4143 DCC_PRF_PORT_XMIT_CORRECTABLE_CNT, CNTR_SYNTH), 4144 [C_DC_RCV_CERR] = DC_PERF_CNTR(DcRcvCorrCnt, DCC_PRF_PORT_RCV_CORRECTABLE_CNT, 4145 CNTR_SYNTH), 4146 [C_DC_RCV_FCC] = DC_PERF_CNTR(DcRxFCntl, DCC_PRF_RX_FLOW_CRTL_CNT, 4147 CNTR_SYNTH), 4148 [C_DC_XMIT_FCC] = DC_PERF_CNTR(DcXmitFCntl, DCC_PRF_TX_FLOW_CRTL_CNT, 4149 CNTR_SYNTH), 4150 [C_DC_XMIT_FLITS] = DC_PERF_CNTR(DcXmitFlits, DCC_PRF_PORT_XMIT_DATA_CNT, 4151 CNTR_SYNTH), 4152 [C_DC_RCV_FLITS] = DC_PERF_CNTR(DcRcvFlits, DCC_PRF_PORT_RCV_DATA_CNT, 4153 CNTR_SYNTH), 4154 [C_DC_XMIT_PKTS] = DC_PERF_CNTR(DcXmitPkts, DCC_PRF_PORT_XMIT_PKTS_CNT, 4155 CNTR_SYNTH), 4156 [C_DC_RCV_PKTS] = DC_PERF_CNTR(DcRcvPkts, DCC_PRF_PORT_RCV_PKTS_CNT, 4157 CNTR_SYNTH), 4158 [C_DC_RX_FLIT_VL] = DC_PERF_CNTR(DcRxFlitVl, DCC_PRF_PORT_VL_RCV_DATA_CNT, 4159 CNTR_SYNTH | CNTR_VL), 4160 [C_DC_RX_PKT_VL] = DC_PERF_CNTR(DcRxPktVl, DCC_PRF_PORT_VL_RCV_PKTS_CNT, 4161 CNTR_SYNTH | CNTR_VL), 4162 [C_DC_RCV_FCN] = DC_PERF_CNTR(DcRcvFcn, DCC_PRF_PORT_RCV_FECN_CNT, CNTR_SYNTH), 4163 [C_DC_RCV_FCN_VL] = DC_PERF_CNTR(DcRcvFcnVl, DCC_PRF_PORT_VL_RCV_FECN_CNT, 4164 CNTR_SYNTH | CNTR_VL), 4165 [C_DC_RCV_BCN] = DC_PERF_CNTR(DcRcvBcn, DCC_PRF_PORT_RCV_BECN_CNT, CNTR_SYNTH), 4166 [C_DC_RCV_BCN_VL] = DC_PERF_CNTR(DcRcvBcnVl, DCC_PRF_PORT_VL_RCV_BECN_CNT, 4167 CNTR_SYNTH | CNTR_VL), 4168 [C_DC_RCV_BBL] = DC_PERF_CNTR(DcRcvBbl, DCC_PRF_PORT_RCV_BUBBLE_CNT, 4169 CNTR_SYNTH), 4170 [C_DC_RCV_BBL_VL] = DC_PERF_CNTR(DcRcvBblVl, DCC_PRF_PORT_VL_RCV_BUBBLE_CNT, 4171 CNTR_SYNTH | CNTR_VL), 4172 [C_DC_MARK_FECN] = DC_PERF_CNTR(DcMarkFcn, DCC_PRF_PORT_MARK_FECN_CNT, 4173 CNTR_SYNTH), 4174 [C_DC_MARK_FECN_VL] = DC_PERF_CNTR(DcMarkFcnVl, DCC_PRF_PORT_VL_MARK_FECN_CNT, 4175 CNTR_SYNTH | CNTR_VL), 4176 [C_DC_TOTAL_CRC] = 4177 DC_PERF_CNTR_LCB(DcTotCrc, DC_LCB_ERR_INFO_TOTAL_CRC_ERR, 4178 CNTR_SYNTH), 4179 [C_DC_CRC_LN0] = DC_PERF_CNTR_LCB(DcCrcLn0, DC_LCB_ERR_INFO_CRC_ERR_LN0, 4180 CNTR_SYNTH), 4181 [C_DC_CRC_LN1] = DC_PERF_CNTR_LCB(DcCrcLn1, DC_LCB_ERR_INFO_CRC_ERR_LN1, 4182 CNTR_SYNTH), 4183 [C_DC_CRC_LN2] = DC_PERF_CNTR_LCB(DcCrcLn2, DC_LCB_ERR_INFO_CRC_ERR_LN2, 4184 CNTR_SYNTH), 4185 [C_DC_CRC_LN3] = DC_PERF_CNTR_LCB(DcCrcLn3, DC_LCB_ERR_INFO_CRC_ERR_LN3, 4186 CNTR_SYNTH), 4187 [C_DC_CRC_MULT_LN] = 4188 DC_PERF_CNTR_LCB(DcMultLn, DC_LCB_ERR_INFO_CRC_ERR_MULTI_LN, 4189 CNTR_SYNTH), 4190 [C_DC_TX_REPLAY] = DC_PERF_CNTR_LCB(DcTxReplay, DC_LCB_ERR_INFO_TX_REPLAY_CNT, 4191 CNTR_SYNTH), 4192 [C_DC_RX_REPLAY] = DC_PERF_CNTR_LCB(DcRxReplay, DC_LCB_ERR_INFO_RX_REPLAY_CNT, 4193 CNTR_SYNTH), 4194 [C_DC_SEQ_CRC_CNT] = 4195 DC_PERF_CNTR_LCB(DcLinkSeqCrc, DC_LCB_ERR_INFO_SEQ_CRC_CNT, 4196 CNTR_SYNTH), 4197 [C_DC_ESC0_ONLY_CNT] = 4198 DC_PERF_CNTR_LCB(DcEsc0, DC_LCB_ERR_INFO_ESCAPE_0_ONLY_CNT, 4199 CNTR_SYNTH), 4200 [C_DC_ESC0_PLUS1_CNT] = 4201 DC_PERF_CNTR_LCB(DcEsc1, DC_LCB_ERR_INFO_ESCAPE_0_PLUS1_CNT, 4202 CNTR_SYNTH), 4203 [C_DC_ESC0_PLUS2_CNT] = 4204 DC_PERF_CNTR_LCB(DcEsc0Plus2, DC_LCB_ERR_INFO_ESCAPE_0_PLUS2_CNT, 4205 CNTR_SYNTH), 4206 [C_DC_REINIT_FROM_PEER_CNT] = 4207 DC_PERF_CNTR_LCB(DcReinitPeer, DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT, 4208 CNTR_SYNTH), 4209 [C_DC_SBE_CNT] = DC_PERF_CNTR_LCB(DcSbe, DC_LCB_ERR_INFO_SBE_CNT, 4210 CNTR_SYNTH), 4211 [C_DC_MISC_FLG_CNT] = 4212 DC_PERF_CNTR_LCB(DcMiscFlg, DC_LCB_ERR_INFO_MISC_FLG_CNT, 4213 CNTR_SYNTH), 4214 [C_DC_PRF_GOOD_LTP_CNT] = 4215 DC_PERF_CNTR_LCB(DcGoodLTP, DC_LCB_PRF_GOOD_LTP_CNT, CNTR_SYNTH), 4216 [C_DC_PRF_ACCEPTED_LTP_CNT] = 4217 DC_PERF_CNTR_LCB(DcAccLTP, DC_LCB_PRF_ACCEPTED_LTP_CNT, 4218 CNTR_SYNTH), 4219 [C_DC_PRF_RX_FLIT_CNT] = 4220 DC_PERF_CNTR_LCB(DcPrfRxFlit, DC_LCB_PRF_RX_FLIT_CNT, CNTR_SYNTH), 4221 [C_DC_PRF_TX_FLIT_CNT] = 4222 DC_PERF_CNTR_LCB(DcPrfTxFlit, DC_LCB_PRF_TX_FLIT_CNT, CNTR_SYNTH), 4223 [C_DC_PRF_CLK_CNTR] = 4224 DC_PERF_CNTR_LCB(DcPrfClk, DC_LCB_PRF_CLK_CNTR, CNTR_SYNTH), 4225 [C_DC_PG_DBG_FLIT_CRDTS_CNT] = 4226 DC_PERF_CNTR_LCB(DcFltCrdts, DC_LCB_PG_DBG_FLIT_CRDTS_CNT, CNTR_SYNTH), 4227 [C_DC_PG_STS_PAUSE_COMPLETE_CNT] = 4228 DC_PERF_CNTR_LCB(DcPauseComp, DC_LCB_PG_STS_PAUSE_COMPLETE_CNT, 4229 CNTR_SYNTH), 4230 [C_DC_PG_STS_TX_SBE_CNT] = 4231 DC_PERF_CNTR_LCB(DcStsTxSbe, DC_LCB_PG_STS_TX_SBE_CNT, CNTR_SYNTH), 4232 [C_DC_PG_STS_TX_MBE_CNT] = 4233 DC_PERF_CNTR_LCB(DcStsTxMbe, DC_LCB_PG_STS_TX_MBE_CNT, 4234 CNTR_SYNTH), 4235 [C_SW_CPU_INTR] = CNTR_ELEM("Intr", 0, 0, CNTR_NORMAL, 4236 access_sw_cpu_intr), 4237 [C_SW_CPU_RCV_LIM] = CNTR_ELEM("RcvLimit", 0, 0, CNTR_NORMAL, 4238 access_sw_cpu_rcv_limit), 4239 [C_SW_CTX0_SEQ_DROP] = CNTR_ELEM("SeqDrop0", 0, 0, CNTR_NORMAL, 4240 access_sw_ctx0_seq_drop), 4241 [C_SW_VTX_WAIT] = CNTR_ELEM("vTxWait", 0, 0, CNTR_NORMAL, 4242 access_sw_vtx_wait), 4243 [C_SW_PIO_WAIT] = CNTR_ELEM("PioWait", 0, 0, CNTR_NORMAL, 4244 access_sw_pio_wait), 4245 [C_SW_PIO_DRAIN] = CNTR_ELEM("PioDrain", 0, 0, CNTR_NORMAL, 4246 access_sw_pio_drain), 4247 [C_SW_KMEM_WAIT] = CNTR_ELEM("KmemWait", 0, 0, CNTR_NORMAL, 4248 access_sw_kmem_wait), 4249 [C_SW_TID_WAIT] = CNTR_ELEM("TidWait", 0, 0, CNTR_NORMAL, 4250 hfi1_access_sw_tid_wait), 4251 [C_SW_SEND_SCHED] = CNTR_ELEM("SendSched", 0, 0, CNTR_NORMAL, 4252 access_sw_send_schedule), 4253 [C_SDMA_DESC_FETCHED_CNT] = CNTR_ELEM("SDEDscFdCn", 4254 SEND_DMA_DESC_FETCHED_CNT, 0, 4255 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4256 dev_access_u32_csr), 4257 [C_SDMA_INT_CNT] = CNTR_ELEM("SDMAInt", 0, 0, 4258 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4259 access_sde_int_cnt), 4260 [C_SDMA_ERR_CNT] = CNTR_ELEM("SDMAErrCt", 0, 0, 4261 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4262 access_sde_err_cnt), 4263 [C_SDMA_IDLE_INT_CNT] = CNTR_ELEM("SDMAIdInt", 0, 0, 4264 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4265 access_sde_idle_int_cnt), 4266 [C_SDMA_PROGRESS_INT_CNT] = CNTR_ELEM("SDMAPrIntCn", 0, 0, 4267 CNTR_NORMAL | CNTR_32BIT | CNTR_SDMA, 4268 access_sde_progress_int_cnt), 4269 /* MISC_ERR_STATUS */ 4270 [C_MISC_PLL_LOCK_FAIL_ERR] = CNTR_ELEM("MISC_PLL_LOCK_FAIL_ERR", 0, 0, 4271 CNTR_NORMAL, 4272 access_misc_pll_lock_fail_err_cnt), 4273 [C_MISC_MBIST_FAIL_ERR] = CNTR_ELEM("MISC_MBIST_FAIL_ERR", 0, 0, 4274 CNTR_NORMAL, 4275 access_misc_mbist_fail_err_cnt), 4276 [C_MISC_INVALID_EEP_CMD_ERR] = CNTR_ELEM("MISC_INVALID_EEP_CMD_ERR", 0, 0, 4277 CNTR_NORMAL, 4278 access_misc_invalid_eep_cmd_err_cnt), 4279 [C_MISC_EFUSE_DONE_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_DONE_PARITY_ERR", 0, 0, 4280 CNTR_NORMAL, 4281 access_misc_efuse_done_parity_err_cnt), 4282 [C_MISC_EFUSE_WRITE_ERR] = CNTR_ELEM("MISC_EFUSE_WRITE_ERR", 0, 0, 4283 CNTR_NORMAL, 4284 access_misc_efuse_write_err_cnt), 4285 [C_MISC_EFUSE_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_EFUSE_READ_BAD_ADDR_ERR", 0, 4286 0, CNTR_NORMAL, 4287 access_misc_efuse_read_bad_addr_err_cnt), 4288 [C_MISC_EFUSE_CSR_PARITY_ERR] = CNTR_ELEM("MISC_EFUSE_CSR_PARITY_ERR", 0, 0, 4289 CNTR_NORMAL, 4290 access_misc_efuse_csr_parity_err_cnt), 4291 [C_MISC_FW_AUTH_FAILED_ERR] = CNTR_ELEM("MISC_FW_AUTH_FAILED_ERR", 0, 0, 4292 CNTR_NORMAL, 4293 access_misc_fw_auth_failed_err_cnt), 4294 [C_MISC_KEY_MISMATCH_ERR] = CNTR_ELEM("MISC_KEY_MISMATCH_ERR", 0, 0, 4295 CNTR_NORMAL, 4296 access_misc_key_mismatch_err_cnt), 4297 [C_MISC_SBUS_WRITE_FAILED_ERR] = CNTR_ELEM("MISC_SBUS_WRITE_FAILED_ERR", 0, 0, 4298 CNTR_NORMAL, 4299 access_misc_sbus_write_failed_err_cnt), 4300 [C_MISC_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_WRITE_BAD_ADDR_ERR", 0, 0, 4301 CNTR_NORMAL, 4302 access_misc_csr_write_bad_addr_err_cnt), 4303 [C_MISC_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("MISC_CSR_READ_BAD_ADDR_ERR", 0, 0, 4304 CNTR_NORMAL, 4305 access_misc_csr_read_bad_addr_err_cnt), 4306 [C_MISC_CSR_PARITY_ERR] = CNTR_ELEM("MISC_CSR_PARITY_ERR", 0, 0, 4307 CNTR_NORMAL, 4308 access_misc_csr_parity_err_cnt), 4309 /* CceErrStatus */ 4310 [C_CCE_ERR_STATUS_AGGREGATED_CNT] = CNTR_ELEM("CceErrStatusAggregatedCnt", 0, 0, 4311 CNTR_NORMAL, 4312 access_sw_cce_err_status_aggregated_cnt), 4313 [C_CCE_MSIX_CSR_PARITY_ERR] = CNTR_ELEM("CceMsixCsrParityErr", 0, 0, 4314 CNTR_NORMAL, 4315 access_cce_msix_csr_parity_err_cnt), 4316 [C_CCE_INT_MAP_UNC_ERR] = CNTR_ELEM("CceIntMapUncErr", 0, 0, 4317 CNTR_NORMAL, 4318 access_cce_int_map_unc_err_cnt), 4319 [C_CCE_INT_MAP_COR_ERR] = CNTR_ELEM("CceIntMapCorErr", 0, 0, 4320 CNTR_NORMAL, 4321 access_cce_int_map_cor_err_cnt), 4322 [C_CCE_MSIX_TABLE_UNC_ERR] = CNTR_ELEM("CceMsixTableUncErr", 0, 0, 4323 CNTR_NORMAL, 4324 access_cce_msix_table_unc_err_cnt), 4325 [C_CCE_MSIX_TABLE_COR_ERR] = CNTR_ELEM("CceMsixTableCorErr", 0, 0, 4326 CNTR_NORMAL, 4327 access_cce_msix_table_cor_err_cnt), 4328 [C_CCE_RXDMA_CONV_FIFO_PARITY_ERR] = CNTR_ELEM("CceRxdmaConvFifoParityErr", 0, 4329 0, CNTR_NORMAL, 4330 access_cce_rxdma_conv_fifo_parity_err_cnt), 4331 [C_CCE_RCPL_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceRcplAsyncFifoParityErr", 0, 4332 0, CNTR_NORMAL, 4333 access_cce_rcpl_async_fifo_parity_err_cnt), 4334 [C_CCE_SEG_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceSegWriteBadAddrErr", 0, 0, 4335 CNTR_NORMAL, 4336 access_cce_seg_write_bad_addr_err_cnt), 4337 [C_CCE_SEG_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceSegReadBadAddrErr", 0, 0, 4338 CNTR_NORMAL, 4339 access_cce_seg_read_bad_addr_err_cnt), 4340 [C_LA_TRIGGERED] = CNTR_ELEM("Cce LATriggered", 0, 0, 4341 CNTR_NORMAL, 4342 access_la_triggered_cnt), 4343 [C_CCE_TRGT_CPL_TIMEOUT_ERR] = CNTR_ELEM("CceTrgtCplTimeoutErr", 0, 0, 4344 CNTR_NORMAL, 4345 access_cce_trgt_cpl_timeout_err_cnt), 4346 [C_PCIC_RECEIVE_PARITY_ERR] = CNTR_ELEM("PcicReceiveParityErr", 0, 0, 4347 CNTR_NORMAL, 4348 access_pcic_receive_parity_err_cnt), 4349 [C_PCIC_TRANSMIT_BACK_PARITY_ERR] = CNTR_ELEM("PcicTransmitBackParityErr", 0, 0, 4350 CNTR_NORMAL, 4351 access_pcic_transmit_back_parity_err_cnt), 4352 [C_PCIC_TRANSMIT_FRONT_PARITY_ERR] = CNTR_ELEM("PcicTransmitFrontParityErr", 0, 4353 0, CNTR_NORMAL, 4354 access_pcic_transmit_front_parity_err_cnt), 4355 [C_PCIC_CPL_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicCplDatQUncErr", 0, 0, 4356 CNTR_NORMAL, 4357 access_pcic_cpl_dat_q_unc_err_cnt), 4358 [C_PCIC_CPL_HD_Q_UNC_ERR] = CNTR_ELEM("PcicCplHdQUncErr", 0, 0, 4359 CNTR_NORMAL, 4360 access_pcic_cpl_hd_q_unc_err_cnt), 4361 [C_PCIC_POST_DAT_Q_UNC_ERR] = CNTR_ELEM("PcicPostDatQUncErr", 0, 0, 4362 CNTR_NORMAL, 4363 access_pcic_post_dat_q_unc_err_cnt), 4364 [C_PCIC_POST_HD_Q_UNC_ERR] = CNTR_ELEM("PcicPostHdQUncErr", 0, 0, 4365 CNTR_NORMAL, 4366 access_pcic_post_hd_q_unc_err_cnt), 4367 [C_PCIC_RETRY_SOT_MEM_UNC_ERR] = CNTR_ELEM("PcicRetrySotMemUncErr", 0, 0, 4368 CNTR_NORMAL, 4369 access_pcic_retry_sot_mem_unc_err_cnt), 4370 [C_PCIC_RETRY_MEM_UNC_ERR] = CNTR_ELEM("PcicRetryMemUncErr", 0, 0, 4371 CNTR_NORMAL, 4372 access_pcic_retry_mem_unc_err), 4373 [C_PCIC_N_POST_DAT_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostDatQParityErr", 0, 0, 4374 CNTR_NORMAL, 4375 access_pcic_n_post_dat_q_parity_err_cnt), 4376 [C_PCIC_N_POST_H_Q_PARITY_ERR] = CNTR_ELEM("PcicNPostHQParityErr", 0, 0, 4377 CNTR_NORMAL, 4378 access_pcic_n_post_h_q_parity_err_cnt), 4379 [C_PCIC_CPL_DAT_Q_COR_ERR] = CNTR_ELEM("PcicCplDatQCorErr", 0, 0, 4380 CNTR_NORMAL, 4381 access_pcic_cpl_dat_q_cor_err_cnt), 4382 [C_PCIC_CPL_HD_Q_COR_ERR] = CNTR_ELEM("PcicCplHdQCorErr", 0, 0, 4383 CNTR_NORMAL, 4384 access_pcic_cpl_hd_q_cor_err_cnt), 4385 [C_PCIC_POST_DAT_Q_COR_ERR] = CNTR_ELEM("PcicPostDatQCorErr", 0, 0, 4386 CNTR_NORMAL, 4387 access_pcic_post_dat_q_cor_err_cnt), 4388 [C_PCIC_POST_HD_Q_COR_ERR] = CNTR_ELEM("PcicPostHdQCorErr", 0, 0, 4389 CNTR_NORMAL, 4390 access_pcic_post_hd_q_cor_err_cnt), 4391 [C_PCIC_RETRY_SOT_MEM_COR_ERR] = CNTR_ELEM("PcicRetrySotMemCorErr", 0, 0, 4392 CNTR_NORMAL, 4393 access_pcic_retry_sot_mem_cor_err_cnt), 4394 [C_PCIC_RETRY_MEM_COR_ERR] = CNTR_ELEM("PcicRetryMemCorErr", 0, 0, 4395 CNTR_NORMAL, 4396 access_pcic_retry_mem_cor_err_cnt), 4397 [C_CCE_CLI1_ASYNC_FIFO_DBG_PARITY_ERR] = CNTR_ELEM( 4398 "CceCli1AsyncFifoDbgParityError", 0, 0, 4399 CNTR_NORMAL, 4400 access_cce_cli1_async_fifo_dbg_parity_err_cnt), 4401 [C_CCE_CLI1_ASYNC_FIFO_RXDMA_PARITY_ERR] = CNTR_ELEM( 4402 "CceCli1AsyncFifoRxdmaParityError", 0, 0, 4403 CNTR_NORMAL, 4404 access_cce_cli1_async_fifo_rxdma_parity_err_cnt 4405 ), 4406 [C_CCE_CLI1_ASYNC_FIFO_SDMA_HD_PARITY_ERR] = CNTR_ELEM( 4407 "CceCli1AsyncFifoSdmaHdParityErr", 0, 0, 4408 CNTR_NORMAL, 4409 access_cce_cli1_async_fifo_sdma_hd_parity_err_cnt), 4410 [C_CCE_CLI1_ASYNC_FIFO_PIO_CRDT_PARITY_ERR] = CNTR_ELEM( 4411 "CceCli1AsyncFifoPioCrdtParityErr", 0, 0, 4412 CNTR_NORMAL, 4413 access_cce_cl1_async_fifo_pio_crdt_parity_err_cnt), 4414 [C_CCE_CLI2_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceCli2AsyncFifoParityErr", 0, 4415 0, CNTR_NORMAL, 4416 access_cce_cli2_async_fifo_parity_err_cnt), 4417 [C_CCE_CSR_CFG_BUS_PARITY_ERR] = CNTR_ELEM("CceCsrCfgBusParityErr", 0, 0, 4418 CNTR_NORMAL, 4419 access_cce_csr_cfg_bus_parity_err_cnt), 4420 [C_CCE_CLI0_ASYNC_FIFO_PARTIY_ERR] = CNTR_ELEM("CceCli0AsyncFifoParityErr", 0, 4421 0, CNTR_NORMAL, 4422 access_cce_cli0_async_fifo_parity_err_cnt), 4423 [C_CCE_RSPD_DATA_PARITY_ERR] = CNTR_ELEM("CceRspdDataParityErr", 0, 0, 4424 CNTR_NORMAL, 4425 access_cce_rspd_data_parity_err_cnt), 4426 [C_CCE_TRGT_ACCESS_ERR] = CNTR_ELEM("CceTrgtAccessErr", 0, 0, 4427 CNTR_NORMAL, 4428 access_cce_trgt_access_err_cnt), 4429 [C_CCE_TRGT_ASYNC_FIFO_PARITY_ERR] = CNTR_ELEM("CceTrgtAsyncFifoParityErr", 0, 4430 0, CNTR_NORMAL, 4431 access_cce_trgt_async_fifo_parity_err_cnt), 4432 [C_CCE_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrWriteBadAddrErr", 0, 0, 4433 CNTR_NORMAL, 4434 access_cce_csr_write_bad_addr_err_cnt), 4435 [C_CCE_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("CceCsrReadBadAddrErr", 0, 0, 4436 CNTR_NORMAL, 4437 access_cce_csr_read_bad_addr_err_cnt), 4438 [C_CCE_CSR_PARITY_ERR] = CNTR_ELEM("CceCsrParityErr", 0, 0, 4439 CNTR_NORMAL, 4440 access_ccs_csr_parity_err_cnt), 4441 4442 /* RcvErrStatus */ 4443 [C_RX_CSR_PARITY_ERR] = CNTR_ELEM("RxCsrParityErr", 0, 0, 4444 CNTR_NORMAL, 4445 access_rx_csr_parity_err_cnt), 4446 [C_RX_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrWriteBadAddrErr", 0, 0, 4447 CNTR_NORMAL, 4448 access_rx_csr_write_bad_addr_err_cnt), 4449 [C_RX_CSR_READ_BAD_ADDR_ERR] = CNTR_ELEM("RxCsrReadBadAddrErr", 0, 0, 4450 CNTR_NORMAL, 4451 access_rx_csr_read_bad_addr_err_cnt), 4452 [C_RX_DMA_CSR_UNC_ERR] = CNTR_ELEM("RxDmaCsrUncErr", 0, 0, 4453 CNTR_NORMAL, 4454 access_rx_dma_csr_unc_err_cnt), 4455 [C_RX_DMA_DQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaDqFsmEncodingErr", 0, 0, 4456 CNTR_NORMAL, 4457 access_rx_dma_dq_fsm_encoding_err_cnt), 4458 [C_RX_DMA_EQ_FSM_ENCODING_ERR] = CNTR_ELEM("RxDmaEqFsmEncodingErr", 0, 0, 4459 CNTR_NORMAL, 4460 access_rx_dma_eq_fsm_encoding_err_cnt), 4461 [C_RX_DMA_CSR_PARITY_ERR] = CNTR_ELEM("RxDmaCsrParityErr", 0, 0, 4462 CNTR_NORMAL, 4463 access_rx_dma_csr_parity_err_cnt), 4464 [C_RX_RBUF_DATA_COR_ERR] = CNTR_ELEM("RxRbufDataCorErr", 0, 0, 4465 CNTR_NORMAL, 4466 access_rx_rbuf_data_cor_err_cnt), 4467 [C_RX_RBUF_DATA_UNC_ERR] = CNTR_ELEM("RxRbufDataUncErr", 0, 0, 4468 CNTR_NORMAL, 4469 access_rx_rbuf_data_unc_err_cnt), 4470 [C_RX_DMA_DATA_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaDataFifoRdCorErr", 0, 0, 4471 CNTR_NORMAL, 4472 access_rx_dma_data_fifo_rd_cor_err_cnt), 4473 [C_RX_DMA_DATA_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaDataFifoRdUncErr", 0, 0, 4474 CNTR_NORMAL, 4475 access_rx_dma_data_fifo_rd_unc_err_cnt), 4476 [C_RX_DMA_HDR_FIFO_RD_COR_ERR] = CNTR_ELEM("RxDmaHdrFifoRdCorErr", 0, 0, 4477 CNTR_NORMAL, 4478 access_rx_dma_hdr_fifo_rd_cor_err_cnt), 4479 [C_RX_DMA_HDR_FIFO_RD_UNC_ERR] = CNTR_ELEM("RxDmaHdrFifoRdUncErr", 0, 0, 4480 CNTR_NORMAL, 4481 access_rx_dma_hdr_fifo_rd_unc_err_cnt), 4482 [C_RX_RBUF_DESC_PART2_COR_ERR] = CNTR_ELEM("RxRbufDescPart2CorErr", 0, 0, 4483 CNTR_NORMAL, 4484 access_rx_rbuf_desc_part2_cor_err_cnt), 4485 [C_RX_RBUF_DESC_PART2_UNC_ERR] = CNTR_ELEM("RxRbufDescPart2UncErr", 0, 0, 4486 CNTR_NORMAL, 4487 access_rx_rbuf_desc_part2_unc_err_cnt), 4488 [C_RX_RBUF_DESC_PART1_COR_ERR] = CNTR_ELEM("RxRbufDescPart1CorErr", 0, 0, 4489 CNTR_NORMAL, 4490 access_rx_rbuf_desc_part1_cor_err_cnt), 4491 [C_RX_RBUF_DESC_PART1_UNC_ERR] = CNTR_ELEM("RxRbufDescPart1UncErr", 0, 0, 4492 CNTR_NORMAL, 4493 access_rx_rbuf_desc_part1_unc_err_cnt), 4494 [C_RX_HQ_INTR_FSM_ERR] = CNTR_ELEM("RxHqIntrFsmErr", 0, 0, 4495 CNTR_NORMAL, 4496 access_rx_hq_intr_fsm_err_cnt), 4497 [C_RX_HQ_INTR_CSR_PARITY_ERR] = CNTR_ELEM("RxHqIntrCsrParityErr", 0, 0, 4498 CNTR_NORMAL, 4499 access_rx_hq_intr_csr_parity_err_cnt), 4500 [C_RX_LOOKUP_CSR_PARITY_ERR] = CNTR_ELEM("RxLookupCsrParityErr", 0, 0, 4501 CNTR_NORMAL, 4502 access_rx_lookup_csr_parity_err_cnt), 4503 [C_RX_LOOKUP_RCV_ARRAY_COR_ERR] = CNTR_ELEM("RxLookupRcvArrayCorErr", 0, 0, 4504 CNTR_NORMAL, 4505 access_rx_lookup_rcv_array_cor_err_cnt), 4506 [C_RX_LOOKUP_RCV_ARRAY_UNC_ERR] = CNTR_ELEM("RxLookupRcvArrayUncErr", 0, 0, 4507 CNTR_NORMAL, 4508 access_rx_lookup_rcv_array_unc_err_cnt), 4509 [C_RX_LOOKUP_DES_PART2_PARITY_ERR] = CNTR_ELEM("RxLookupDesPart2ParityErr", 0, 4510 0, CNTR_NORMAL, 4511 access_rx_lookup_des_part2_parity_err_cnt), 4512 [C_RX_LOOKUP_DES_PART1_UNC_COR_ERR] = CNTR_ELEM("RxLookupDesPart1UncCorErr", 0, 4513 0, CNTR_NORMAL, 4514 access_rx_lookup_des_part1_unc_cor_err_cnt), 4515 [C_RX_LOOKUP_DES_PART1_UNC_ERR] = CNTR_ELEM("RxLookupDesPart1UncErr", 0, 0, 4516 CNTR_NORMAL, 4517 access_rx_lookup_des_part1_unc_err_cnt), 4518 [C_RX_RBUF_NEXT_FREE_BUF_COR_ERR] = CNTR_ELEM("RxRbufNextFreeBufCorErr", 0, 0, 4519 CNTR_NORMAL, 4520 access_rx_rbuf_next_free_buf_cor_err_cnt), 4521 [C_RX_RBUF_NEXT_FREE_BUF_UNC_ERR] = CNTR_ELEM("RxRbufNextFreeBufUncErr", 0, 0, 4522 CNTR_NORMAL, 4523 access_rx_rbuf_next_free_buf_unc_err_cnt), 4524 [C_RX_RBUF_FL_INIT_WR_ADDR_PARITY_ERR] = CNTR_ELEM( 4525 "RxRbufFlInitWrAddrParityErr", 0, 0, 4526 CNTR_NORMAL, 4527 access_rbuf_fl_init_wr_addr_parity_err_cnt), 4528 [C_RX_RBUF_FL_INITDONE_PARITY_ERR] = CNTR_ELEM("RxRbufFlInitdoneParityErr", 0, 4529 0, CNTR_NORMAL, 4530 access_rx_rbuf_fl_initdone_parity_err_cnt), 4531 [C_RX_RBUF_FL_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlWrAddrParityErr", 0, 4532 0, CNTR_NORMAL, 4533 access_rx_rbuf_fl_write_addr_parity_err_cnt), 4534 [C_RX_RBUF_FL_RD_ADDR_PARITY_ERR] = CNTR_ELEM("RxRbufFlRdAddrParityErr", 0, 0, 4535 CNTR_NORMAL, 4536 access_rx_rbuf_fl_rd_addr_parity_err_cnt), 4537 [C_RX_RBUF_EMPTY_ERR] = CNTR_ELEM("RxRbufEmptyErr", 0, 0, 4538 CNTR_NORMAL, 4539 access_rx_rbuf_empty_err_cnt), 4540 [C_RX_RBUF_FULL_ERR] = CNTR_ELEM("RxRbufFullErr", 0, 0, 4541 CNTR_NORMAL, 4542 access_rx_rbuf_full_err_cnt), 4543 [C_RX_RBUF_BAD_LOOKUP_ERR] = CNTR_ELEM("RxRBufBadLookupErr", 0, 0, 4544 CNTR_NORMAL, 4545 access_rbuf_bad_lookup_err_cnt), 4546 [C_RX_RBUF_CTX_ID_PARITY_ERR] = CNTR_ELEM("RxRbufCtxIdParityErr", 0, 0, 4547 CNTR_NORMAL, 4548 access_rbuf_ctx_id_parity_err_cnt), 4549 [C_RX_RBUF_CSR_QEOPDW_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEOPDWParityErr", 0, 0, 4550 CNTR_NORMAL, 4551 access_rbuf_csr_qeopdw_parity_err_cnt), 4552 [C_RX_RBUF_CSR_Q_NUM_OF_PKT_PARITY_ERR] = CNTR_ELEM( 4553 "RxRbufCsrQNumOfPktParityErr", 0, 0, 4554 CNTR_NORMAL, 4555 access_rx_rbuf_csr_q_num_of_pkt_parity_err_cnt), 4556 [C_RX_RBUF_CSR_Q_T1_PTR_PARITY_ERR] = CNTR_ELEM( 4557 "RxRbufCsrQTlPtrParityErr", 0, 0, 4558 CNTR_NORMAL, 4559 access_rx_rbuf_csr_q_t1_ptr_parity_err_cnt), 4560 [C_RX_RBUF_CSR_Q_HD_PTR_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQHdPtrParityErr", 0, 4561 0, CNTR_NORMAL, 4562 access_rx_rbuf_csr_q_hd_ptr_parity_err_cnt), 4563 [C_RX_RBUF_CSR_Q_VLD_BIT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQVldBitParityErr", 0, 4564 0, CNTR_NORMAL, 4565 access_rx_rbuf_csr_q_vld_bit_parity_err_cnt), 4566 [C_RX_RBUF_CSR_Q_NEXT_BUF_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQNextBufParityErr", 4567 0, 0, CNTR_NORMAL, 4568 access_rx_rbuf_csr_q_next_buf_parity_err_cnt), 4569 [C_RX_RBUF_CSR_Q_ENT_CNT_PARITY_ERR] = CNTR_ELEM("RxRbufCsrQEntCntParityErr", 0, 4570 0, CNTR_NORMAL, 4571 access_rx_rbuf_csr_q_ent_cnt_parity_err_cnt), 4572 [C_RX_RBUF_CSR_Q_HEAD_BUF_NUM_PARITY_ERR] = CNTR_ELEM( 4573 "RxRbufCsrQHeadBufNumParityErr", 0, 0, 4574 CNTR_NORMAL, 4575 access_rx_rbuf_csr_q_head_buf_num_parity_err_cnt), 4576 [C_RX_RBUF_BLOCK_LIST_READ_COR_ERR] = CNTR_ELEM("RxRbufBlockListReadCorErr", 0, 4577 0, CNTR_NORMAL, 4578 access_rx_rbuf_block_list_read_cor_err_cnt), 4579 [C_RX_RBUF_BLOCK_LIST_READ_UNC_ERR] = CNTR_ELEM("RxRbufBlockListReadUncErr", 0, 4580 0, CNTR_NORMAL, 4581 access_rx_rbuf_block_list_read_unc_err_cnt), 4582 [C_RX_RBUF_LOOKUP_DES_COR_ERR] = CNTR_ELEM("RxRbufLookupDesCorErr", 0, 0, 4583 CNTR_NORMAL, 4584 access_rx_rbuf_lookup_des_cor_err_cnt), 4585 [C_RX_RBUF_LOOKUP_DES_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesUncErr", 0, 0, 4586 CNTR_NORMAL, 4587 access_rx_rbuf_lookup_des_unc_err_cnt), 4588 [C_RX_RBUF_LOOKUP_DES_REG_UNC_COR_ERR] = CNTR_ELEM( 4589 "RxRbufLookupDesRegUncCorErr", 0, 0, 4590 CNTR_NORMAL, 4591 access_rx_rbuf_lookup_des_reg_unc_cor_err_cnt), 4592 [C_RX_RBUF_LOOKUP_DES_REG_UNC_ERR] = CNTR_ELEM("RxRbufLookupDesRegUncErr", 0, 0, 4593 CNTR_NORMAL, 4594 access_rx_rbuf_lookup_des_reg_unc_err_cnt), 4595 [C_RX_RBUF_FREE_LIST_COR_ERR] = CNTR_ELEM("RxRbufFreeListCorErr", 0, 0, 4596 CNTR_NORMAL, 4597 access_rx_rbuf_free_list_cor_err_cnt), 4598 [C_RX_RBUF_FREE_LIST_UNC_ERR] = CNTR_ELEM("RxRbufFreeListUncErr", 0, 0, 4599 CNTR_NORMAL, 4600 access_rx_rbuf_free_list_unc_err_cnt), 4601 [C_RX_RCV_FSM_ENCODING_ERR] = CNTR_ELEM("RxRcvFsmEncodingErr", 0, 0, 4602 CNTR_NORMAL, 4603 access_rx_rcv_fsm_encoding_err_cnt), 4604 [C_RX_DMA_FLAG_COR_ERR] = CNTR_ELEM("RxDmaFlagCorErr", 0, 0, 4605 CNTR_NORMAL, 4606 access_rx_dma_flag_cor_err_cnt), 4607 [C_RX_DMA_FLAG_UNC_ERR] = CNTR_ELEM("RxDmaFlagUncErr", 0, 0, 4608 CNTR_NORMAL, 4609 access_rx_dma_flag_unc_err_cnt), 4610 [C_RX_DC_SOP_EOP_PARITY_ERR] = CNTR_ELEM("RxDcSopEopParityErr", 0, 0, 4611 CNTR_NORMAL, 4612 access_rx_dc_sop_eop_parity_err_cnt), 4613 [C_RX_RCV_CSR_PARITY_ERR] = CNTR_ELEM("RxRcvCsrParityErr", 0, 0, 4614 CNTR_NORMAL, 4615 access_rx_rcv_csr_parity_err_cnt), 4616 [C_RX_RCV_QP_MAP_TABLE_COR_ERR] = CNTR_ELEM("RxRcvQpMapTableCorErr", 0, 0, 4617 CNTR_NORMAL, 4618 access_rx_rcv_qp_map_table_cor_err_cnt), 4619 [C_RX_RCV_QP_MAP_TABLE_UNC_ERR] = CNTR_ELEM("RxRcvQpMapTableUncErr", 0, 0, 4620 CNTR_NORMAL, 4621 access_rx_rcv_qp_map_table_unc_err_cnt), 4622 [C_RX_RCV_DATA_COR_ERR] = CNTR_ELEM("RxRcvDataCorErr", 0, 0, 4623 CNTR_NORMAL, 4624 access_rx_rcv_data_cor_err_cnt), 4625 [C_RX_RCV_DATA_UNC_ERR] = CNTR_ELEM("RxRcvDataUncErr", 0, 0, 4626 CNTR_NORMAL, 4627 access_rx_rcv_data_unc_err_cnt), 4628 [C_RX_RCV_HDR_COR_ERR] = CNTR_ELEM("RxRcvHdrCorErr", 0, 0, 4629 CNTR_NORMAL, 4630 access_rx_rcv_hdr_cor_err_cnt), 4631 [C_RX_RCV_HDR_UNC_ERR] = CNTR_ELEM("RxRcvHdrUncErr", 0, 0, 4632 CNTR_NORMAL, 4633 access_rx_rcv_hdr_unc_err_cnt), 4634 [C_RX_DC_INTF_PARITY_ERR] = CNTR_ELEM("RxDcIntfParityErr", 0, 0, 4635 CNTR_NORMAL, 4636 access_rx_dc_intf_parity_err_cnt), 4637 [C_RX_DMA_CSR_COR_ERR] = CNTR_ELEM("RxDmaCsrCorErr", 0, 0, 4638 CNTR_NORMAL, 4639 access_rx_dma_csr_cor_err_cnt), 4640 /* SendPioErrStatus */ 4641 [C_PIO_PEC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPecSopHeadParityErr", 0, 0, 4642 CNTR_NORMAL, 4643 access_pio_pec_sop_head_parity_err_cnt), 4644 [C_PIO_PCC_SOP_HEAD_PARITY_ERR] = CNTR_ELEM("PioPccSopHeadParityErr", 0, 0, 4645 CNTR_NORMAL, 4646 access_pio_pcc_sop_head_parity_err_cnt), 4647 [C_PIO_LAST_RETURNED_CNT_PARITY_ERR] = CNTR_ELEM("PioLastReturnedCntParityErr", 4648 0, 0, CNTR_NORMAL, 4649 access_pio_last_returned_cnt_parity_err_cnt), 4650 [C_PIO_CURRENT_FREE_CNT_PARITY_ERR] = CNTR_ELEM("PioCurrentFreeCntParityErr", 0, 4651 0, CNTR_NORMAL, 4652 access_pio_current_free_cnt_parity_err_cnt), 4653 [C_PIO_RSVD_31_ERR] = CNTR_ELEM("Pio Reserved 31", 0, 0, 4654 CNTR_NORMAL, 4655 access_pio_reserved_31_err_cnt), 4656 [C_PIO_RSVD_30_ERR] = CNTR_ELEM("Pio Reserved 30", 0, 0, 4657 CNTR_NORMAL, 4658 access_pio_reserved_30_err_cnt), 4659 [C_PIO_PPMC_SOP_LEN_ERR] = CNTR_ELEM("PioPpmcSopLenErr", 0, 0, 4660 CNTR_NORMAL, 4661 access_pio_ppmc_sop_len_err_cnt), 4662 [C_PIO_PPMC_BQC_MEM_PARITY_ERR] = CNTR_ELEM("PioPpmcBqcMemParityErr", 0, 0, 4663 CNTR_NORMAL, 4664 access_pio_ppmc_bqc_mem_parity_err_cnt), 4665 [C_PIO_VL_FIFO_PARITY_ERR] = CNTR_ELEM("PioVlFifoParityErr", 0, 0, 4666 CNTR_NORMAL, 4667 access_pio_vl_fifo_parity_err_cnt), 4668 [C_PIO_VLF_SOP_PARITY_ERR] = CNTR_ELEM("PioVlfSopParityErr", 0, 0, 4669 CNTR_NORMAL, 4670 access_pio_vlf_sop_parity_err_cnt), 4671 [C_PIO_VLF_V1_LEN_PARITY_ERR] = CNTR_ELEM("PioVlfVlLenParityErr", 0, 0, 4672 CNTR_NORMAL, 4673 access_pio_vlf_v1_len_parity_err_cnt), 4674 [C_PIO_BLOCK_QW_COUNT_PARITY_ERR] = CNTR_ELEM("PioBlockQwCountParityErr", 0, 0, 4675 CNTR_NORMAL, 4676 access_pio_block_qw_count_parity_err_cnt), 4677 [C_PIO_WRITE_QW_VALID_PARITY_ERR] = CNTR_ELEM("PioWriteQwValidParityErr", 0, 0, 4678 CNTR_NORMAL, 4679 access_pio_write_qw_valid_parity_err_cnt), 4680 [C_PIO_STATE_MACHINE_ERR] = CNTR_ELEM("PioStateMachineErr", 0, 0, 4681 CNTR_NORMAL, 4682 access_pio_state_machine_err_cnt), 4683 [C_PIO_WRITE_DATA_PARITY_ERR] = CNTR_ELEM("PioWriteDataParityErr", 0, 0, 4684 CNTR_NORMAL, 4685 access_pio_write_data_parity_err_cnt), 4686 [C_PIO_HOST_ADDR_MEM_COR_ERR] = CNTR_ELEM("PioHostAddrMemCorErr", 0, 0, 4687 CNTR_NORMAL, 4688 access_pio_host_addr_mem_cor_err_cnt), 4689 [C_PIO_HOST_ADDR_MEM_UNC_ERR] = CNTR_ELEM("PioHostAddrMemUncErr", 0, 0, 4690 CNTR_NORMAL, 4691 access_pio_host_addr_mem_unc_err_cnt), 4692 [C_PIO_PKT_EVICT_SM_OR_ARM_SM_ERR] = CNTR_ELEM("PioPktEvictSmOrArbSmErr", 0, 0, 4693 CNTR_NORMAL, 4694 access_pio_pkt_evict_sm_or_arb_sm_err_cnt), 4695 [C_PIO_INIT_SM_IN_ERR] = CNTR_ELEM("PioInitSmInErr", 0, 0, 4696 CNTR_NORMAL, 4697 access_pio_init_sm_in_err_cnt), 4698 [C_PIO_PPMC_PBL_FIFO_ERR] = CNTR_ELEM("PioPpmcPblFifoErr", 0, 0, 4699 CNTR_NORMAL, 4700 access_pio_ppmc_pbl_fifo_err_cnt), 4701 [C_PIO_CREDIT_RET_FIFO_PARITY_ERR] = CNTR_ELEM("PioCreditRetFifoParityErr", 0, 4702 0, CNTR_NORMAL, 4703 access_pio_credit_ret_fifo_parity_err_cnt), 4704 [C_PIO_V1_LEN_MEM_BANK1_COR_ERR] = CNTR_ELEM("PioVlLenMemBank1CorErr", 0, 0, 4705 CNTR_NORMAL, 4706 access_pio_v1_len_mem_bank1_cor_err_cnt), 4707 [C_PIO_V1_LEN_MEM_BANK0_COR_ERR] = CNTR_ELEM("PioVlLenMemBank0CorErr", 0, 0, 4708 CNTR_NORMAL, 4709 access_pio_v1_len_mem_bank0_cor_err_cnt), 4710 [C_PIO_V1_LEN_MEM_BANK1_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank1UncErr", 0, 0, 4711 CNTR_NORMAL, 4712 access_pio_v1_len_mem_bank1_unc_err_cnt), 4713 [C_PIO_V1_LEN_MEM_BANK0_UNC_ERR] = CNTR_ELEM("PioVlLenMemBank0UncErr", 0, 0, 4714 CNTR_NORMAL, 4715 access_pio_v1_len_mem_bank0_unc_err_cnt), 4716 [C_PIO_SM_PKT_RESET_PARITY_ERR] = CNTR_ELEM("PioSmPktResetParityErr", 0, 0, 4717 CNTR_NORMAL, 4718 access_pio_sm_pkt_reset_parity_err_cnt), 4719 [C_PIO_PKT_EVICT_FIFO_PARITY_ERR] = CNTR_ELEM("PioPktEvictFifoParityErr", 0, 0, 4720 CNTR_NORMAL, 4721 access_pio_pkt_evict_fifo_parity_err_cnt), 4722 [C_PIO_SBRDCTRL_CRREL_FIFO_PARITY_ERR] = CNTR_ELEM( 4723 "PioSbrdctrlCrrelFifoParityErr", 0, 0, 4724 CNTR_NORMAL, 4725 access_pio_sbrdctrl_crrel_fifo_parity_err_cnt), 4726 [C_PIO_SBRDCTL_CRREL_PARITY_ERR] = CNTR_ELEM("PioSbrdctlCrrelParityErr", 0, 0, 4727 CNTR_NORMAL, 4728 access_pio_sbrdctl_crrel_parity_err_cnt), 4729 [C_PIO_PEC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPecFifoParityErr", 0, 0, 4730 CNTR_NORMAL, 4731 access_pio_pec_fifo_parity_err_cnt), 4732 [C_PIO_PCC_FIFO_PARITY_ERR] = CNTR_ELEM("PioPccFifoParityErr", 0, 0, 4733 CNTR_NORMAL, 4734 access_pio_pcc_fifo_parity_err_cnt), 4735 [C_PIO_SB_MEM_FIFO1_ERR] = CNTR_ELEM("PioSbMemFifo1Err", 0, 0, 4736 CNTR_NORMAL, 4737 access_pio_sb_mem_fifo1_err_cnt), 4738 [C_PIO_SB_MEM_FIFO0_ERR] = CNTR_ELEM("PioSbMemFifo0Err", 0, 0, 4739 CNTR_NORMAL, 4740 access_pio_sb_mem_fifo0_err_cnt), 4741 [C_PIO_CSR_PARITY_ERR] = CNTR_ELEM("PioCsrParityErr", 0, 0, 4742 CNTR_NORMAL, 4743 access_pio_csr_parity_err_cnt), 4744 [C_PIO_WRITE_ADDR_PARITY_ERR] = CNTR_ELEM("PioWriteAddrParityErr", 0, 0, 4745 CNTR_NORMAL, 4746 access_pio_write_addr_parity_err_cnt), 4747 [C_PIO_WRITE_BAD_CTXT_ERR] = CNTR_ELEM("PioWriteBadCtxtErr", 0, 0, 4748 CNTR_NORMAL, 4749 access_pio_write_bad_ctxt_err_cnt), 4750 /* SendDmaErrStatus */ 4751 [C_SDMA_PCIE_REQ_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPcieReqTrackingCorErr", 0, 4752 0, CNTR_NORMAL, 4753 access_sdma_pcie_req_tracking_cor_err_cnt), 4754 [C_SDMA_PCIE_REQ_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPcieReqTrackingUncErr", 0, 4755 0, CNTR_NORMAL, 4756 access_sdma_pcie_req_tracking_unc_err_cnt), 4757 [C_SDMA_CSR_PARITY_ERR] = CNTR_ELEM("SDmaCsrParityErr", 0, 0, 4758 CNTR_NORMAL, 4759 access_sdma_csr_parity_err_cnt), 4760 [C_SDMA_RPY_TAG_ERR] = CNTR_ELEM("SDmaRpyTagErr", 0, 0, 4761 CNTR_NORMAL, 4762 access_sdma_rpy_tag_err_cnt), 4763 /* SendEgressErrStatus */ 4764 [C_TX_READ_PIO_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryCsrUncErr", 0, 0, 4765 CNTR_NORMAL, 4766 access_tx_read_pio_memory_csr_unc_err_cnt), 4767 [C_TX_READ_SDMA_MEMORY_CSR_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryCsrUncErr", 0, 4768 0, CNTR_NORMAL, 4769 access_tx_read_sdma_memory_csr_err_cnt), 4770 [C_TX_EGRESS_FIFO_COR_ERR] = CNTR_ELEM("TxEgressFifoCorErr", 0, 0, 4771 CNTR_NORMAL, 4772 access_tx_egress_fifo_cor_err_cnt), 4773 [C_TX_READ_PIO_MEMORY_COR_ERR] = CNTR_ELEM("TxReadPioMemoryCorErr", 0, 0, 4774 CNTR_NORMAL, 4775 access_tx_read_pio_memory_cor_err_cnt), 4776 [C_TX_READ_SDMA_MEMORY_COR_ERR] = CNTR_ELEM("TxReadSdmaMemoryCorErr", 0, 0, 4777 CNTR_NORMAL, 4778 access_tx_read_sdma_memory_cor_err_cnt), 4779 [C_TX_SB_HDR_COR_ERR] = CNTR_ELEM("TxSbHdrCorErr", 0, 0, 4780 CNTR_NORMAL, 4781 access_tx_sb_hdr_cor_err_cnt), 4782 [C_TX_CREDIT_OVERRUN_ERR] = CNTR_ELEM("TxCreditOverrunErr", 0, 0, 4783 CNTR_NORMAL, 4784 access_tx_credit_overrun_err_cnt), 4785 [C_TX_LAUNCH_FIFO8_COR_ERR] = CNTR_ELEM("TxLaunchFifo8CorErr", 0, 0, 4786 CNTR_NORMAL, 4787 access_tx_launch_fifo8_cor_err_cnt), 4788 [C_TX_LAUNCH_FIFO7_COR_ERR] = CNTR_ELEM("TxLaunchFifo7CorErr", 0, 0, 4789 CNTR_NORMAL, 4790 access_tx_launch_fifo7_cor_err_cnt), 4791 [C_TX_LAUNCH_FIFO6_COR_ERR] = CNTR_ELEM("TxLaunchFifo6CorErr", 0, 0, 4792 CNTR_NORMAL, 4793 access_tx_launch_fifo6_cor_err_cnt), 4794 [C_TX_LAUNCH_FIFO5_COR_ERR] = CNTR_ELEM("TxLaunchFifo5CorErr", 0, 0, 4795 CNTR_NORMAL, 4796 access_tx_launch_fifo5_cor_err_cnt), 4797 [C_TX_LAUNCH_FIFO4_COR_ERR] = CNTR_ELEM("TxLaunchFifo4CorErr", 0, 0, 4798 CNTR_NORMAL, 4799 access_tx_launch_fifo4_cor_err_cnt), 4800 [C_TX_LAUNCH_FIFO3_COR_ERR] = CNTR_ELEM("TxLaunchFifo3CorErr", 0, 0, 4801 CNTR_NORMAL, 4802 access_tx_launch_fifo3_cor_err_cnt), 4803 [C_TX_LAUNCH_FIFO2_COR_ERR] = CNTR_ELEM("TxLaunchFifo2CorErr", 0, 0, 4804 CNTR_NORMAL, 4805 access_tx_launch_fifo2_cor_err_cnt), 4806 [C_TX_LAUNCH_FIFO1_COR_ERR] = CNTR_ELEM("TxLaunchFifo1CorErr", 0, 0, 4807 CNTR_NORMAL, 4808 access_tx_launch_fifo1_cor_err_cnt), 4809 [C_TX_LAUNCH_FIFO0_COR_ERR] = CNTR_ELEM("TxLaunchFifo0CorErr", 0, 0, 4810 CNTR_NORMAL, 4811 access_tx_launch_fifo0_cor_err_cnt), 4812 [C_TX_CREDIT_RETURN_VL_ERR] = CNTR_ELEM("TxCreditReturnVLErr", 0, 0, 4813 CNTR_NORMAL, 4814 access_tx_credit_return_vl_err_cnt), 4815 [C_TX_HCRC_INSERTION_ERR] = CNTR_ELEM("TxHcrcInsertionErr", 0, 0, 4816 CNTR_NORMAL, 4817 access_tx_hcrc_insertion_err_cnt), 4818 [C_TX_EGRESS_FIFI_UNC_ERR] = CNTR_ELEM("TxEgressFifoUncErr", 0, 0, 4819 CNTR_NORMAL, 4820 access_tx_egress_fifo_unc_err_cnt), 4821 [C_TX_READ_PIO_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadPioMemoryUncErr", 0, 0, 4822 CNTR_NORMAL, 4823 access_tx_read_pio_memory_unc_err_cnt), 4824 [C_TX_READ_SDMA_MEMORY_UNC_ERR] = CNTR_ELEM("TxReadSdmaMemoryUncErr", 0, 0, 4825 CNTR_NORMAL, 4826 access_tx_read_sdma_memory_unc_err_cnt), 4827 [C_TX_SB_HDR_UNC_ERR] = CNTR_ELEM("TxSbHdrUncErr", 0, 0, 4828 CNTR_NORMAL, 4829 access_tx_sb_hdr_unc_err_cnt), 4830 [C_TX_CREDIT_RETURN_PARITY_ERR] = CNTR_ELEM("TxCreditReturnParityErr", 0, 0, 4831 CNTR_NORMAL, 4832 access_tx_credit_return_partiy_err_cnt), 4833 [C_TX_LAUNCH_FIFO8_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo8UncOrParityErr", 4834 0, 0, CNTR_NORMAL, 4835 access_tx_launch_fifo8_unc_or_parity_err_cnt), 4836 [C_TX_LAUNCH_FIFO7_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo7UncOrParityErr", 4837 0, 0, CNTR_NORMAL, 4838 access_tx_launch_fifo7_unc_or_parity_err_cnt), 4839 [C_TX_LAUNCH_FIFO6_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo6UncOrParityErr", 4840 0, 0, CNTR_NORMAL, 4841 access_tx_launch_fifo6_unc_or_parity_err_cnt), 4842 [C_TX_LAUNCH_FIFO5_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo5UncOrParityErr", 4843 0, 0, CNTR_NORMAL, 4844 access_tx_launch_fifo5_unc_or_parity_err_cnt), 4845 [C_TX_LAUNCH_FIFO4_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo4UncOrParityErr", 4846 0, 0, CNTR_NORMAL, 4847 access_tx_launch_fifo4_unc_or_parity_err_cnt), 4848 [C_TX_LAUNCH_FIFO3_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo3UncOrParityErr", 4849 0, 0, CNTR_NORMAL, 4850 access_tx_launch_fifo3_unc_or_parity_err_cnt), 4851 [C_TX_LAUNCH_FIFO2_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo2UncOrParityErr", 4852 0, 0, CNTR_NORMAL, 4853 access_tx_launch_fifo2_unc_or_parity_err_cnt), 4854 [C_TX_LAUNCH_FIFO1_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo1UncOrParityErr", 4855 0, 0, CNTR_NORMAL, 4856 access_tx_launch_fifo1_unc_or_parity_err_cnt), 4857 [C_TX_LAUNCH_FIFO0_UNC_OR_PARITY_ERR] = CNTR_ELEM("TxLaunchFifo0UncOrParityErr", 4858 0, 0, CNTR_NORMAL, 4859 access_tx_launch_fifo0_unc_or_parity_err_cnt), 4860 [C_TX_SDMA15_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma15DisallowedPacketErr", 4861 0, 0, CNTR_NORMAL, 4862 access_tx_sdma15_disallowed_packet_err_cnt), 4863 [C_TX_SDMA14_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma14DisallowedPacketErr", 4864 0, 0, CNTR_NORMAL, 4865 access_tx_sdma14_disallowed_packet_err_cnt), 4866 [C_TX_SDMA13_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma13DisallowedPacketErr", 4867 0, 0, CNTR_NORMAL, 4868 access_tx_sdma13_disallowed_packet_err_cnt), 4869 [C_TX_SDMA12_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma12DisallowedPacketErr", 4870 0, 0, CNTR_NORMAL, 4871 access_tx_sdma12_disallowed_packet_err_cnt), 4872 [C_TX_SDMA11_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma11DisallowedPacketErr", 4873 0, 0, CNTR_NORMAL, 4874 access_tx_sdma11_disallowed_packet_err_cnt), 4875 [C_TX_SDMA10_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma10DisallowedPacketErr", 4876 0, 0, CNTR_NORMAL, 4877 access_tx_sdma10_disallowed_packet_err_cnt), 4878 [C_TX_SDMA9_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma9DisallowedPacketErr", 4879 0, 0, CNTR_NORMAL, 4880 access_tx_sdma9_disallowed_packet_err_cnt), 4881 [C_TX_SDMA8_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma8DisallowedPacketErr", 4882 0, 0, CNTR_NORMAL, 4883 access_tx_sdma8_disallowed_packet_err_cnt), 4884 [C_TX_SDMA7_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma7DisallowedPacketErr", 4885 0, 0, CNTR_NORMAL, 4886 access_tx_sdma7_disallowed_packet_err_cnt), 4887 [C_TX_SDMA6_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma6DisallowedPacketErr", 4888 0, 0, CNTR_NORMAL, 4889 access_tx_sdma6_disallowed_packet_err_cnt), 4890 [C_TX_SDMA5_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma5DisallowedPacketErr", 4891 0, 0, CNTR_NORMAL, 4892 access_tx_sdma5_disallowed_packet_err_cnt), 4893 [C_TX_SDMA4_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma4DisallowedPacketErr", 4894 0, 0, CNTR_NORMAL, 4895 access_tx_sdma4_disallowed_packet_err_cnt), 4896 [C_TX_SDMA3_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma3DisallowedPacketErr", 4897 0, 0, CNTR_NORMAL, 4898 access_tx_sdma3_disallowed_packet_err_cnt), 4899 [C_TX_SDMA2_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma2DisallowedPacketErr", 4900 0, 0, CNTR_NORMAL, 4901 access_tx_sdma2_disallowed_packet_err_cnt), 4902 [C_TX_SDMA1_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma1DisallowedPacketErr", 4903 0, 0, CNTR_NORMAL, 4904 access_tx_sdma1_disallowed_packet_err_cnt), 4905 [C_TX_SDMA0_DISALLOWED_PACKET_ERR] = CNTR_ELEM("TxSdma0DisallowedPacketErr", 4906 0, 0, CNTR_NORMAL, 4907 access_tx_sdma0_disallowed_packet_err_cnt), 4908 [C_TX_CONFIG_PARITY_ERR] = CNTR_ELEM("TxConfigParityErr", 0, 0, 4909 CNTR_NORMAL, 4910 access_tx_config_parity_err_cnt), 4911 [C_TX_SBRD_CTL_CSR_PARITY_ERR] = CNTR_ELEM("TxSbrdCtlCsrParityErr", 0, 0, 4912 CNTR_NORMAL, 4913 access_tx_sbrd_ctl_csr_parity_err_cnt), 4914 [C_TX_LAUNCH_CSR_PARITY_ERR] = CNTR_ELEM("TxLaunchCsrParityErr", 0, 0, 4915 CNTR_NORMAL, 4916 access_tx_launch_csr_parity_err_cnt), 4917 [C_TX_ILLEGAL_CL_ERR] = CNTR_ELEM("TxIllegalVLErr", 0, 0, 4918 CNTR_NORMAL, 4919 access_tx_illegal_vl_err_cnt), 4920 [C_TX_SBRD_CTL_STATE_MACHINE_PARITY_ERR] = CNTR_ELEM( 4921 "TxSbrdCtlStateMachineParityErr", 0, 0, 4922 CNTR_NORMAL, 4923 access_tx_sbrd_ctl_state_machine_parity_err_cnt), 4924 [C_TX_RESERVED_10] = CNTR_ELEM("Tx Egress Reserved 10", 0, 0, 4925 CNTR_NORMAL, 4926 access_egress_reserved_10_err_cnt), 4927 [C_TX_RESERVED_9] = CNTR_ELEM("Tx Egress Reserved 9", 0, 0, 4928 CNTR_NORMAL, 4929 access_egress_reserved_9_err_cnt), 4930 [C_TX_SDMA_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxSdmaLaunchIntfParityErr", 4931 0, 0, CNTR_NORMAL, 4932 access_tx_sdma_launch_intf_parity_err_cnt), 4933 [C_TX_PIO_LAUNCH_INTF_PARITY_ERR] = CNTR_ELEM("TxPioLaunchIntfParityErr", 0, 0, 4934 CNTR_NORMAL, 4935 access_tx_pio_launch_intf_parity_err_cnt), 4936 [C_TX_RESERVED_6] = CNTR_ELEM("Tx Egress Reserved 6", 0, 0, 4937 CNTR_NORMAL, 4938 access_egress_reserved_6_err_cnt), 4939 [C_TX_INCORRECT_LINK_STATE_ERR] = CNTR_ELEM("TxIncorrectLinkStateErr", 0, 0, 4940 CNTR_NORMAL, 4941 access_tx_incorrect_link_state_err_cnt), 4942 [C_TX_LINK_DOWN_ERR] = CNTR_ELEM("TxLinkdownErr", 0, 0, 4943 CNTR_NORMAL, 4944 access_tx_linkdown_err_cnt), 4945 [C_TX_EGRESS_FIFO_UNDERRUN_OR_PARITY_ERR] = CNTR_ELEM( 4946 "EgressFifoUnderrunOrParityErr", 0, 0, 4947 CNTR_NORMAL, 4948 access_tx_egress_fifi_underrun_or_parity_err_cnt), 4949 [C_TX_RESERVED_2] = CNTR_ELEM("Tx Egress Reserved 2", 0, 0, 4950 CNTR_NORMAL, 4951 access_egress_reserved_2_err_cnt), 4952 [C_TX_PKT_INTEGRITY_MEM_UNC_ERR] = CNTR_ELEM("TxPktIntegrityMemUncErr", 0, 0, 4953 CNTR_NORMAL, 4954 access_tx_pkt_integrity_mem_unc_err_cnt), 4955 [C_TX_PKT_INTEGRITY_MEM_COR_ERR] = CNTR_ELEM("TxPktIntegrityMemCorErr", 0, 0, 4956 CNTR_NORMAL, 4957 access_tx_pkt_integrity_mem_cor_err_cnt), 4958 /* SendErrStatus */ 4959 [C_SEND_CSR_WRITE_BAD_ADDR_ERR] = CNTR_ELEM("SendCsrWriteBadAddrErr", 0, 0, 4960 CNTR_NORMAL, 4961 access_send_csr_write_bad_addr_err_cnt), 4962 [C_SEND_CSR_READ_BAD_ADD_ERR] = CNTR_ELEM("SendCsrReadBadAddrErr", 0, 0, 4963 CNTR_NORMAL, 4964 access_send_csr_read_bad_addr_err_cnt), 4965 [C_SEND_CSR_PARITY_ERR] = CNTR_ELEM("SendCsrParityErr", 0, 0, 4966 CNTR_NORMAL, 4967 access_send_csr_parity_cnt), 4968 /* SendCtxtErrStatus */ 4969 [C_PIO_WRITE_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("PioWriteOutOfBoundsErr", 0, 0, 4970 CNTR_NORMAL, 4971 access_pio_write_out_of_bounds_err_cnt), 4972 [C_PIO_WRITE_OVERFLOW_ERR] = CNTR_ELEM("PioWriteOverflowErr", 0, 0, 4973 CNTR_NORMAL, 4974 access_pio_write_overflow_err_cnt), 4975 [C_PIO_WRITE_CROSSES_BOUNDARY_ERR] = CNTR_ELEM("PioWriteCrossesBoundaryErr", 4976 0, 0, CNTR_NORMAL, 4977 access_pio_write_crosses_boundary_err_cnt), 4978 [C_PIO_DISALLOWED_PACKET_ERR] = CNTR_ELEM("PioDisallowedPacketErr", 0, 0, 4979 CNTR_NORMAL, 4980 access_pio_disallowed_packet_err_cnt), 4981 [C_PIO_INCONSISTENT_SOP_ERR] = CNTR_ELEM("PioInconsistentSopErr", 0, 0, 4982 CNTR_NORMAL, 4983 access_pio_inconsistent_sop_err_cnt), 4984 /* SendDmaEngErrStatus */ 4985 [C_SDMA_HEADER_REQUEST_FIFO_COR_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoCorErr", 4986 0, 0, CNTR_NORMAL, 4987 access_sdma_header_request_fifo_cor_err_cnt), 4988 [C_SDMA_HEADER_STORAGE_COR_ERR] = CNTR_ELEM("SDmaHeaderStorageCorErr", 0, 0, 4989 CNTR_NORMAL, 4990 access_sdma_header_storage_cor_err_cnt), 4991 [C_SDMA_PACKET_TRACKING_COR_ERR] = CNTR_ELEM("SDmaPacketTrackingCorErr", 0, 0, 4992 CNTR_NORMAL, 4993 access_sdma_packet_tracking_cor_err_cnt), 4994 [C_SDMA_ASSEMBLY_COR_ERR] = CNTR_ELEM("SDmaAssemblyCorErr", 0, 0, 4995 CNTR_NORMAL, 4996 access_sdma_assembly_cor_err_cnt), 4997 [C_SDMA_DESC_TABLE_COR_ERR] = CNTR_ELEM("SDmaDescTableCorErr", 0, 0, 4998 CNTR_NORMAL, 4999 access_sdma_desc_table_cor_err_cnt), 5000 [C_SDMA_HEADER_REQUEST_FIFO_UNC_ERR] = CNTR_ELEM("SDmaHeaderRequestFifoUncErr", 5001 0, 0, CNTR_NORMAL, 5002 access_sdma_header_request_fifo_unc_err_cnt), 5003 [C_SDMA_HEADER_STORAGE_UNC_ERR] = CNTR_ELEM("SDmaHeaderStorageUncErr", 0, 0, 5004 CNTR_NORMAL, 5005 access_sdma_header_storage_unc_err_cnt), 5006 [C_SDMA_PACKET_TRACKING_UNC_ERR] = CNTR_ELEM("SDmaPacketTrackingUncErr", 0, 0, 5007 CNTR_NORMAL, 5008 access_sdma_packet_tracking_unc_err_cnt), 5009 [C_SDMA_ASSEMBLY_UNC_ERR] = CNTR_ELEM("SDmaAssemblyUncErr", 0, 0, 5010 CNTR_NORMAL, 5011 access_sdma_assembly_unc_err_cnt), 5012 [C_SDMA_DESC_TABLE_UNC_ERR] = CNTR_ELEM("SDmaDescTableUncErr", 0, 0, 5013 CNTR_NORMAL, 5014 access_sdma_desc_table_unc_err_cnt), 5015 [C_SDMA_TIMEOUT_ERR] = CNTR_ELEM("SDmaTimeoutErr", 0, 0, 5016 CNTR_NORMAL, 5017 access_sdma_timeout_err_cnt), 5018 [C_SDMA_HEADER_LENGTH_ERR] = CNTR_ELEM("SDmaHeaderLengthErr", 0, 0, 5019 CNTR_NORMAL, 5020 access_sdma_header_length_err_cnt), 5021 [C_SDMA_HEADER_ADDRESS_ERR] = CNTR_ELEM("SDmaHeaderAddressErr", 0, 0, 5022 CNTR_NORMAL, 5023 access_sdma_header_address_err_cnt), 5024 [C_SDMA_HEADER_SELECT_ERR] = CNTR_ELEM("SDmaHeaderSelectErr", 0, 0, 5025 CNTR_NORMAL, 5026 access_sdma_header_select_err_cnt), 5027 [C_SMDA_RESERVED_9] = CNTR_ELEM("SDma Reserved 9", 0, 0, 5028 CNTR_NORMAL, 5029 access_sdma_reserved_9_err_cnt), 5030 [C_SDMA_PACKET_DESC_OVERFLOW_ERR] = CNTR_ELEM("SDmaPacketDescOverflowErr", 0, 0, 5031 CNTR_NORMAL, 5032 access_sdma_packet_desc_overflow_err_cnt), 5033 [C_SDMA_LENGTH_MISMATCH_ERR] = CNTR_ELEM("SDmaLengthMismatchErr", 0, 0, 5034 CNTR_NORMAL, 5035 access_sdma_length_mismatch_err_cnt), 5036 [C_SDMA_HALT_ERR] = CNTR_ELEM("SDmaHaltErr", 0, 0, 5037 CNTR_NORMAL, 5038 access_sdma_halt_err_cnt), 5039 [C_SDMA_MEM_READ_ERR] = CNTR_ELEM("SDmaMemReadErr", 0, 0, 5040 CNTR_NORMAL, 5041 access_sdma_mem_read_err_cnt), 5042 [C_SDMA_FIRST_DESC_ERR] = CNTR_ELEM("SDmaFirstDescErr", 0, 0, 5043 CNTR_NORMAL, 5044 access_sdma_first_desc_err_cnt), 5045 [C_SDMA_TAIL_OUT_OF_BOUNDS_ERR] = CNTR_ELEM("SDmaTailOutOfBoundsErr", 0, 0, 5046 CNTR_NORMAL, 5047 access_sdma_tail_out_of_bounds_err_cnt), 5048 [C_SDMA_TOO_LONG_ERR] = CNTR_ELEM("SDmaTooLongErr", 0, 0, 5049 CNTR_NORMAL, 5050 access_sdma_too_long_err_cnt), 5051 [C_SDMA_GEN_MISMATCH_ERR] = CNTR_ELEM("SDmaGenMismatchErr", 0, 0, 5052 CNTR_NORMAL, 5053 access_sdma_gen_mismatch_err_cnt), 5054 [C_SDMA_WRONG_DW_ERR] = CNTR_ELEM("SDmaWrongDwErr", 0, 0, 5055 CNTR_NORMAL, 5056 access_sdma_wrong_dw_err_cnt), 5057 }; 5058 5059 static struct cntr_entry port_cntrs[PORT_CNTR_LAST] = { 5060 [C_TX_UNSUP_VL] = TXE32_PORT_CNTR_ELEM(TxUnVLErr, SEND_UNSUP_VL_ERR_CNT, 5061 CNTR_NORMAL), 5062 [C_TX_INVAL_LEN] = TXE32_PORT_CNTR_ELEM(TxInvalLen, SEND_LEN_ERR_CNT, 5063 CNTR_NORMAL), 5064 [C_TX_MM_LEN_ERR] = TXE32_PORT_CNTR_ELEM(TxMMLenErr, SEND_MAX_MIN_LEN_ERR_CNT, 5065 CNTR_NORMAL), 5066 [C_TX_UNDERRUN] = TXE32_PORT_CNTR_ELEM(TxUnderrun, SEND_UNDERRUN_CNT, 5067 CNTR_NORMAL), 5068 [C_TX_FLOW_STALL] = TXE32_PORT_CNTR_ELEM(TxFlowStall, SEND_FLOW_STALL_CNT, 5069 CNTR_NORMAL), 5070 [C_TX_DROPPED] = TXE32_PORT_CNTR_ELEM(TxDropped, SEND_DROPPED_PKT_CNT, 5071 CNTR_NORMAL), 5072 [C_TX_HDR_ERR] = TXE32_PORT_CNTR_ELEM(TxHdrErr, SEND_HEADERS_ERR_CNT, 5073 CNTR_NORMAL), 5074 [C_TX_PKT] = TXE64_PORT_CNTR_ELEM(TxPkt, SEND_DATA_PKT_CNT, CNTR_NORMAL), 5075 [C_TX_WORDS] = TXE64_PORT_CNTR_ELEM(TxWords, SEND_DWORD_CNT, CNTR_NORMAL), 5076 [C_TX_WAIT] = TXE64_PORT_CNTR_ELEM(TxWait, SEND_WAIT_CNT, CNTR_SYNTH), 5077 [C_TX_FLIT_VL] = TXE64_PORT_CNTR_ELEM(TxFlitVL, SEND_DATA_VL0_CNT, 5078 CNTR_SYNTH | CNTR_VL), 5079 [C_TX_PKT_VL] = TXE64_PORT_CNTR_ELEM(TxPktVL, SEND_DATA_PKT_VL0_CNT, 5080 CNTR_SYNTH | CNTR_VL), 5081 [C_TX_WAIT_VL] = TXE64_PORT_CNTR_ELEM(TxWaitVL, SEND_WAIT_VL0_CNT, 5082 CNTR_SYNTH | CNTR_VL), 5083 [C_RX_PKT] = RXE64_PORT_CNTR_ELEM(RxPkt, RCV_DATA_PKT_CNT, CNTR_NORMAL), 5084 [C_RX_WORDS] = RXE64_PORT_CNTR_ELEM(RxWords, RCV_DWORD_CNT, CNTR_NORMAL), 5085 [C_SW_LINK_DOWN] = CNTR_ELEM("SwLinkDown", 0, 0, CNTR_SYNTH | CNTR_32BIT, 5086 access_sw_link_dn_cnt), 5087 [C_SW_LINK_UP] = CNTR_ELEM("SwLinkUp", 0, 0, CNTR_SYNTH | CNTR_32BIT, 5088 access_sw_link_up_cnt), 5089 [C_SW_UNKNOWN_FRAME] = CNTR_ELEM("UnknownFrame", 0, 0, CNTR_NORMAL, 5090 access_sw_unknown_frame_cnt), 5091 [C_SW_XMIT_DSCD] = CNTR_ELEM("XmitDscd", 0, 0, CNTR_SYNTH | CNTR_32BIT, 5092 access_sw_xmit_discards), 5093 [C_SW_XMIT_DSCD_VL] = CNTR_ELEM("XmitDscdVl", 0, 0, 5094 CNTR_SYNTH | CNTR_32BIT | CNTR_VL, 5095 access_sw_xmit_discards), 5096 [C_SW_XMIT_CSTR_ERR] = CNTR_ELEM("XmitCstrErr", 0, 0, CNTR_SYNTH, 5097 access_xmit_constraint_errs), 5098 [C_SW_RCV_CSTR_ERR] = CNTR_ELEM("RcvCstrErr", 0, 0, CNTR_SYNTH, 5099 access_rcv_constraint_errs), 5100 [C_SW_IBP_LOOP_PKTS] = SW_IBP_CNTR(LoopPkts, loop_pkts), 5101 [C_SW_IBP_RC_RESENDS] = SW_IBP_CNTR(RcResend, rc_resends), 5102 [C_SW_IBP_RNR_NAKS] = SW_IBP_CNTR(RnrNak, rnr_naks), 5103 [C_SW_IBP_OTHER_NAKS] = SW_IBP_CNTR(OtherNak, other_naks), 5104 [C_SW_IBP_RC_TIMEOUTS] = SW_IBP_CNTR(RcTimeOut, rc_timeouts), 5105 [C_SW_IBP_PKT_DROPS] = SW_IBP_CNTR(PktDrop, pkt_drops), 5106 [C_SW_IBP_DMA_WAIT] = SW_IBP_CNTR(DmaWait, dmawait), 5107 [C_SW_IBP_RC_SEQNAK] = SW_IBP_CNTR(RcSeqNak, rc_seqnak), 5108 [C_SW_IBP_RC_DUPREQ] = SW_IBP_CNTR(RcDupRew, rc_dupreq), 5109 [C_SW_IBP_RDMA_SEQ] = SW_IBP_CNTR(RdmaSeq, rdma_seq), 5110 [C_SW_IBP_UNALIGNED] = SW_IBP_CNTR(Unaligned, unaligned), 5111 [C_SW_IBP_SEQ_NAK] = SW_IBP_CNTR(SeqNak, seq_naks), 5112 [C_SW_IBP_RC_CRWAITS] = SW_IBP_CNTR(RcCrWait, rc_crwaits), 5113 [C_SW_CPU_RC_ACKS] = CNTR_ELEM("RcAcks", 0, 0, CNTR_NORMAL, 5114 access_sw_cpu_rc_acks), 5115 [C_SW_CPU_RC_QACKS] = CNTR_ELEM("RcQacks", 0, 0, CNTR_NORMAL, 5116 access_sw_cpu_rc_qacks), 5117 [C_SW_CPU_RC_DELAYED_COMP] = CNTR_ELEM("RcDelayComp", 0, 0, CNTR_NORMAL, 5118 access_sw_cpu_rc_delayed_comp), 5119 [OVR_LBL(0)] = OVR_ELM(0), [OVR_LBL(1)] = OVR_ELM(1), 5120 [OVR_LBL(2)] = OVR_ELM(2), [OVR_LBL(3)] = OVR_ELM(3), 5121 [OVR_LBL(4)] = OVR_ELM(4), [OVR_LBL(5)] = OVR_ELM(5), 5122 [OVR_LBL(6)] = OVR_ELM(6), [OVR_LBL(7)] = OVR_ELM(7), 5123 [OVR_LBL(8)] = OVR_ELM(8), [OVR_LBL(9)] = OVR_ELM(9), 5124 [OVR_LBL(10)] = OVR_ELM(10), [OVR_LBL(11)] = OVR_ELM(11), 5125 [OVR_LBL(12)] = OVR_ELM(12), [OVR_LBL(13)] = OVR_ELM(13), 5126 [OVR_LBL(14)] = OVR_ELM(14), [OVR_LBL(15)] = OVR_ELM(15), 5127 [OVR_LBL(16)] = OVR_ELM(16), [OVR_LBL(17)] = OVR_ELM(17), 5128 [OVR_LBL(18)] = OVR_ELM(18), [OVR_LBL(19)] = OVR_ELM(19), 5129 [OVR_LBL(20)] = OVR_ELM(20), [OVR_LBL(21)] = OVR_ELM(21), 5130 [OVR_LBL(22)] = OVR_ELM(22), [OVR_LBL(23)] = OVR_ELM(23), 5131 [OVR_LBL(24)] = OVR_ELM(24), [OVR_LBL(25)] = OVR_ELM(25), 5132 [OVR_LBL(26)] = OVR_ELM(26), [OVR_LBL(27)] = OVR_ELM(27), 5133 [OVR_LBL(28)] = OVR_ELM(28), [OVR_LBL(29)] = OVR_ELM(29), 5134 [OVR_LBL(30)] = OVR_ELM(30), [OVR_LBL(31)] = OVR_ELM(31), 5135 [OVR_LBL(32)] = OVR_ELM(32), [OVR_LBL(33)] = OVR_ELM(33), 5136 [OVR_LBL(34)] = OVR_ELM(34), [OVR_LBL(35)] = OVR_ELM(35), 5137 [OVR_LBL(36)] = OVR_ELM(36), [OVR_LBL(37)] = OVR_ELM(37), 5138 [OVR_LBL(38)] = OVR_ELM(38), [OVR_LBL(39)] = OVR_ELM(39), 5139 [OVR_LBL(40)] = OVR_ELM(40), [OVR_LBL(41)] = OVR_ELM(41), 5140 [OVR_LBL(42)] = OVR_ELM(42), [OVR_LBL(43)] = OVR_ELM(43), 5141 [OVR_LBL(44)] = OVR_ELM(44), [OVR_LBL(45)] = OVR_ELM(45), 5142 [OVR_LBL(46)] = OVR_ELM(46), [OVR_LBL(47)] = OVR_ELM(47), 5143 [OVR_LBL(48)] = OVR_ELM(48), [OVR_LBL(49)] = OVR_ELM(49), 5144 [OVR_LBL(50)] = OVR_ELM(50), [OVR_LBL(51)] = OVR_ELM(51), 5145 [OVR_LBL(52)] = OVR_ELM(52), [OVR_LBL(53)] = OVR_ELM(53), 5146 [OVR_LBL(54)] = OVR_ELM(54), [OVR_LBL(55)] = OVR_ELM(55), 5147 [OVR_LBL(56)] = OVR_ELM(56), [OVR_LBL(57)] = OVR_ELM(57), 5148 [OVR_LBL(58)] = OVR_ELM(58), [OVR_LBL(59)] = OVR_ELM(59), 5149 [OVR_LBL(60)] = OVR_ELM(60), [OVR_LBL(61)] = OVR_ELM(61), 5150 [OVR_LBL(62)] = OVR_ELM(62), [OVR_LBL(63)] = OVR_ELM(63), 5151 [OVR_LBL(64)] = OVR_ELM(64), [OVR_LBL(65)] = OVR_ELM(65), 5152 [OVR_LBL(66)] = OVR_ELM(66), [OVR_LBL(67)] = OVR_ELM(67), 5153 [OVR_LBL(68)] = OVR_ELM(68), [OVR_LBL(69)] = OVR_ELM(69), 5154 [OVR_LBL(70)] = OVR_ELM(70), [OVR_LBL(71)] = OVR_ELM(71), 5155 [OVR_LBL(72)] = OVR_ELM(72), [OVR_LBL(73)] = OVR_ELM(73), 5156 [OVR_LBL(74)] = OVR_ELM(74), [OVR_LBL(75)] = OVR_ELM(75), 5157 [OVR_LBL(76)] = OVR_ELM(76), [OVR_LBL(77)] = OVR_ELM(77), 5158 [OVR_LBL(78)] = OVR_ELM(78), [OVR_LBL(79)] = OVR_ELM(79), 5159 [OVR_LBL(80)] = OVR_ELM(80), [OVR_LBL(81)] = OVR_ELM(81), 5160 [OVR_LBL(82)] = OVR_ELM(82), [OVR_LBL(83)] = OVR_ELM(83), 5161 [OVR_LBL(84)] = OVR_ELM(84), [OVR_LBL(85)] = OVR_ELM(85), 5162 [OVR_LBL(86)] = OVR_ELM(86), [OVR_LBL(87)] = OVR_ELM(87), 5163 [OVR_LBL(88)] = OVR_ELM(88), [OVR_LBL(89)] = OVR_ELM(89), 5164 [OVR_LBL(90)] = OVR_ELM(90), [OVR_LBL(91)] = OVR_ELM(91), 5165 [OVR_LBL(92)] = OVR_ELM(92), [OVR_LBL(93)] = OVR_ELM(93), 5166 [OVR_LBL(94)] = OVR_ELM(94), [OVR_LBL(95)] = OVR_ELM(95), 5167 [OVR_LBL(96)] = OVR_ELM(96), [OVR_LBL(97)] = OVR_ELM(97), 5168 [OVR_LBL(98)] = OVR_ELM(98), [OVR_LBL(99)] = OVR_ELM(99), 5169 [OVR_LBL(100)] = OVR_ELM(100), [OVR_LBL(101)] = OVR_ELM(101), 5170 [OVR_LBL(102)] = OVR_ELM(102), [OVR_LBL(103)] = OVR_ELM(103), 5171 [OVR_LBL(104)] = OVR_ELM(104), [OVR_LBL(105)] = OVR_ELM(105), 5172 [OVR_LBL(106)] = OVR_ELM(106), [OVR_LBL(107)] = OVR_ELM(107), 5173 [OVR_LBL(108)] = OVR_ELM(108), [OVR_LBL(109)] = OVR_ELM(109), 5174 [OVR_LBL(110)] = OVR_ELM(110), [OVR_LBL(111)] = OVR_ELM(111), 5175 [OVR_LBL(112)] = OVR_ELM(112), [OVR_LBL(113)] = OVR_ELM(113), 5176 [OVR_LBL(114)] = OVR_ELM(114), [OVR_LBL(115)] = OVR_ELM(115), 5177 [OVR_LBL(116)] = OVR_ELM(116), [OVR_LBL(117)] = OVR_ELM(117), 5178 [OVR_LBL(118)] = OVR_ELM(118), [OVR_LBL(119)] = OVR_ELM(119), 5179 [OVR_LBL(120)] = OVR_ELM(120), [OVR_LBL(121)] = OVR_ELM(121), 5180 [OVR_LBL(122)] = OVR_ELM(122), [OVR_LBL(123)] = OVR_ELM(123), 5181 [OVR_LBL(124)] = OVR_ELM(124), [OVR_LBL(125)] = OVR_ELM(125), 5182 [OVR_LBL(126)] = OVR_ELM(126), [OVR_LBL(127)] = OVR_ELM(127), 5183 [OVR_LBL(128)] = OVR_ELM(128), [OVR_LBL(129)] = OVR_ELM(129), 5184 [OVR_LBL(130)] = OVR_ELM(130), [OVR_LBL(131)] = OVR_ELM(131), 5185 [OVR_LBL(132)] = OVR_ELM(132), [OVR_LBL(133)] = OVR_ELM(133), 5186 [OVR_LBL(134)] = OVR_ELM(134), [OVR_LBL(135)] = OVR_ELM(135), 5187 [OVR_LBL(136)] = OVR_ELM(136), [OVR_LBL(137)] = OVR_ELM(137), 5188 [OVR_LBL(138)] = OVR_ELM(138), [OVR_LBL(139)] = OVR_ELM(139), 5189 [OVR_LBL(140)] = OVR_ELM(140), [OVR_LBL(141)] = OVR_ELM(141), 5190 [OVR_LBL(142)] = OVR_ELM(142), [OVR_LBL(143)] = OVR_ELM(143), 5191 [OVR_LBL(144)] = OVR_ELM(144), [OVR_LBL(145)] = OVR_ELM(145), 5192 [OVR_LBL(146)] = OVR_ELM(146), [OVR_LBL(147)] = OVR_ELM(147), 5193 [OVR_LBL(148)] = OVR_ELM(148), [OVR_LBL(149)] = OVR_ELM(149), 5194 [OVR_LBL(150)] = OVR_ELM(150), [OVR_LBL(151)] = OVR_ELM(151), 5195 [OVR_LBL(152)] = OVR_ELM(152), [OVR_LBL(153)] = OVR_ELM(153), 5196 [OVR_LBL(154)] = OVR_ELM(154), [OVR_LBL(155)] = OVR_ELM(155), 5197 [OVR_LBL(156)] = OVR_ELM(156), [OVR_LBL(157)] = OVR_ELM(157), 5198 [OVR_LBL(158)] = OVR_ELM(158), [OVR_LBL(159)] = OVR_ELM(159), 5199 }; 5200 5201 /* ======================================================================== */ 5202 5203 /* return true if this is chip revision revision a */ 5204 int is_ax(struct hfi1_devdata *dd) 5205 { 5206 u8 chip_rev_minor = 5207 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT 5208 & CCE_REVISION_CHIP_REV_MINOR_MASK; 5209 return (chip_rev_minor & 0xf0) == 0; 5210 } 5211 5212 /* return true if this is chip revision revision b */ 5213 int is_bx(struct hfi1_devdata *dd) 5214 { 5215 u8 chip_rev_minor = 5216 dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT 5217 & CCE_REVISION_CHIP_REV_MINOR_MASK; 5218 return (chip_rev_minor & 0xF0) == 0x10; 5219 } 5220 5221 /* return true is kernel urg disabled for rcd */ 5222 bool is_urg_masked(struct hfi1_ctxtdata *rcd) 5223 { 5224 u64 mask; 5225 u32 is = IS_RCVURGENT_START + rcd->ctxt; 5226 u8 bit = is % 64; 5227 5228 mask = read_csr(rcd->dd, CCE_INT_MASK + (8 * (is / 64))); 5229 return !(mask & BIT_ULL(bit)); 5230 } 5231 5232 /* 5233 * Append string s to buffer buf. Arguments curp and len are the current 5234 * position and remaining length, respectively. 5235 * 5236 * return 0 on success, 1 on out of room 5237 */ 5238 static int append_str(char *buf, char **curp, int *lenp, const char *s) 5239 { 5240 char *p = *curp; 5241 int len = *lenp; 5242 int result = 0; /* success */ 5243 char c; 5244 5245 /* add a comma, if first in the buffer */ 5246 if (p != buf) { 5247 if (len == 0) { 5248 result = 1; /* out of room */ 5249 goto done; 5250 } 5251 *p++ = ','; 5252 len--; 5253 } 5254 5255 /* copy the string */ 5256 while ((c = *s++) != 0) { 5257 if (len == 0) { 5258 result = 1; /* out of room */ 5259 goto done; 5260 } 5261 *p++ = c; 5262 len--; 5263 } 5264 5265 done: 5266 /* write return values */ 5267 *curp = p; 5268 *lenp = len; 5269 5270 return result; 5271 } 5272 5273 /* 5274 * Using the given flag table, print a comma separated string into 5275 * the buffer. End in '*' if the buffer is too short. 5276 */ 5277 static char *flag_string(char *buf, int buf_len, u64 flags, 5278 const struct flag_table *table, int table_size) 5279 { 5280 char extra[32]; 5281 char *p = buf; 5282 int len = buf_len; 5283 int no_room = 0; 5284 int i; 5285 5286 /* make sure there is at least 2 so we can form "*" */ 5287 if (len < 2) 5288 return ""; 5289 5290 len--; /* leave room for a nul */ 5291 for (i = 0; i < table_size; i++) { 5292 if (flags & table[i].flag) { 5293 no_room = append_str(buf, &p, &len, table[i].str); 5294 if (no_room) 5295 break; 5296 flags &= ~table[i].flag; 5297 } 5298 } 5299 5300 /* any undocumented bits left? */ 5301 if (!no_room && flags) { 5302 snprintf(extra, sizeof(extra), "bits 0x%llx", flags); 5303 no_room = append_str(buf, &p, &len, extra); 5304 } 5305 5306 /* add * if ran out of room */ 5307 if (no_room) { 5308 /* may need to back up to add space for a '*' */ 5309 if (len == 0) 5310 --p; 5311 *p++ = '*'; 5312 } 5313 5314 /* add final nul - space already allocated above */ 5315 *p = 0; 5316 return buf; 5317 } 5318 5319 /* first 8 CCE error interrupt source names */ 5320 static const char * const cce_misc_names[] = { 5321 "CceErrInt", /* 0 */ 5322 "RxeErrInt", /* 1 */ 5323 "MiscErrInt", /* 2 */ 5324 "Reserved3", /* 3 */ 5325 "PioErrInt", /* 4 */ 5326 "SDmaErrInt", /* 5 */ 5327 "EgressErrInt", /* 6 */ 5328 "TxeErrInt" /* 7 */ 5329 }; 5330 5331 /* 5332 * Return the miscellaneous error interrupt name. 5333 */ 5334 static char *is_misc_err_name(char *buf, size_t bsize, unsigned int source) 5335 { 5336 if (source < ARRAY_SIZE(cce_misc_names)) 5337 strscpy_pad(buf, cce_misc_names[source], bsize); 5338 else 5339 snprintf(buf, bsize, "Reserved%u", 5340 source + IS_GENERAL_ERR_START); 5341 5342 return buf; 5343 } 5344 5345 /* 5346 * Return the SDMA engine error interrupt name. 5347 */ 5348 static char *is_sdma_eng_err_name(char *buf, size_t bsize, unsigned int source) 5349 { 5350 snprintf(buf, bsize, "SDmaEngErrInt%u", source); 5351 return buf; 5352 } 5353 5354 /* 5355 * Return the send context error interrupt name. 5356 */ 5357 static char *is_sendctxt_err_name(char *buf, size_t bsize, unsigned int source) 5358 { 5359 snprintf(buf, bsize, "SendCtxtErrInt%u", source); 5360 return buf; 5361 } 5362 5363 static const char * const various_names[] = { 5364 "PbcInt", 5365 "GpioAssertInt", 5366 "Qsfp1Int", 5367 "Qsfp2Int", 5368 "TCritInt" 5369 }; 5370 5371 /* 5372 * Return the various interrupt name. 5373 */ 5374 static char *is_various_name(char *buf, size_t bsize, unsigned int source) 5375 { 5376 if (source < ARRAY_SIZE(various_names)) 5377 strscpy_pad(buf, various_names[source], bsize); 5378 else 5379 snprintf(buf, bsize, "Reserved%u", source + IS_VARIOUS_START); 5380 return buf; 5381 } 5382 5383 /* 5384 * Return the DC interrupt name. 5385 */ 5386 static char *is_dc_name(char *buf, size_t bsize, unsigned int source) 5387 { 5388 static const char * const dc_int_names[] = { 5389 "common", 5390 "lcb", 5391 "8051", 5392 "lbm" /* local block merge */ 5393 }; 5394 5395 if (source < ARRAY_SIZE(dc_int_names)) 5396 snprintf(buf, bsize, "dc_%s_int", dc_int_names[source]); 5397 else 5398 snprintf(buf, bsize, "DCInt%u", source); 5399 return buf; 5400 } 5401 5402 static const char * const sdma_int_names[] = { 5403 "SDmaInt", 5404 "SdmaIdleInt", 5405 "SdmaProgressInt", 5406 }; 5407 5408 /* 5409 * Return the SDMA engine interrupt name. 5410 */ 5411 static char *is_sdma_eng_name(char *buf, size_t bsize, unsigned int source) 5412 { 5413 /* what interrupt */ 5414 unsigned int what = source / TXE_NUM_SDMA_ENGINES; 5415 /* which engine */ 5416 unsigned int which = source % TXE_NUM_SDMA_ENGINES; 5417 5418 if (likely(what < 3)) 5419 snprintf(buf, bsize, "%s%u", sdma_int_names[what], which); 5420 else 5421 snprintf(buf, bsize, "Invalid SDMA interrupt %u", source); 5422 return buf; 5423 } 5424 5425 /* 5426 * Return the receive available interrupt name. 5427 */ 5428 static char *is_rcv_avail_name(char *buf, size_t bsize, unsigned int source) 5429 { 5430 snprintf(buf, bsize, "RcvAvailInt%u", source); 5431 return buf; 5432 } 5433 5434 /* 5435 * Return the receive urgent interrupt name. 5436 */ 5437 static char *is_rcv_urgent_name(char *buf, size_t bsize, unsigned int source) 5438 { 5439 snprintf(buf, bsize, "RcvUrgentInt%u", source); 5440 return buf; 5441 } 5442 5443 /* 5444 * Return the send credit interrupt name. 5445 */ 5446 static char *is_send_credit_name(char *buf, size_t bsize, unsigned int source) 5447 { 5448 snprintf(buf, bsize, "SendCreditInt%u", source); 5449 return buf; 5450 } 5451 5452 /* 5453 * Return the reserved interrupt name. 5454 */ 5455 static char *is_reserved_name(char *buf, size_t bsize, unsigned int source) 5456 { 5457 snprintf(buf, bsize, "Reserved%u", source + IS_RESERVED_START); 5458 return buf; 5459 } 5460 5461 static char *cce_err_status_string(char *buf, int buf_len, u64 flags) 5462 { 5463 return flag_string(buf, buf_len, flags, 5464 cce_err_status_flags, 5465 ARRAY_SIZE(cce_err_status_flags)); 5466 } 5467 5468 static char *rxe_err_status_string(char *buf, int buf_len, u64 flags) 5469 { 5470 return flag_string(buf, buf_len, flags, 5471 rxe_err_status_flags, 5472 ARRAY_SIZE(rxe_err_status_flags)); 5473 } 5474 5475 static char *misc_err_status_string(char *buf, int buf_len, u64 flags) 5476 { 5477 return flag_string(buf, buf_len, flags, misc_err_status_flags, 5478 ARRAY_SIZE(misc_err_status_flags)); 5479 } 5480 5481 static char *pio_err_status_string(char *buf, int buf_len, u64 flags) 5482 { 5483 return flag_string(buf, buf_len, flags, 5484 pio_err_status_flags, 5485 ARRAY_SIZE(pio_err_status_flags)); 5486 } 5487 5488 static char *sdma_err_status_string(char *buf, int buf_len, u64 flags) 5489 { 5490 return flag_string(buf, buf_len, flags, 5491 sdma_err_status_flags, 5492 ARRAY_SIZE(sdma_err_status_flags)); 5493 } 5494 5495 static char *egress_err_status_string(char *buf, int buf_len, u64 flags) 5496 { 5497 return flag_string(buf, buf_len, flags, 5498 egress_err_status_flags, 5499 ARRAY_SIZE(egress_err_status_flags)); 5500 } 5501 5502 static char *egress_err_info_string(char *buf, int buf_len, u64 flags) 5503 { 5504 return flag_string(buf, buf_len, flags, 5505 egress_err_info_flags, 5506 ARRAY_SIZE(egress_err_info_flags)); 5507 } 5508 5509 static char *send_err_status_string(char *buf, int buf_len, u64 flags) 5510 { 5511 return flag_string(buf, buf_len, flags, 5512 send_err_status_flags, 5513 ARRAY_SIZE(send_err_status_flags)); 5514 } 5515 5516 static void handle_cce_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5517 { 5518 char buf[96]; 5519 int i = 0; 5520 5521 /* 5522 * For most these errors, there is nothing that can be done except 5523 * report or record it. 5524 */ 5525 dd_dev_info(dd, "CCE Error: %s\n", 5526 cce_err_status_string(buf, sizeof(buf), reg)); 5527 5528 if ((reg & CCE_ERR_STATUS_CCE_CLI2_ASYNC_FIFO_PARITY_ERR_SMASK) && 5529 is_ax(dd) && (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)) { 5530 /* this error requires a manual drop into SPC freeze mode */ 5531 /* then a fix up */ 5532 start_freeze_handling(dd->pport, FREEZE_SELF); 5533 } 5534 5535 for (i = 0; i < NUM_CCE_ERR_STATUS_COUNTERS; i++) { 5536 if (reg & (1ull << i)) { 5537 incr_cntr64(&dd->cce_err_status_cnt[i]); 5538 /* maintain a counter over all cce_err_status errors */ 5539 incr_cntr64(&dd->sw_cce_err_status_aggregate); 5540 } 5541 } 5542 } 5543 5544 /* 5545 * Check counters for receive errors that do not have an interrupt 5546 * associated with them. 5547 */ 5548 #define RCVERR_CHECK_TIME 10 5549 static void update_rcverr_timer(struct timer_list *t) 5550 { 5551 struct hfi1_devdata *dd = from_timer(dd, t, rcverr_timer); 5552 struct hfi1_pportdata *ppd = dd->pport; 5553 u32 cur_ovfl_cnt = read_dev_cntr(dd, C_RCV_OVF, CNTR_INVALID_VL); 5554 5555 if (dd->rcv_ovfl_cnt < cur_ovfl_cnt && 5556 ppd->port_error_action & OPA_PI_MASK_EX_BUFFER_OVERRUN) { 5557 dd_dev_info(dd, "%s: PortErrorAction bounce\n", __func__); 5558 set_link_down_reason( 5559 ppd, OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN, 0, 5560 OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN); 5561 queue_work(ppd->link_wq, &ppd->link_bounce_work); 5562 } 5563 dd->rcv_ovfl_cnt = (u32)cur_ovfl_cnt; 5564 5565 mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME); 5566 } 5567 5568 static int init_rcverr(struct hfi1_devdata *dd) 5569 { 5570 timer_setup(&dd->rcverr_timer, update_rcverr_timer, 0); 5571 /* Assume the hardware counter has been reset */ 5572 dd->rcv_ovfl_cnt = 0; 5573 return mod_timer(&dd->rcverr_timer, jiffies + HZ * RCVERR_CHECK_TIME); 5574 } 5575 5576 static void free_rcverr(struct hfi1_devdata *dd) 5577 { 5578 if (dd->rcverr_timer.function) 5579 del_timer_sync(&dd->rcverr_timer); 5580 } 5581 5582 static void handle_rxe_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5583 { 5584 char buf[96]; 5585 int i = 0; 5586 5587 dd_dev_info(dd, "Receive Error: %s\n", 5588 rxe_err_status_string(buf, sizeof(buf), reg)); 5589 5590 if (reg & ALL_RXE_FREEZE_ERR) { 5591 int flags = 0; 5592 5593 /* 5594 * Freeze mode recovery is disabled for the errors 5595 * in RXE_FREEZE_ABORT_MASK 5596 */ 5597 if (is_ax(dd) && (reg & RXE_FREEZE_ABORT_MASK)) 5598 flags = FREEZE_ABORT; 5599 5600 start_freeze_handling(dd->pport, flags); 5601 } 5602 5603 for (i = 0; i < NUM_RCV_ERR_STATUS_COUNTERS; i++) { 5604 if (reg & (1ull << i)) 5605 incr_cntr64(&dd->rcv_err_status_cnt[i]); 5606 } 5607 } 5608 5609 static void handle_misc_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5610 { 5611 char buf[96]; 5612 int i = 0; 5613 5614 dd_dev_info(dd, "Misc Error: %s", 5615 misc_err_status_string(buf, sizeof(buf), reg)); 5616 for (i = 0; i < NUM_MISC_ERR_STATUS_COUNTERS; i++) { 5617 if (reg & (1ull << i)) 5618 incr_cntr64(&dd->misc_err_status_cnt[i]); 5619 } 5620 } 5621 5622 static void handle_pio_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5623 { 5624 char buf[96]; 5625 int i = 0; 5626 5627 dd_dev_info(dd, "PIO Error: %s\n", 5628 pio_err_status_string(buf, sizeof(buf), reg)); 5629 5630 if (reg & ALL_PIO_FREEZE_ERR) 5631 start_freeze_handling(dd->pport, 0); 5632 5633 for (i = 0; i < NUM_SEND_PIO_ERR_STATUS_COUNTERS; i++) { 5634 if (reg & (1ull << i)) 5635 incr_cntr64(&dd->send_pio_err_status_cnt[i]); 5636 } 5637 } 5638 5639 static void handle_sdma_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5640 { 5641 char buf[96]; 5642 int i = 0; 5643 5644 dd_dev_info(dd, "SDMA Error: %s\n", 5645 sdma_err_status_string(buf, sizeof(buf), reg)); 5646 5647 if (reg & ALL_SDMA_FREEZE_ERR) 5648 start_freeze_handling(dd->pport, 0); 5649 5650 for (i = 0; i < NUM_SEND_DMA_ERR_STATUS_COUNTERS; i++) { 5651 if (reg & (1ull << i)) 5652 incr_cntr64(&dd->send_dma_err_status_cnt[i]); 5653 } 5654 } 5655 5656 static inline void __count_port_discards(struct hfi1_pportdata *ppd) 5657 { 5658 incr_cntr64(&ppd->port_xmit_discards); 5659 } 5660 5661 static void count_port_inactive(struct hfi1_devdata *dd) 5662 { 5663 __count_port_discards(dd->pport); 5664 } 5665 5666 /* 5667 * We have had a "disallowed packet" error during egress. Determine the 5668 * integrity check which failed, and update relevant error counter, etc. 5669 * 5670 * Note that the SEND_EGRESS_ERR_INFO register has only a single 5671 * bit of state per integrity check, and so we can miss the reason for an 5672 * egress error if more than one packet fails the same integrity check 5673 * since we cleared the corresponding bit in SEND_EGRESS_ERR_INFO. 5674 */ 5675 static void handle_send_egress_err_info(struct hfi1_devdata *dd, 5676 int vl) 5677 { 5678 struct hfi1_pportdata *ppd = dd->pport; 5679 u64 src = read_csr(dd, SEND_EGRESS_ERR_SOURCE); /* read first */ 5680 u64 info = read_csr(dd, SEND_EGRESS_ERR_INFO); 5681 char buf[96]; 5682 5683 /* clear down all observed info as quickly as possible after read */ 5684 write_csr(dd, SEND_EGRESS_ERR_INFO, info); 5685 5686 dd_dev_info(dd, 5687 "Egress Error Info: 0x%llx, %s Egress Error Src 0x%llx\n", 5688 info, egress_err_info_string(buf, sizeof(buf), info), src); 5689 5690 /* Eventually add other counters for each bit */ 5691 if (info & PORT_DISCARD_EGRESS_ERRS) { 5692 int weight, i; 5693 5694 /* 5695 * Count all applicable bits as individual errors and 5696 * attribute them to the packet that triggered this handler. 5697 * This may not be completely accurate due to limitations 5698 * on the available hardware error information. There is 5699 * a single information register and any number of error 5700 * packets may have occurred and contributed to it before 5701 * this routine is called. This means that: 5702 * a) If multiple packets with the same error occur before 5703 * this routine is called, earlier packets are missed. 5704 * There is only a single bit for each error type. 5705 * b) Errors may not be attributed to the correct VL. 5706 * The driver is attributing all bits in the info register 5707 * to the packet that triggered this call, but bits 5708 * could be an accumulation of different packets with 5709 * different VLs. 5710 * c) A single error packet may have multiple counts attached 5711 * to it. There is no way for the driver to know if 5712 * multiple bits set in the info register are due to a 5713 * single packet or multiple packets. The driver assumes 5714 * multiple packets. 5715 */ 5716 weight = hweight64(info & PORT_DISCARD_EGRESS_ERRS); 5717 for (i = 0; i < weight; i++) { 5718 __count_port_discards(ppd); 5719 if (vl >= 0 && vl < TXE_NUM_DATA_VL) 5720 incr_cntr64(&ppd->port_xmit_discards_vl[vl]); 5721 else if (vl == 15) 5722 incr_cntr64(&ppd->port_xmit_discards_vl 5723 [C_VL_15]); 5724 } 5725 } 5726 } 5727 5728 /* 5729 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS 5730 * register. Does it represent a 'port inactive' error? 5731 */ 5732 static inline int port_inactive_err(u64 posn) 5733 { 5734 return (posn >= SEES(TX_LINKDOWN) && 5735 posn <= SEES(TX_INCORRECT_LINK_STATE)); 5736 } 5737 5738 /* 5739 * Input value is a bit position within the SEND_EGRESS_ERR_STATUS 5740 * register. Does it represent a 'disallowed packet' error? 5741 */ 5742 static inline int disallowed_pkt_err(int posn) 5743 { 5744 return (posn >= SEES(TX_SDMA0_DISALLOWED_PACKET) && 5745 posn <= SEES(TX_SDMA15_DISALLOWED_PACKET)); 5746 } 5747 5748 /* 5749 * Input value is a bit position of one of the SDMA engine disallowed 5750 * packet errors. Return which engine. Use of this must be guarded by 5751 * disallowed_pkt_err(). 5752 */ 5753 static inline int disallowed_pkt_engine(int posn) 5754 { 5755 return posn - SEES(TX_SDMA0_DISALLOWED_PACKET); 5756 } 5757 5758 /* 5759 * Translate an SDMA engine to a VL. Return -1 if the tranlation cannot 5760 * be done. 5761 */ 5762 static int engine_to_vl(struct hfi1_devdata *dd, int engine) 5763 { 5764 struct sdma_vl_map *m; 5765 int vl; 5766 5767 /* range check */ 5768 if (engine < 0 || engine >= TXE_NUM_SDMA_ENGINES) 5769 return -1; 5770 5771 rcu_read_lock(); 5772 m = rcu_dereference(dd->sdma_map); 5773 vl = m->engine_to_vl[engine]; 5774 rcu_read_unlock(); 5775 5776 return vl; 5777 } 5778 5779 /* 5780 * Translate the send context (sofware index) into a VL. Return -1 if the 5781 * translation cannot be done. 5782 */ 5783 static int sc_to_vl(struct hfi1_devdata *dd, int sw_index) 5784 { 5785 struct send_context_info *sci; 5786 struct send_context *sc; 5787 int i; 5788 5789 sci = &dd->send_contexts[sw_index]; 5790 5791 /* there is no information for user (PSM) and ack contexts */ 5792 if ((sci->type != SC_KERNEL) && (sci->type != SC_VL15)) 5793 return -1; 5794 5795 sc = sci->sc; 5796 if (!sc) 5797 return -1; 5798 if (dd->vld[15].sc == sc) 5799 return 15; 5800 for (i = 0; i < num_vls; i++) 5801 if (dd->vld[i].sc == sc) 5802 return i; 5803 5804 return -1; 5805 } 5806 5807 static void handle_egress_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5808 { 5809 u64 reg_copy = reg, handled = 0; 5810 char buf[96]; 5811 int i = 0; 5812 5813 if (reg & ALL_TXE_EGRESS_FREEZE_ERR) 5814 start_freeze_handling(dd->pport, 0); 5815 else if (is_ax(dd) && 5816 (reg & SEND_EGRESS_ERR_STATUS_TX_CREDIT_RETURN_VL_ERR_SMASK) && 5817 (dd->icode != ICODE_FUNCTIONAL_SIMULATOR)) 5818 start_freeze_handling(dd->pport, 0); 5819 5820 while (reg_copy) { 5821 int posn = fls64(reg_copy); 5822 /* fls64() returns a 1-based offset, we want it zero based */ 5823 int shift = posn - 1; 5824 u64 mask = 1ULL << shift; 5825 5826 if (port_inactive_err(shift)) { 5827 count_port_inactive(dd); 5828 handled |= mask; 5829 } else if (disallowed_pkt_err(shift)) { 5830 int vl = engine_to_vl(dd, disallowed_pkt_engine(shift)); 5831 5832 handle_send_egress_err_info(dd, vl); 5833 handled |= mask; 5834 } 5835 reg_copy &= ~mask; 5836 } 5837 5838 reg &= ~handled; 5839 5840 if (reg) 5841 dd_dev_info(dd, "Egress Error: %s\n", 5842 egress_err_status_string(buf, sizeof(buf), reg)); 5843 5844 for (i = 0; i < NUM_SEND_EGRESS_ERR_STATUS_COUNTERS; i++) { 5845 if (reg & (1ull << i)) 5846 incr_cntr64(&dd->send_egress_err_status_cnt[i]); 5847 } 5848 } 5849 5850 static void handle_txe_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 5851 { 5852 char buf[96]; 5853 int i = 0; 5854 5855 dd_dev_info(dd, "Send Error: %s\n", 5856 send_err_status_string(buf, sizeof(buf), reg)); 5857 5858 for (i = 0; i < NUM_SEND_ERR_STATUS_COUNTERS; i++) { 5859 if (reg & (1ull << i)) 5860 incr_cntr64(&dd->send_err_status_cnt[i]); 5861 } 5862 } 5863 5864 /* 5865 * The maximum number of times the error clear down will loop before 5866 * blocking a repeating error. This value is arbitrary. 5867 */ 5868 #define MAX_CLEAR_COUNT 20 5869 5870 /* 5871 * Clear and handle an error register. All error interrupts are funneled 5872 * through here to have a central location to correctly handle single- 5873 * or multi-shot errors. 5874 * 5875 * For non per-context registers, call this routine with a context value 5876 * of 0 so the per-context offset is zero. 5877 * 5878 * If the handler loops too many times, assume that something is wrong 5879 * and can't be fixed, so mask the error bits. 5880 */ 5881 static void interrupt_clear_down(struct hfi1_devdata *dd, 5882 u32 context, 5883 const struct err_reg_info *eri) 5884 { 5885 u64 reg; 5886 u32 count; 5887 5888 /* read in a loop until no more errors are seen */ 5889 count = 0; 5890 while (1) { 5891 reg = read_kctxt_csr(dd, context, eri->status); 5892 if (reg == 0) 5893 break; 5894 write_kctxt_csr(dd, context, eri->clear, reg); 5895 if (likely(eri->handler)) 5896 eri->handler(dd, context, reg); 5897 count++; 5898 if (count > MAX_CLEAR_COUNT) { 5899 u64 mask; 5900 5901 dd_dev_err(dd, "Repeating %s bits 0x%llx - masking\n", 5902 eri->desc, reg); 5903 /* 5904 * Read-modify-write so any other masked bits 5905 * remain masked. 5906 */ 5907 mask = read_kctxt_csr(dd, context, eri->mask); 5908 mask &= ~reg; 5909 write_kctxt_csr(dd, context, eri->mask, mask); 5910 break; 5911 } 5912 } 5913 } 5914 5915 /* 5916 * CCE block "misc" interrupt. Source is < 16. 5917 */ 5918 static void is_misc_err_int(struct hfi1_devdata *dd, unsigned int source) 5919 { 5920 const struct err_reg_info *eri = &misc_errs[source]; 5921 5922 if (eri->handler) { 5923 interrupt_clear_down(dd, 0, eri); 5924 } else { 5925 dd_dev_err(dd, "Unexpected misc interrupt (%u) - reserved\n", 5926 source); 5927 } 5928 } 5929 5930 static char *send_context_err_status_string(char *buf, int buf_len, u64 flags) 5931 { 5932 return flag_string(buf, buf_len, flags, 5933 sc_err_status_flags, 5934 ARRAY_SIZE(sc_err_status_flags)); 5935 } 5936 5937 /* 5938 * Send context error interrupt. Source (hw_context) is < 160. 5939 * 5940 * All send context errors cause the send context to halt. The normal 5941 * clear-down mechanism cannot be used because we cannot clear the 5942 * error bits until several other long-running items are done first. 5943 * This is OK because with the context halted, nothing else is going 5944 * to happen on it anyway. 5945 */ 5946 static void is_sendctxt_err_int(struct hfi1_devdata *dd, 5947 unsigned int hw_context) 5948 { 5949 struct send_context_info *sci; 5950 struct send_context *sc; 5951 char flags[96]; 5952 u64 status; 5953 u32 sw_index; 5954 int i = 0; 5955 unsigned long irq_flags; 5956 5957 sw_index = dd->hw_to_sw[hw_context]; 5958 if (sw_index >= dd->num_send_contexts) { 5959 dd_dev_err(dd, 5960 "out of range sw index %u for send context %u\n", 5961 sw_index, hw_context); 5962 return; 5963 } 5964 sci = &dd->send_contexts[sw_index]; 5965 spin_lock_irqsave(&dd->sc_lock, irq_flags); 5966 sc = sci->sc; 5967 if (!sc) { 5968 dd_dev_err(dd, "%s: context %u(%u): no sc?\n", __func__, 5969 sw_index, hw_context); 5970 spin_unlock_irqrestore(&dd->sc_lock, irq_flags); 5971 return; 5972 } 5973 5974 /* tell the software that a halt has begun */ 5975 sc_stop(sc, SCF_HALTED); 5976 5977 status = read_kctxt_csr(dd, hw_context, SEND_CTXT_ERR_STATUS); 5978 5979 dd_dev_info(dd, "Send Context %u(%u) Error: %s\n", sw_index, hw_context, 5980 send_context_err_status_string(flags, sizeof(flags), 5981 status)); 5982 5983 if (status & SEND_CTXT_ERR_STATUS_PIO_DISALLOWED_PACKET_ERR_SMASK) 5984 handle_send_egress_err_info(dd, sc_to_vl(dd, sw_index)); 5985 5986 /* 5987 * Automatically restart halted kernel contexts out of interrupt 5988 * context. User contexts must ask the driver to restart the context. 5989 */ 5990 if (sc->type != SC_USER) 5991 queue_work(dd->pport->hfi1_wq, &sc->halt_work); 5992 spin_unlock_irqrestore(&dd->sc_lock, irq_flags); 5993 5994 /* 5995 * Update the counters for the corresponding status bits. 5996 * Note that these particular counters are aggregated over all 5997 * 160 contexts. 5998 */ 5999 for (i = 0; i < NUM_SEND_CTXT_ERR_STATUS_COUNTERS; i++) { 6000 if (status & (1ull << i)) 6001 incr_cntr64(&dd->sw_ctxt_err_status_cnt[i]); 6002 } 6003 } 6004 6005 static void handle_sdma_eng_err(struct hfi1_devdata *dd, 6006 unsigned int source, u64 status) 6007 { 6008 struct sdma_engine *sde; 6009 int i = 0; 6010 6011 sde = &dd->per_sdma[source]; 6012 #ifdef CONFIG_SDMA_VERBOSITY 6013 dd_dev_err(sde->dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx, 6014 slashstrip(__FILE__), __LINE__, __func__); 6015 dd_dev_err(sde->dd, "CONFIG SDMA(%u) source: %u status 0x%llx\n", 6016 sde->this_idx, source, (unsigned long long)status); 6017 #endif 6018 sde->err_cnt++; 6019 sdma_engine_error(sde, status); 6020 6021 /* 6022 * Update the counters for the corresponding status bits. 6023 * Note that these particular counters are aggregated over 6024 * all 16 DMA engines. 6025 */ 6026 for (i = 0; i < NUM_SEND_DMA_ENG_ERR_STATUS_COUNTERS; i++) { 6027 if (status & (1ull << i)) 6028 incr_cntr64(&dd->sw_send_dma_eng_err_status_cnt[i]); 6029 } 6030 } 6031 6032 /* 6033 * CCE block SDMA error interrupt. Source is < 16. 6034 */ 6035 static void is_sdma_eng_err_int(struct hfi1_devdata *dd, unsigned int source) 6036 { 6037 #ifdef CONFIG_SDMA_VERBOSITY 6038 struct sdma_engine *sde = &dd->per_sdma[source]; 6039 6040 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx, 6041 slashstrip(__FILE__), __LINE__, __func__); 6042 dd_dev_err(dd, "CONFIG SDMA(%u) source: %u\n", sde->this_idx, 6043 source); 6044 sdma_dumpstate(sde); 6045 #endif 6046 interrupt_clear_down(dd, source, &sdma_eng_err); 6047 } 6048 6049 /* 6050 * CCE block "various" interrupt. Source is < 8. 6051 */ 6052 static void is_various_int(struct hfi1_devdata *dd, unsigned int source) 6053 { 6054 const struct err_reg_info *eri = &various_err[source]; 6055 6056 /* 6057 * TCritInt cannot go through interrupt_clear_down() 6058 * because it is not a second tier interrupt. The handler 6059 * should be called directly. 6060 */ 6061 if (source == TCRIT_INT_SOURCE) 6062 handle_temp_err(dd); 6063 else if (eri->handler) 6064 interrupt_clear_down(dd, 0, eri); 6065 else 6066 dd_dev_info(dd, 6067 "%s: Unimplemented/reserved interrupt %d\n", 6068 __func__, source); 6069 } 6070 6071 static void handle_qsfp_int(struct hfi1_devdata *dd, u32 src_ctx, u64 reg) 6072 { 6073 /* src_ctx is always zero */ 6074 struct hfi1_pportdata *ppd = dd->pport; 6075 unsigned long flags; 6076 u64 qsfp_int_mgmt = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N); 6077 6078 if (reg & QSFP_HFI0_MODPRST_N) { 6079 if (!qsfp_mod_present(ppd)) { 6080 dd_dev_info(dd, "%s: QSFP module removed\n", 6081 __func__); 6082 6083 ppd->driver_link_ready = 0; 6084 /* 6085 * Cable removed, reset all our information about the 6086 * cache and cable capabilities 6087 */ 6088 6089 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags); 6090 /* 6091 * We don't set cache_refresh_required here as we expect 6092 * an interrupt when a cable is inserted 6093 */ 6094 ppd->qsfp_info.cache_valid = 0; 6095 ppd->qsfp_info.reset_needed = 0; 6096 ppd->qsfp_info.limiting_active = 0; 6097 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, 6098 flags); 6099 /* Invert the ModPresent pin now to detect plug-in */ 6100 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT : 6101 ASIC_QSFP1_INVERT, qsfp_int_mgmt); 6102 6103 if ((ppd->offline_disabled_reason > 6104 HFI1_ODR_MASK( 6105 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED)) || 6106 (ppd->offline_disabled_reason == 6107 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE))) 6108 ppd->offline_disabled_reason = 6109 HFI1_ODR_MASK( 6110 OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED); 6111 6112 if (ppd->host_link_state == HLS_DN_POLL) { 6113 /* 6114 * The link is still in POLL. This means 6115 * that the normal link down processing 6116 * will not happen. We have to do it here 6117 * before turning the DC off. 6118 */ 6119 queue_work(ppd->link_wq, &ppd->link_down_work); 6120 } 6121 } else { 6122 dd_dev_info(dd, "%s: QSFP module inserted\n", 6123 __func__); 6124 6125 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags); 6126 ppd->qsfp_info.cache_valid = 0; 6127 ppd->qsfp_info.cache_refresh_required = 1; 6128 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, 6129 flags); 6130 6131 /* 6132 * Stop inversion of ModPresent pin to detect 6133 * removal of the cable 6134 */ 6135 qsfp_int_mgmt &= ~(u64)QSFP_HFI0_MODPRST_N; 6136 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_INVERT : 6137 ASIC_QSFP1_INVERT, qsfp_int_mgmt); 6138 6139 ppd->offline_disabled_reason = 6140 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT); 6141 } 6142 } 6143 6144 if (reg & QSFP_HFI0_INT_N) { 6145 dd_dev_info(dd, "%s: Interrupt received from QSFP module\n", 6146 __func__); 6147 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags); 6148 ppd->qsfp_info.check_interrupt_flags = 1; 6149 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, flags); 6150 } 6151 6152 /* Schedule the QSFP work only if there is a cable attached. */ 6153 if (qsfp_mod_present(ppd)) 6154 queue_work(ppd->link_wq, &ppd->qsfp_info.qsfp_work); 6155 } 6156 6157 static int request_host_lcb_access(struct hfi1_devdata *dd) 6158 { 6159 int ret; 6160 6161 ret = do_8051_command(dd, HCMD_MISC, 6162 (u64)HCMD_MISC_REQUEST_LCB_ACCESS << 6163 LOAD_DATA_FIELD_ID_SHIFT, NULL); 6164 if (ret != HCMD_SUCCESS && !(dd->flags & HFI1_SHUTDOWN)) { 6165 dd_dev_err(dd, "%s: command failed with error %d\n", 6166 __func__, ret); 6167 } 6168 return ret == HCMD_SUCCESS ? 0 : -EBUSY; 6169 } 6170 6171 static int request_8051_lcb_access(struct hfi1_devdata *dd) 6172 { 6173 int ret; 6174 6175 ret = do_8051_command(dd, HCMD_MISC, 6176 (u64)HCMD_MISC_GRANT_LCB_ACCESS << 6177 LOAD_DATA_FIELD_ID_SHIFT, NULL); 6178 if (ret != HCMD_SUCCESS) { 6179 dd_dev_err(dd, "%s: command failed with error %d\n", 6180 __func__, ret); 6181 } 6182 return ret == HCMD_SUCCESS ? 0 : -EBUSY; 6183 } 6184 6185 /* 6186 * Set the LCB selector - allow host access. The DCC selector always 6187 * points to the host. 6188 */ 6189 static inline void set_host_lcb_access(struct hfi1_devdata *dd) 6190 { 6191 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL, 6192 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK | 6193 DC_DC8051_CFG_CSR_ACCESS_SEL_LCB_SMASK); 6194 } 6195 6196 /* 6197 * Clear the LCB selector - allow 8051 access. The DCC selector always 6198 * points to the host. 6199 */ 6200 static inline void set_8051_lcb_access(struct hfi1_devdata *dd) 6201 { 6202 write_csr(dd, DC_DC8051_CFG_CSR_ACCESS_SEL, 6203 DC_DC8051_CFG_CSR_ACCESS_SEL_DCC_SMASK); 6204 } 6205 6206 /* 6207 * Acquire LCB access from the 8051. If the host already has access, 6208 * just increment a counter. Otherwise, inform the 8051 that the 6209 * host is taking access. 6210 * 6211 * Returns: 6212 * 0 on success 6213 * -EBUSY if the 8051 has control and cannot be disturbed 6214 * -errno if unable to acquire access from the 8051 6215 */ 6216 int acquire_lcb_access(struct hfi1_devdata *dd, int sleep_ok) 6217 { 6218 struct hfi1_pportdata *ppd = dd->pport; 6219 int ret = 0; 6220 6221 /* 6222 * Use the host link state lock so the operation of this routine 6223 * { link state check, selector change, count increment } can occur 6224 * as a unit against a link state change. Otherwise there is a 6225 * race between the state change and the count increment. 6226 */ 6227 if (sleep_ok) { 6228 mutex_lock(&ppd->hls_lock); 6229 } else { 6230 while (!mutex_trylock(&ppd->hls_lock)) 6231 udelay(1); 6232 } 6233 6234 /* this access is valid only when the link is up */ 6235 if (ppd->host_link_state & HLS_DOWN) { 6236 dd_dev_info(dd, "%s: link state %s not up\n", 6237 __func__, link_state_name(ppd->host_link_state)); 6238 ret = -EBUSY; 6239 goto done; 6240 } 6241 6242 if (dd->lcb_access_count == 0) { 6243 ret = request_host_lcb_access(dd); 6244 if (ret) { 6245 if (!(dd->flags & HFI1_SHUTDOWN)) 6246 dd_dev_err(dd, 6247 "%s: unable to acquire LCB access, err %d\n", 6248 __func__, ret); 6249 goto done; 6250 } 6251 set_host_lcb_access(dd); 6252 } 6253 dd->lcb_access_count++; 6254 done: 6255 mutex_unlock(&ppd->hls_lock); 6256 return ret; 6257 } 6258 6259 /* 6260 * Release LCB access by decrementing the use count. If the count is moving 6261 * from 1 to 0, inform 8051 that it has control back. 6262 * 6263 * Returns: 6264 * 0 on success 6265 * -errno if unable to release access to the 8051 6266 */ 6267 int release_lcb_access(struct hfi1_devdata *dd, int sleep_ok) 6268 { 6269 int ret = 0; 6270 6271 /* 6272 * Use the host link state lock because the acquire needed it. 6273 * Here, we only need to keep { selector change, count decrement } 6274 * as a unit. 6275 */ 6276 if (sleep_ok) { 6277 mutex_lock(&dd->pport->hls_lock); 6278 } else { 6279 while (!mutex_trylock(&dd->pport->hls_lock)) 6280 udelay(1); 6281 } 6282 6283 if (dd->lcb_access_count == 0) { 6284 dd_dev_err(dd, "%s: LCB access count is zero. Skipping.\n", 6285 __func__); 6286 goto done; 6287 } 6288 6289 if (dd->lcb_access_count == 1) { 6290 set_8051_lcb_access(dd); 6291 ret = request_8051_lcb_access(dd); 6292 if (ret) { 6293 dd_dev_err(dd, 6294 "%s: unable to release LCB access, err %d\n", 6295 __func__, ret); 6296 /* restore host access if the grant didn't work */ 6297 set_host_lcb_access(dd); 6298 goto done; 6299 } 6300 } 6301 dd->lcb_access_count--; 6302 done: 6303 mutex_unlock(&dd->pport->hls_lock); 6304 return ret; 6305 } 6306 6307 /* 6308 * Initialize LCB access variables and state. Called during driver load, 6309 * after most of the initialization is finished. 6310 * 6311 * The DC default is LCB access on for the host. The driver defaults to 6312 * leaving access to the 8051. Assign access now - this constrains the call 6313 * to this routine to be after all LCB set-up is done. In particular, after 6314 * hf1_init_dd() -> set_up_interrupts() -> clear_all_interrupts() 6315 */ 6316 static void init_lcb_access(struct hfi1_devdata *dd) 6317 { 6318 dd->lcb_access_count = 0; 6319 } 6320 6321 /* 6322 * Write a response back to a 8051 request. 6323 */ 6324 static void hreq_response(struct hfi1_devdata *dd, u8 return_code, u16 rsp_data) 6325 { 6326 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, 6327 DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK | 6328 (u64)return_code << 6329 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT | 6330 (u64)rsp_data << DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT); 6331 } 6332 6333 /* 6334 * Handle host requests from the 8051. 6335 */ 6336 static void handle_8051_request(struct hfi1_pportdata *ppd) 6337 { 6338 struct hfi1_devdata *dd = ppd->dd; 6339 u64 reg; 6340 u16 data = 0; 6341 u8 type; 6342 6343 reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_1); 6344 if ((reg & DC_DC8051_CFG_EXT_DEV_1_REQ_NEW_SMASK) == 0) 6345 return; /* no request */ 6346 6347 /* zero out COMPLETED so the response is seen */ 6348 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, 0); 6349 6350 /* extract request details */ 6351 type = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_SHIFT) 6352 & DC_DC8051_CFG_EXT_DEV_1_REQ_TYPE_MASK; 6353 data = (reg >> DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT) 6354 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_MASK; 6355 6356 switch (type) { 6357 case HREQ_LOAD_CONFIG: 6358 case HREQ_SAVE_CONFIG: 6359 case HREQ_READ_CONFIG: 6360 case HREQ_SET_TX_EQ_ABS: 6361 case HREQ_SET_TX_EQ_REL: 6362 case HREQ_ENABLE: 6363 dd_dev_info(dd, "8051 request: request 0x%x not supported\n", 6364 type); 6365 hreq_response(dd, HREQ_NOT_SUPPORTED, 0); 6366 break; 6367 case HREQ_LCB_RESET: 6368 /* Put the LCB, RX FPE and TX FPE into reset */ 6369 write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_INTO_RESET); 6370 /* Make sure the write completed */ 6371 (void)read_csr(dd, DCC_CFG_RESET); 6372 /* Hold the reset long enough to take effect */ 6373 udelay(1); 6374 /* Take the LCB, RX FPE and TX FPE out of reset */ 6375 write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_OUT_OF_RESET); 6376 hreq_response(dd, HREQ_SUCCESS, 0); 6377 6378 break; 6379 case HREQ_CONFIG_DONE: 6380 hreq_response(dd, HREQ_SUCCESS, 0); 6381 break; 6382 6383 case HREQ_INTERFACE_TEST: 6384 hreq_response(dd, HREQ_SUCCESS, data); 6385 break; 6386 default: 6387 dd_dev_err(dd, "8051 request: unknown request 0x%x\n", type); 6388 hreq_response(dd, HREQ_NOT_SUPPORTED, 0); 6389 break; 6390 } 6391 } 6392 6393 /* 6394 * Set up allocation unit vaulue. 6395 */ 6396 void set_up_vau(struct hfi1_devdata *dd, u8 vau) 6397 { 6398 u64 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT); 6399 6400 /* do not modify other values in the register */ 6401 reg &= ~SEND_CM_GLOBAL_CREDIT_AU_SMASK; 6402 reg |= (u64)vau << SEND_CM_GLOBAL_CREDIT_AU_SHIFT; 6403 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg); 6404 } 6405 6406 /* 6407 * Set up initial VL15 credits of the remote. Assumes the rest of 6408 * the CM credit registers are zero from a previous global or credit reset. 6409 * Shared limit for VL15 will always be 0. 6410 */ 6411 void set_up_vl15(struct hfi1_devdata *dd, u16 vl15buf) 6412 { 6413 u64 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT); 6414 6415 /* set initial values for total and shared credit limit */ 6416 reg &= ~(SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK | 6417 SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK); 6418 6419 /* 6420 * Set total limit to be equal to VL15 credits. 6421 * Leave shared limit at 0. 6422 */ 6423 reg |= (u64)vl15buf << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT; 6424 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg); 6425 6426 write_csr(dd, SEND_CM_CREDIT_VL15, (u64)vl15buf 6427 << SEND_CM_CREDIT_VL15_DEDICATED_LIMIT_VL_SHIFT); 6428 } 6429 6430 /* 6431 * Zero all credit details from the previous connection and 6432 * reset the CM manager's internal counters. 6433 */ 6434 void reset_link_credits(struct hfi1_devdata *dd) 6435 { 6436 int i; 6437 6438 /* remove all previous VL credit limits */ 6439 for (i = 0; i < TXE_NUM_DATA_VL; i++) 6440 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0); 6441 write_csr(dd, SEND_CM_CREDIT_VL15, 0); 6442 write_csr(dd, SEND_CM_GLOBAL_CREDIT, 0); 6443 /* reset the CM block */ 6444 pio_send_control(dd, PSC_CM_RESET); 6445 /* reset cached value */ 6446 dd->vl15buf_cached = 0; 6447 } 6448 6449 /* convert a vCU to a CU */ 6450 static u32 vcu_to_cu(u8 vcu) 6451 { 6452 return 1 << vcu; 6453 } 6454 6455 /* convert a CU to a vCU */ 6456 static u8 cu_to_vcu(u32 cu) 6457 { 6458 return ilog2(cu); 6459 } 6460 6461 /* convert a vAU to an AU */ 6462 static u32 vau_to_au(u8 vau) 6463 { 6464 return 8 * (1 << vau); 6465 } 6466 6467 static void set_linkup_defaults(struct hfi1_pportdata *ppd) 6468 { 6469 ppd->sm_trap_qp = 0x0; 6470 ppd->sa_qp = 0x1; 6471 } 6472 6473 /* 6474 * Graceful LCB shutdown. This leaves the LCB FIFOs in reset. 6475 */ 6476 static void lcb_shutdown(struct hfi1_devdata *dd, int abort) 6477 { 6478 u64 reg; 6479 6480 /* clear lcb run: LCB_CFG_RUN.EN = 0 */ 6481 write_csr(dd, DC_LCB_CFG_RUN, 0); 6482 /* set tx fifo reset: LCB_CFG_TX_FIFOS_RESET.VAL = 1 */ 6483 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 6484 1ull << DC_LCB_CFG_TX_FIFOS_RESET_VAL_SHIFT); 6485 /* set dcc reset csr: DCC_CFG_RESET.{reset_lcb,reset_rx_fpe} = 1 */ 6486 dd->lcb_err_en = read_csr(dd, DC_LCB_ERR_EN); 6487 reg = read_csr(dd, DCC_CFG_RESET); 6488 write_csr(dd, DCC_CFG_RESET, reg | 6489 DCC_CFG_RESET_RESET_LCB | DCC_CFG_RESET_RESET_RX_FPE); 6490 (void)read_csr(dd, DCC_CFG_RESET); /* make sure the write completed */ 6491 if (!abort) { 6492 udelay(1); /* must hold for the longer of 16cclks or 20ns */ 6493 write_csr(dd, DCC_CFG_RESET, reg); 6494 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en); 6495 } 6496 } 6497 6498 /* 6499 * This routine should be called after the link has been transitioned to 6500 * OFFLINE (OFFLINE state has the side effect of putting the SerDes into 6501 * reset). 6502 * 6503 * The expectation is that the caller of this routine would have taken 6504 * care of properly transitioning the link into the correct state. 6505 * NOTE: the caller needs to acquire the dd->dc8051_lock lock 6506 * before calling this function. 6507 */ 6508 static void _dc_shutdown(struct hfi1_devdata *dd) 6509 { 6510 lockdep_assert_held(&dd->dc8051_lock); 6511 6512 if (dd->dc_shutdown) 6513 return; 6514 6515 dd->dc_shutdown = 1; 6516 /* Shutdown the LCB */ 6517 lcb_shutdown(dd, 1); 6518 /* 6519 * Going to OFFLINE would have causes the 8051 to put the 6520 * SerDes into reset already. Just need to shut down the 8051, 6521 * itself. 6522 */ 6523 write_csr(dd, DC_DC8051_CFG_RST, 0x1); 6524 } 6525 6526 static void dc_shutdown(struct hfi1_devdata *dd) 6527 { 6528 mutex_lock(&dd->dc8051_lock); 6529 _dc_shutdown(dd); 6530 mutex_unlock(&dd->dc8051_lock); 6531 } 6532 6533 /* 6534 * Calling this after the DC has been brought out of reset should not 6535 * do any damage. 6536 * NOTE: the caller needs to acquire the dd->dc8051_lock lock 6537 * before calling this function. 6538 */ 6539 static void _dc_start(struct hfi1_devdata *dd) 6540 { 6541 lockdep_assert_held(&dd->dc8051_lock); 6542 6543 if (!dd->dc_shutdown) 6544 return; 6545 6546 /* Take the 8051 out of reset */ 6547 write_csr(dd, DC_DC8051_CFG_RST, 0ull); 6548 /* Wait until 8051 is ready */ 6549 if (wait_fm_ready(dd, TIMEOUT_8051_START)) 6550 dd_dev_err(dd, "%s: timeout starting 8051 firmware\n", 6551 __func__); 6552 6553 /* Take away reset for LCB and RX FPE (set in lcb_shutdown). */ 6554 write_csr(dd, DCC_CFG_RESET, LCB_RX_FPE_TX_FPE_OUT_OF_RESET); 6555 /* lcb_shutdown() with abort=1 does not restore these */ 6556 write_csr(dd, DC_LCB_ERR_EN, dd->lcb_err_en); 6557 dd->dc_shutdown = 0; 6558 } 6559 6560 static void dc_start(struct hfi1_devdata *dd) 6561 { 6562 mutex_lock(&dd->dc8051_lock); 6563 _dc_start(dd); 6564 mutex_unlock(&dd->dc8051_lock); 6565 } 6566 6567 /* 6568 * These LCB adjustments are for the Aurora SerDes core in the FPGA. 6569 */ 6570 static void adjust_lcb_for_fpga_serdes(struct hfi1_devdata *dd) 6571 { 6572 u64 rx_radr, tx_radr; 6573 u32 version; 6574 6575 if (dd->icode != ICODE_FPGA_EMULATION) 6576 return; 6577 6578 /* 6579 * These LCB defaults on emulator _s are good, nothing to do here: 6580 * LCB_CFG_TX_FIFOS_RADR 6581 * LCB_CFG_RX_FIFOS_RADR 6582 * LCB_CFG_LN_DCLK 6583 * LCB_CFG_IGNORE_LOST_RCLK 6584 */ 6585 if (is_emulator_s(dd)) 6586 return; 6587 /* else this is _p */ 6588 6589 version = emulator_rev(dd); 6590 if (!is_ax(dd)) 6591 version = 0x2d; /* all B0 use 0x2d or higher settings */ 6592 6593 if (version <= 0x12) { 6594 /* release 0x12 and below */ 6595 6596 /* 6597 * LCB_CFG_RX_FIFOS_RADR.RST_VAL = 0x9 6598 * LCB_CFG_RX_FIFOS_RADR.OK_TO_JUMP_VAL = 0x9 6599 * LCB_CFG_RX_FIFOS_RADR.DO_NOT_JUMP_VAL = 0xa 6600 */ 6601 rx_radr = 6602 0xaull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6603 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6604 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6605 /* 6606 * LCB_CFG_TX_FIFOS_RADR.ON_REINIT = 0 (default) 6607 * LCB_CFG_TX_FIFOS_RADR.RST_VAL = 6 6608 */ 6609 tx_radr = 6ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6610 } else if (version <= 0x18) { 6611 /* release 0x13 up to 0x18 */ 6612 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */ 6613 rx_radr = 6614 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6615 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6616 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6617 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6618 } else if (version == 0x19) { 6619 /* release 0x19 */ 6620 /* LCB_CFG_RX_FIFOS_RADR = 0xa99 */ 6621 rx_radr = 6622 0xAull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6623 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6624 | 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6625 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6626 } else if (version == 0x1a) { 6627 /* release 0x1a */ 6628 /* LCB_CFG_RX_FIFOS_RADR = 0x988 */ 6629 rx_radr = 6630 0x9ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6631 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6632 | 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6633 tx_radr = 7ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6634 write_csr(dd, DC_LCB_CFG_LN_DCLK, 1ull); 6635 } else { 6636 /* release 0x1b and higher */ 6637 /* LCB_CFG_RX_FIFOS_RADR = 0x877 */ 6638 rx_radr = 6639 0x8ull << DC_LCB_CFG_RX_FIFOS_RADR_DO_NOT_JUMP_VAL_SHIFT 6640 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_OK_TO_JUMP_VAL_SHIFT 6641 | 0x7ull << DC_LCB_CFG_RX_FIFOS_RADR_RST_VAL_SHIFT; 6642 tx_radr = 3ull << DC_LCB_CFG_TX_FIFOS_RADR_RST_VAL_SHIFT; 6643 } 6644 6645 write_csr(dd, DC_LCB_CFG_RX_FIFOS_RADR, rx_radr); 6646 /* LCB_CFG_IGNORE_LOST_RCLK.EN = 1 */ 6647 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK, 6648 DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK); 6649 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RADR, tx_radr); 6650 } 6651 6652 /* 6653 * Handle a SMA idle message 6654 * 6655 * This is a work-queue function outside of the interrupt. 6656 */ 6657 void handle_sma_message(struct work_struct *work) 6658 { 6659 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 6660 sma_message_work); 6661 struct hfi1_devdata *dd = ppd->dd; 6662 u64 msg; 6663 int ret; 6664 6665 /* 6666 * msg is bytes 1-4 of the 40-bit idle message - the command code 6667 * is stripped off 6668 */ 6669 ret = read_idle_sma(dd, &msg); 6670 if (ret) 6671 return; 6672 dd_dev_info(dd, "%s: SMA message 0x%llx\n", __func__, msg); 6673 /* 6674 * React to the SMA message. Byte[1] (0 for us) is the command. 6675 */ 6676 switch (msg & 0xff) { 6677 case SMA_IDLE_ARM: 6678 /* 6679 * See OPAv1 table 9-14 - HFI and External Switch Ports Key 6680 * State Transitions 6681 * 6682 * Only expected in INIT or ARMED, discard otherwise. 6683 */ 6684 if (ppd->host_link_state & (HLS_UP_INIT | HLS_UP_ARMED)) 6685 ppd->neighbor_normal = 1; 6686 break; 6687 case SMA_IDLE_ACTIVE: 6688 /* 6689 * See OPAv1 table 9-14 - HFI and External Switch Ports Key 6690 * State Transitions 6691 * 6692 * Can activate the node. Discard otherwise. 6693 */ 6694 if (ppd->host_link_state == HLS_UP_ARMED && 6695 ppd->is_active_optimize_enabled) { 6696 ppd->neighbor_normal = 1; 6697 ret = set_link_state(ppd, HLS_UP_ACTIVE); 6698 if (ret) 6699 dd_dev_err( 6700 dd, 6701 "%s: received Active SMA idle message, couldn't set link to Active\n", 6702 __func__); 6703 } 6704 break; 6705 default: 6706 dd_dev_err(dd, 6707 "%s: received unexpected SMA idle message 0x%llx\n", 6708 __func__, msg); 6709 break; 6710 } 6711 } 6712 6713 static void adjust_rcvctrl(struct hfi1_devdata *dd, u64 add, u64 clear) 6714 { 6715 u64 rcvctrl; 6716 unsigned long flags; 6717 6718 spin_lock_irqsave(&dd->rcvctrl_lock, flags); 6719 rcvctrl = read_csr(dd, RCV_CTRL); 6720 rcvctrl |= add; 6721 rcvctrl &= ~clear; 6722 write_csr(dd, RCV_CTRL, rcvctrl); 6723 spin_unlock_irqrestore(&dd->rcvctrl_lock, flags); 6724 } 6725 6726 static inline void add_rcvctrl(struct hfi1_devdata *dd, u64 add) 6727 { 6728 adjust_rcvctrl(dd, add, 0); 6729 } 6730 6731 static inline void clear_rcvctrl(struct hfi1_devdata *dd, u64 clear) 6732 { 6733 adjust_rcvctrl(dd, 0, clear); 6734 } 6735 6736 /* 6737 * Called from all interrupt handlers to start handling an SPC freeze. 6738 */ 6739 void start_freeze_handling(struct hfi1_pportdata *ppd, int flags) 6740 { 6741 struct hfi1_devdata *dd = ppd->dd; 6742 struct send_context *sc; 6743 int i; 6744 int sc_flags; 6745 6746 if (flags & FREEZE_SELF) 6747 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK); 6748 6749 /* enter frozen mode */ 6750 dd->flags |= HFI1_FROZEN; 6751 6752 /* notify all SDMA engines that they are going into a freeze */ 6753 sdma_freeze_notify(dd, !!(flags & FREEZE_LINK_DOWN)); 6754 6755 sc_flags = SCF_FROZEN | SCF_HALTED | (flags & FREEZE_LINK_DOWN ? 6756 SCF_LINK_DOWN : 0); 6757 /* do halt pre-handling on all enabled send contexts */ 6758 for (i = 0; i < dd->num_send_contexts; i++) { 6759 sc = dd->send_contexts[i].sc; 6760 if (sc && (sc->flags & SCF_ENABLED)) 6761 sc_stop(sc, sc_flags); 6762 } 6763 6764 /* Send context are frozen. Notify user space */ 6765 hfi1_set_uevent_bits(ppd, _HFI1_EVENT_FROZEN_BIT); 6766 6767 if (flags & FREEZE_ABORT) { 6768 dd_dev_err(dd, 6769 "Aborted freeze recovery. Please REBOOT system\n"); 6770 return; 6771 } 6772 /* queue non-interrupt handler */ 6773 queue_work(ppd->hfi1_wq, &ppd->freeze_work); 6774 } 6775 6776 /* 6777 * Wait until all 4 sub-blocks indicate that they have frozen or unfrozen, 6778 * depending on the "freeze" parameter. 6779 * 6780 * No need to return an error if it times out, our only option 6781 * is to proceed anyway. 6782 */ 6783 static void wait_for_freeze_status(struct hfi1_devdata *dd, int freeze) 6784 { 6785 unsigned long timeout; 6786 u64 reg; 6787 6788 timeout = jiffies + msecs_to_jiffies(FREEZE_STATUS_TIMEOUT); 6789 while (1) { 6790 reg = read_csr(dd, CCE_STATUS); 6791 if (freeze) { 6792 /* waiting until all indicators are set */ 6793 if ((reg & ALL_FROZE) == ALL_FROZE) 6794 return; /* all done */ 6795 } else { 6796 /* waiting until all indicators are clear */ 6797 if ((reg & ALL_FROZE) == 0) 6798 return; /* all done */ 6799 } 6800 6801 if (time_after(jiffies, timeout)) { 6802 dd_dev_err(dd, 6803 "Time out waiting for SPC %sfreeze, bits 0x%llx, expecting 0x%llx, continuing", 6804 freeze ? "" : "un", reg & ALL_FROZE, 6805 freeze ? ALL_FROZE : 0ull); 6806 return; 6807 } 6808 usleep_range(80, 120); 6809 } 6810 } 6811 6812 /* 6813 * Do all freeze handling for the RXE block. 6814 */ 6815 static void rxe_freeze(struct hfi1_devdata *dd) 6816 { 6817 int i; 6818 struct hfi1_ctxtdata *rcd; 6819 6820 /* disable port */ 6821 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 6822 6823 /* disable all receive contexts */ 6824 for (i = 0; i < dd->num_rcv_contexts; i++) { 6825 rcd = hfi1_rcd_get_by_index(dd, i); 6826 hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS, rcd); 6827 hfi1_rcd_put(rcd); 6828 } 6829 } 6830 6831 /* 6832 * Unfreeze handling for the RXE block - kernel contexts only. 6833 * This will also enable the port. User contexts will do unfreeze 6834 * handling on a per-context basis as they call into the driver. 6835 * 6836 */ 6837 static void rxe_kernel_unfreeze(struct hfi1_devdata *dd) 6838 { 6839 u32 rcvmask; 6840 u16 i; 6841 struct hfi1_ctxtdata *rcd; 6842 6843 /* enable all kernel contexts */ 6844 for (i = 0; i < dd->num_rcv_contexts; i++) { 6845 rcd = hfi1_rcd_get_by_index(dd, i); 6846 6847 /* Ensure all non-user contexts(including vnic) are enabled */ 6848 if (!rcd || 6849 (i >= dd->first_dyn_alloc_ctxt && !rcd->is_vnic)) { 6850 hfi1_rcd_put(rcd); 6851 continue; 6852 } 6853 rcvmask = HFI1_RCVCTRL_CTXT_ENB; 6854 /* HFI1_RCVCTRL_TAILUPD_[ENB|DIS] needs to be set explicitly */ 6855 rcvmask |= hfi1_rcvhdrtail_kvaddr(rcd) ? 6856 HFI1_RCVCTRL_TAILUPD_ENB : HFI1_RCVCTRL_TAILUPD_DIS; 6857 hfi1_rcvctrl(dd, rcvmask, rcd); 6858 hfi1_rcd_put(rcd); 6859 } 6860 6861 /* enable port */ 6862 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 6863 } 6864 6865 /* 6866 * Non-interrupt SPC freeze handling. 6867 * 6868 * This is a work-queue function outside of the triggering interrupt. 6869 */ 6870 void handle_freeze(struct work_struct *work) 6871 { 6872 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 6873 freeze_work); 6874 struct hfi1_devdata *dd = ppd->dd; 6875 6876 /* wait for freeze indicators on all affected blocks */ 6877 wait_for_freeze_status(dd, 1); 6878 6879 /* SPC is now frozen */ 6880 6881 /* do send PIO freeze steps */ 6882 pio_freeze(dd); 6883 6884 /* do send DMA freeze steps */ 6885 sdma_freeze(dd); 6886 6887 /* do send egress freeze steps - nothing to do */ 6888 6889 /* do receive freeze steps */ 6890 rxe_freeze(dd); 6891 6892 /* 6893 * Unfreeze the hardware - clear the freeze, wait for each 6894 * block's frozen bit to clear, then clear the frozen flag. 6895 */ 6896 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK); 6897 wait_for_freeze_status(dd, 0); 6898 6899 if (is_ax(dd)) { 6900 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_FREEZE_SMASK); 6901 wait_for_freeze_status(dd, 1); 6902 write_csr(dd, CCE_CTRL, CCE_CTRL_SPC_UNFREEZE_SMASK); 6903 wait_for_freeze_status(dd, 0); 6904 } 6905 6906 /* do send PIO unfreeze steps for kernel contexts */ 6907 pio_kernel_unfreeze(dd); 6908 6909 /* do send DMA unfreeze steps */ 6910 sdma_unfreeze(dd); 6911 6912 /* do send egress unfreeze steps - nothing to do */ 6913 6914 /* do receive unfreeze steps for kernel contexts */ 6915 rxe_kernel_unfreeze(dd); 6916 6917 /* 6918 * The unfreeze procedure touches global device registers when 6919 * it disables and re-enables RXE. Mark the device unfrozen 6920 * after all that is done so other parts of the driver waiting 6921 * for the device to unfreeze don't do things out of order. 6922 * 6923 * The above implies that the meaning of HFI1_FROZEN flag is 6924 * "Device has gone into freeze mode and freeze mode handling 6925 * is still in progress." 6926 * 6927 * The flag will be removed when freeze mode processing has 6928 * completed. 6929 */ 6930 dd->flags &= ~HFI1_FROZEN; 6931 wake_up(&dd->event_queue); 6932 6933 /* no longer frozen */ 6934 } 6935 6936 /** 6937 * update_xmit_counters - update PortXmitWait/PortVlXmitWait 6938 * counters. 6939 * @ppd: info of physical Hfi port 6940 * @link_width: new link width after link up or downgrade 6941 * 6942 * Update the PortXmitWait and PortVlXmitWait counters after 6943 * a link up or downgrade event to reflect a link width change. 6944 */ 6945 static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width) 6946 { 6947 int i; 6948 u16 tx_width; 6949 u16 link_speed; 6950 6951 tx_width = tx_link_width(link_width); 6952 link_speed = get_link_speed(ppd->link_speed_active); 6953 6954 /* 6955 * There are C_VL_COUNT number of PortVLXmitWait counters. 6956 * Adding 1 to C_VL_COUNT to include the PortXmitWait counter. 6957 */ 6958 for (i = 0; i < C_VL_COUNT + 1; i++) 6959 get_xmit_wait_counters(ppd, tx_width, link_speed, i); 6960 } 6961 6962 /* 6963 * Handle a link up interrupt from the 8051. 6964 * 6965 * This is a work-queue function outside of the interrupt. 6966 */ 6967 void handle_link_up(struct work_struct *work) 6968 { 6969 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 6970 link_up_work); 6971 struct hfi1_devdata *dd = ppd->dd; 6972 6973 set_link_state(ppd, HLS_UP_INIT); 6974 6975 /* cache the read of DC_LCB_STS_ROUND_TRIP_LTP_CNT */ 6976 read_ltp_rtt(dd); 6977 /* 6978 * OPA specifies that certain counters are cleared on a transition 6979 * to link up, so do that. 6980 */ 6981 clear_linkup_counters(dd); 6982 /* 6983 * And (re)set link up default values. 6984 */ 6985 set_linkup_defaults(ppd); 6986 6987 /* 6988 * Set VL15 credits. Use cached value from verify cap interrupt. 6989 * In case of quick linkup or simulator, vl15 value will be set by 6990 * handle_linkup_change. VerifyCap interrupt handler will not be 6991 * called in those scenarios. 6992 */ 6993 if (!(quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR)) 6994 set_up_vl15(dd, dd->vl15buf_cached); 6995 6996 /* enforce link speed enabled */ 6997 if ((ppd->link_speed_active & ppd->link_speed_enabled) == 0) { 6998 /* oops - current speed is not enabled, bounce */ 6999 dd_dev_err(dd, 7000 "Link speed active 0x%x is outside enabled 0x%x, downing link\n", 7001 ppd->link_speed_active, ppd->link_speed_enabled); 7002 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_SPEED_POLICY, 0, 7003 OPA_LINKDOWN_REASON_SPEED_POLICY); 7004 set_link_state(ppd, HLS_DN_OFFLINE); 7005 start_link(ppd); 7006 } 7007 } 7008 7009 /* 7010 * Several pieces of LNI information were cached for SMA in ppd. 7011 * Reset these on link down 7012 */ 7013 static void reset_neighbor_info(struct hfi1_pportdata *ppd) 7014 { 7015 ppd->neighbor_guid = 0; 7016 ppd->neighbor_port_number = 0; 7017 ppd->neighbor_type = 0; 7018 ppd->neighbor_fm_security = 0; 7019 } 7020 7021 static const char * const link_down_reason_strs[] = { 7022 [OPA_LINKDOWN_REASON_NONE] = "None", 7023 [OPA_LINKDOWN_REASON_RCV_ERROR_0] = "Receive error 0", 7024 [OPA_LINKDOWN_REASON_BAD_PKT_LEN] = "Bad packet length", 7025 [OPA_LINKDOWN_REASON_PKT_TOO_LONG] = "Packet too long", 7026 [OPA_LINKDOWN_REASON_PKT_TOO_SHORT] = "Packet too short", 7027 [OPA_LINKDOWN_REASON_BAD_SLID] = "Bad SLID", 7028 [OPA_LINKDOWN_REASON_BAD_DLID] = "Bad DLID", 7029 [OPA_LINKDOWN_REASON_BAD_L2] = "Bad L2", 7030 [OPA_LINKDOWN_REASON_BAD_SC] = "Bad SC", 7031 [OPA_LINKDOWN_REASON_RCV_ERROR_8] = "Receive error 8", 7032 [OPA_LINKDOWN_REASON_BAD_MID_TAIL] = "Bad mid tail", 7033 [OPA_LINKDOWN_REASON_RCV_ERROR_10] = "Receive error 10", 7034 [OPA_LINKDOWN_REASON_PREEMPT_ERROR] = "Preempt error", 7035 [OPA_LINKDOWN_REASON_PREEMPT_VL15] = "Preempt vl15", 7036 [OPA_LINKDOWN_REASON_BAD_VL_MARKER] = "Bad VL marker", 7037 [OPA_LINKDOWN_REASON_RCV_ERROR_14] = "Receive error 14", 7038 [OPA_LINKDOWN_REASON_RCV_ERROR_15] = "Receive error 15", 7039 [OPA_LINKDOWN_REASON_BAD_HEAD_DIST] = "Bad head distance", 7040 [OPA_LINKDOWN_REASON_BAD_TAIL_DIST] = "Bad tail distance", 7041 [OPA_LINKDOWN_REASON_BAD_CTRL_DIST] = "Bad control distance", 7042 [OPA_LINKDOWN_REASON_BAD_CREDIT_ACK] = "Bad credit ack", 7043 [OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER] = "Unsupported VL marker", 7044 [OPA_LINKDOWN_REASON_BAD_PREEMPT] = "Bad preempt", 7045 [OPA_LINKDOWN_REASON_BAD_CONTROL_FLIT] = "Bad control flit", 7046 [OPA_LINKDOWN_REASON_EXCEED_MULTICAST_LIMIT] = "Exceed multicast limit", 7047 [OPA_LINKDOWN_REASON_RCV_ERROR_24] = "Receive error 24", 7048 [OPA_LINKDOWN_REASON_RCV_ERROR_25] = "Receive error 25", 7049 [OPA_LINKDOWN_REASON_RCV_ERROR_26] = "Receive error 26", 7050 [OPA_LINKDOWN_REASON_RCV_ERROR_27] = "Receive error 27", 7051 [OPA_LINKDOWN_REASON_RCV_ERROR_28] = "Receive error 28", 7052 [OPA_LINKDOWN_REASON_RCV_ERROR_29] = "Receive error 29", 7053 [OPA_LINKDOWN_REASON_RCV_ERROR_30] = "Receive error 30", 7054 [OPA_LINKDOWN_REASON_EXCESSIVE_BUFFER_OVERRUN] = 7055 "Excessive buffer overrun", 7056 [OPA_LINKDOWN_REASON_UNKNOWN] = "Unknown", 7057 [OPA_LINKDOWN_REASON_REBOOT] = "Reboot", 7058 [OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN] = "Neighbor unknown", 7059 [OPA_LINKDOWN_REASON_FM_BOUNCE] = "FM bounce", 7060 [OPA_LINKDOWN_REASON_SPEED_POLICY] = "Speed policy", 7061 [OPA_LINKDOWN_REASON_WIDTH_POLICY] = "Width policy", 7062 [OPA_LINKDOWN_REASON_DISCONNECTED] = "Disconnected", 7063 [OPA_LINKDOWN_REASON_LOCAL_MEDIA_NOT_INSTALLED] = 7064 "Local media not installed", 7065 [OPA_LINKDOWN_REASON_NOT_INSTALLED] = "Not installed", 7066 [OPA_LINKDOWN_REASON_CHASSIS_CONFIG] = "Chassis config", 7067 [OPA_LINKDOWN_REASON_END_TO_END_NOT_INSTALLED] = 7068 "End to end not installed", 7069 [OPA_LINKDOWN_REASON_POWER_POLICY] = "Power policy", 7070 [OPA_LINKDOWN_REASON_LINKSPEED_POLICY] = "Link speed policy", 7071 [OPA_LINKDOWN_REASON_LINKWIDTH_POLICY] = "Link width policy", 7072 [OPA_LINKDOWN_REASON_SWITCH_MGMT] = "Switch management", 7073 [OPA_LINKDOWN_REASON_SMA_DISABLED] = "SMA disabled", 7074 [OPA_LINKDOWN_REASON_TRANSIENT] = "Transient" 7075 }; 7076 7077 /* return the neighbor link down reason string */ 7078 static const char *link_down_reason_str(u8 reason) 7079 { 7080 const char *str = NULL; 7081 7082 if (reason < ARRAY_SIZE(link_down_reason_strs)) 7083 str = link_down_reason_strs[reason]; 7084 if (!str) 7085 str = "(invalid)"; 7086 7087 return str; 7088 } 7089 7090 /* 7091 * Handle a link down interrupt from the 8051. 7092 * 7093 * This is a work-queue function outside of the interrupt. 7094 */ 7095 void handle_link_down(struct work_struct *work) 7096 { 7097 u8 lcl_reason, neigh_reason = 0; 7098 u8 link_down_reason; 7099 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 7100 link_down_work); 7101 int was_up; 7102 static const char ldr_str[] = "Link down reason: "; 7103 7104 if ((ppd->host_link_state & 7105 (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) && 7106 ppd->port_type == PORT_TYPE_FIXED) 7107 ppd->offline_disabled_reason = 7108 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NOT_INSTALLED); 7109 7110 /* Go offline first, then deal with reading/writing through 8051 */ 7111 was_up = !!(ppd->host_link_state & HLS_UP); 7112 set_link_state(ppd, HLS_DN_OFFLINE); 7113 xchg(&ppd->is_link_down_queued, 0); 7114 7115 if (was_up) { 7116 lcl_reason = 0; 7117 /* link down reason is only valid if the link was up */ 7118 read_link_down_reason(ppd->dd, &link_down_reason); 7119 switch (link_down_reason) { 7120 case LDR_LINK_TRANSFER_ACTIVE_LOW: 7121 /* the link went down, no idle message reason */ 7122 dd_dev_info(ppd->dd, "%sUnexpected link down\n", 7123 ldr_str); 7124 break; 7125 case LDR_RECEIVED_LINKDOWN_IDLE_MSG: 7126 /* 7127 * The neighbor reason is only valid if an idle message 7128 * was received for it. 7129 */ 7130 read_planned_down_reason_code(ppd->dd, &neigh_reason); 7131 dd_dev_info(ppd->dd, 7132 "%sNeighbor link down message %d, %s\n", 7133 ldr_str, neigh_reason, 7134 link_down_reason_str(neigh_reason)); 7135 break; 7136 case LDR_RECEIVED_HOST_OFFLINE_REQ: 7137 dd_dev_info(ppd->dd, 7138 "%sHost requested link to go offline\n", 7139 ldr_str); 7140 break; 7141 default: 7142 dd_dev_info(ppd->dd, "%sUnknown reason 0x%x\n", 7143 ldr_str, link_down_reason); 7144 break; 7145 } 7146 7147 /* 7148 * If no reason, assume peer-initiated but missed 7149 * LinkGoingDown idle flits. 7150 */ 7151 if (neigh_reason == 0) 7152 lcl_reason = OPA_LINKDOWN_REASON_NEIGHBOR_UNKNOWN; 7153 } else { 7154 /* went down while polling or going up */ 7155 lcl_reason = OPA_LINKDOWN_REASON_TRANSIENT; 7156 } 7157 7158 set_link_down_reason(ppd, lcl_reason, neigh_reason, 0); 7159 7160 /* inform the SMA when the link transitions from up to down */ 7161 if (was_up && ppd->local_link_down_reason.sma == 0 && 7162 ppd->neigh_link_down_reason.sma == 0) { 7163 ppd->local_link_down_reason.sma = 7164 ppd->local_link_down_reason.latest; 7165 ppd->neigh_link_down_reason.sma = 7166 ppd->neigh_link_down_reason.latest; 7167 } 7168 7169 reset_neighbor_info(ppd); 7170 7171 /* disable the port */ 7172 clear_rcvctrl(ppd->dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 7173 7174 /* 7175 * If there is no cable attached, turn the DC off. Otherwise, 7176 * start the link bring up. 7177 */ 7178 if (ppd->port_type == PORT_TYPE_QSFP && !qsfp_mod_present(ppd)) 7179 dc_shutdown(ppd->dd); 7180 else 7181 start_link(ppd); 7182 } 7183 7184 void handle_link_bounce(struct work_struct *work) 7185 { 7186 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 7187 link_bounce_work); 7188 7189 /* 7190 * Only do something if the link is currently up. 7191 */ 7192 if (ppd->host_link_state & HLS_UP) { 7193 set_link_state(ppd, HLS_DN_OFFLINE); 7194 start_link(ppd); 7195 } else { 7196 dd_dev_info(ppd->dd, "%s: link not up (%s), nothing to do\n", 7197 __func__, link_state_name(ppd->host_link_state)); 7198 } 7199 } 7200 7201 /* 7202 * Mask conversion: Capability exchange to Port LTP. The capability 7203 * exchange has an implicit 16b CRC that is mandatory. 7204 */ 7205 static int cap_to_port_ltp(int cap) 7206 { 7207 int port_ltp = PORT_LTP_CRC_MODE_16; /* this mode is mandatory */ 7208 7209 if (cap & CAP_CRC_14B) 7210 port_ltp |= PORT_LTP_CRC_MODE_14; 7211 if (cap & CAP_CRC_48B) 7212 port_ltp |= PORT_LTP_CRC_MODE_48; 7213 if (cap & CAP_CRC_12B_16B_PER_LANE) 7214 port_ltp |= PORT_LTP_CRC_MODE_PER_LANE; 7215 7216 return port_ltp; 7217 } 7218 7219 /* 7220 * Convert an OPA Port LTP mask to capability mask 7221 */ 7222 int port_ltp_to_cap(int port_ltp) 7223 { 7224 int cap_mask = 0; 7225 7226 if (port_ltp & PORT_LTP_CRC_MODE_14) 7227 cap_mask |= CAP_CRC_14B; 7228 if (port_ltp & PORT_LTP_CRC_MODE_48) 7229 cap_mask |= CAP_CRC_48B; 7230 if (port_ltp & PORT_LTP_CRC_MODE_PER_LANE) 7231 cap_mask |= CAP_CRC_12B_16B_PER_LANE; 7232 7233 return cap_mask; 7234 } 7235 7236 /* 7237 * Convert a single DC LCB CRC mode to an OPA Port LTP mask. 7238 */ 7239 static int lcb_to_port_ltp(int lcb_crc) 7240 { 7241 int port_ltp = 0; 7242 7243 if (lcb_crc == LCB_CRC_12B_16B_PER_LANE) 7244 port_ltp = PORT_LTP_CRC_MODE_PER_LANE; 7245 else if (lcb_crc == LCB_CRC_48B) 7246 port_ltp = PORT_LTP_CRC_MODE_48; 7247 else if (lcb_crc == LCB_CRC_14B) 7248 port_ltp = PORT_LTP_CRC_MODE_14; 7249 else 7250 port_ltp = PORT_LTP_CRC_MODE_16; 7251 7252 return port_ltp; 7253 } 7254 7255 static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd) 7256 { 7257 if (ppd->pkeys[2] != 0) { 7258 ppd->pkeys[2] = 0; 7259 (void)hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_PKEYS, 0); 7260 hfi1_event_pkey_change(ppd->dd, ppd->port); 7261 } 7262 } 7263 7264 /* 7265 * Convert the given link width to the OPA link width bitmask. 7266 */ 7267 static u16 link_width_to_bits(struct hfi1_devdata *dd, u16 width) 7268 { 7269 switch (width) { 7270 case 0: 7271 /* 7272 * Simulator and quick linkup do not set the width. 7273 * Just set it to 4x without complaint. 7274 */ 7275 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR || quick_linkup) 7276 return OPA_LINK_WIDTH_4X; 7277 return 0; /* no lanes up */ 7278 case 1: return OPA_LINK_WIDTH_1X; 7279 case 2: return OPA_LINK_WIDTH_2X; 7280 case 3: return OPA_LINK_WIDTH_3X; 7281 case 4: return OPA_LINK_WIDTH_4X; 7282 default: 7283 dd_dev_info(dd, "%s: invalid width %d, using 4\n", 7284 __func__, width); 7285 return OPA_LINK_WIDTH_4X; 7286 } 7287 } 7288 7289 /* 7290 * Do a population count on the bottom nibble. 7291 */ 7292 static const u8 bit_counts[16] = { 7293 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4 7294 }; 7295 7296 static inline u8 nibble_to_count(u8 nibble) 7297 { 7298 return bit_counts[nibble & 0xf]; 7299 } 7300 7301 /* 7302 * Read the active lane information from the 8051 registers and return 7303 * their widths. 7304 * 7305 * Active lane information is found in these 8051 registers: 7306 * enable_lane_tx 7307 * enable_lane_rx 7308 */ 7309 static void get_link_widths(struct hfi1_devdata *dd, u16 *tx_width, 7310 u16 *rx_width) 7311 { 7312 u16 tx, rx; 7313 u8 enable_lane_rx; 7314 u8 enable_lane_tx; 7315 u8 tx_polarity_inversion; 7316 u8 rx_polarity_inversion; 7317 u8 max_rate; 7318 7319 /* read the active lanes */ 7320 read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion, 7321 &rx_polarity_inversion, &max_rate); 7322 read_local_lni(dd, &enable_lane_rx); 7323 7324 /* convert to counts */ 7325 tx = nibble_to_count(enable_lane_tx); 7326 rx = nibble_to_count(enable_lane_rx); 7327 7328 /* 7329 * Set link_speed_active here, overriding what was set in 7330 * handle_verify_cap(). The ASIC 8051 firmware does not correctly 7331 * set the max_rate field in handle_verify_cap until v0.19. 7332 */ 7333 if ((dd->icode == ICODE_RTL_SILICON) && 7334 (dd->dc8051_ver < dc8051_ver(0, 19, 0))) { 7335 /* max_rate: 0 = 12.5G, 1 = 25G */ 7336 switch (max_rate) { 7337 case 0: 7338 dd->pport[0].link_speed_active = OPA_LINK_SPEED_12_5G; 7339 break; 7340 case 1: 7341 dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G; 7342 break; 7343 default: 7344 dd_dev_err(dd, 7345 "%s: unexpected max rate %d, using 25Gb\n", 7346 __func__, (int)max_rate); 7347 dd->pport[0].link_speed_active = OPA_LINK_SPEED_25G; 7348 break; 7349 } 7350 } 7351 7352 dd_dev_info(dd, 7353 "Fabric active lanes (width): tx 0x%x (%d), rx 0x%x (%d)\n", 7354 enable_lane_tx, tx, enable_lane_rx, rx); 7355 *tx_width = link_width_to_bits(dd, tx); 7356 *rx_width = link_width_to_bits(dd, rx); 7357 } 7358 7359 /* 7360 * Read verify_cap_local_fm_link_width[1] to obtain the link widths. 7361 * Valid after the end of VerifyCap and during LinkUp. Does not change 7362 * after link up. I.e. look elsewhere for downgrade information. 7363 * 7364 * Bits are: 7365 * + bits [7:4] contain the number of active transmitters 7366 * + bits [3:0] contain the number of active receivers 7367 * These are numbers 1 through 4 and can be different values if the 7368 * link is asymmetric. 7369 * 7370 * verify_cap_local_fm_link_width[0] retains its original value. 7371 */ 7372 static void get_linkup_widths(struct hfi1_devdata *dd, u16 *tx_width, 7373 u16 *rx_width) 7374 { 7375 u16 widths, tx, rx; 7376 u8 misc_bits, local_flags; 7377 u16 active_tx, active_rx; 7378 7379 read_vc_local_link_mode(dd, &misc_bits, &local_flags, &widths); 7380 tx = widths >> 12; 7381 rx = (widths >> 8) & 0xf; 7382 7383 *tx_width = link_width_to_bits(dd, tx); 7384 *rx_width = link_width_to_bits(dd, rx); 7385 7386 /* print the active widths */ 7387 get_link_widths(dd, &active_tx, &active_rx); 7388 } 7389 7390 /* 7391 * Set ppd->link_width_active and ppd->link_width_downgrade_active using 7392 * hardware information when the link first comes up. 7393 * 7394 * The link width is not available until after VerifyCap.AllFramesReceived 7395 * (the trigger for handle_verify_cap), so this is outside that routine 7396 * and should be called when the 8051 signals linkup. 7397 */ 7398 void get_linkup_link_widths(struct hfi1_pportdata *ppd) 7399 { 7400 u16 tx_width, rx_width; 7401 7402 /* get end-of-LNI link widths */ 7403 get_linkup_widths(ppd->dd, &tx_width, &rx_width); 7404 7405 /* use tx_width as the link is supposed to be symmetric on link up */ 7406 ppd->link_width_active = tx_width; 7407 /* link width downgrade active (LWD.A) starts out matching LW.A */ 7408 ppd->link_width_downgrade_tx_active = ppd->link_width_active; 7409 ppd->link_width_downgrade_rx_active = ppd->link_width_active; 7410 /* per OPA spec, on link up LWD.E resets to LWD.S */ 7411 ppd->link_width_downgrade_enabled = ppd->link_width_downgrade_supported; 7412 /* cache the active egress rate (units {10^6 bits/sec]) */ 7413 ppd->current_egress_rate = active_egress_rate(ppd); 7414 } 7415 7416 /* 7417 * Handle a verify capabilities interrupt from the 8051. 7418 * 7419 * This is a work-queue function outside of the interrupt. 7420 */ 7421 void handle_verify_cap(struct work_struct *work) 7422 { 7423 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 7424 link_vc_work); 7425 struct hfi1_devdata *dd = ppd->dd; 7426 u64 reg; 7427 u8 power_management; 7428 u8 continuous; 7429 u8 vcu; 7430 u8 vau; 7431 u8 z; 7432 u16 vl15buf; 7433 u16 link_widths; 7434 u16 crc_mask; 7435 u16 crc_val; 7436 u16 device_id; 7437 u16 active_tx, active_rx; 7438 u8 partner_supported_crc; 7439 u8 remote_tx_rate; 7440 u8 device_rev; 7441 7442 set_link_state(ppd, HLS_VERIFY_CAP); 7443 7444 lcb_shutdown(dd, 0); 7445 adjust_lcb_for_fpga_serdes(dd); 7446 7447 read_vc_remote_phy(dd, &power_management, &continuous); 7448 read_vc_remote_fabric(dd, &vau, &z, &vcu, &vl15buf, 7449 &partner_supported_crc); 7450 read_vc_remote_link_width(dd, &remote_tx_rate, &link_widths); 7451 read_remote_device_id(dd, &device_id, &device_rev); 7452 7453 /* print the active widths */ 7454 get_link_widths(dd, &active_tx, &active_rx); 7455 dd_dev_info(dd, 7456 "Peer PHY: power management 0x%x, continuous updates 0x%x\n", 7457 (int)power_management, (int)continuous); 7458 dd_dev_info(dd, 7459 "Peer Fabric: vAU %d, Z %d, vCU %d, vl15 credits 0x%x, CRC sizes 0x%x\n", 7460 (int)vau, (int)z, (int)vcu, (int)vl15buf, 7461 (int)partner_supported_crc); 7462 dd_dev_info(dd, "Peer Link Width: tx rate 0x%x, widths 0x%x\n", 7463 (u32)remote_tx_rate, (u32)link_widths); 7464 dd_dev_info(dd, "Peer Device ID: 0x%04x, Revision 0x%02x\n", 7465 (u32)device_id, (u32)device_rev); 7466 /* 7467 * The peer vAU value just read is the peer receiver value. HFI does 7468 * not support a transmit vAU of 0 (AU == 8). We advertised that 7469 * with Z=1 in the fabric capabilities sent to the peer. The peer 7470 * will see our Z=1, and, if it advertised a vAU of 0, will move its 7471 * receive to vAU of 1 (AU == 16). Do the same here. We do not care 7472 * about the peer Z value - our sent vAU is 3 (hardwired) and is not 7473 * subject to the Z value exception. 7474 */ 7475 if (vau == 0) 7476 vau = 1; 7477 set_up_vau(dd, vau); 7478 7479 /* 7480 * Set VL15 credits to 0 in global credit register. Cache remote VL15 7481 * credits value and wait for link-up interrupt ot set it. 7482 */ 7483 set_up_vl15(dd, 0); 7484 dd->vl15buf_cached = vl15buf; 7485 7486 /* set up the LCB CRC mode */ 7487 crc_mask = ppd->port_crc_mode_enabled & partner_supported_crc; 7488 7489 /* order is important: use the lowest bit in common */ 7490 if (crc_mask & CAP_CRC_14B) 7491 crc_val = LCB_CRC_14B; 7492 else if (crc_mask & CAP_CRC_48B) 7493 crc_val = LCB_CRC_48B; 7494 else if (crc_mask & CAP_CRC_12B_16B_PER_LANE) 7495 crc_val = LCB_CRC_12B_16B_PER_LANE; 7496 else 7497 crc_val = LCB_CRC_16B; 7498 7499 dd_dev_info(dd, "Final LCB CRC mode: %d\n", (int)crc_val); 7500 write_csr(dd, DC_LCB_CFG_CRC_MODE, 7501 (u64)crc_val << DC_LCB_CFG_CRC_MODE_TX_VAL_SHIFT); 7502 7503 /* set (14b only) or clear sideband credit */ 7504 reg = read_csr(dd, SEND_CM_CTRL); 7505 if (crc_val == LCB_CRC_14B && crc_14b_sideband) { 7506 write_csr(dd, SEND_CM_CTRL, 7507 reg | SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK); 7508 } else { 7509 write_csr(dd, SEND_CM_CTRL, 7510 reg & ~SEND_CM_CTRL_FORCE_CREDIT_MODE_SMASK); 7511 } 7512 7513 ppd->link_speed_active = 0; /* invalid value */ 7514 if (dd->dc8051_ver < dc8051_ver(0, 20, 0)) { 7515 /* remote_tx_rate: 0 = 12.5G, 1 = 25G */ 7516 switch (remote_tx_rate) { 7517 case 0: 7518 ppd->link_speed_active = OPA_LINK_SPEED_12_5G; 7519 break; 7520 case 1: 7521 ppd->link_speed_active = OPA_LINK_SPEED_25G; 7522 break; 7523 } 7524 } else { 7525 /* actual rate is highest bit of the ANDed rates */ 7526 u8 rate = remote_tx_rate & ppd->local_tx_rate; 7527 7528 if (rate & 2) 7529 ppd->link_speed_active = OPA_LINK_SPEED_25G; 7530 else if (rate & 1) 7531 ppd->link_speed_active = OPA_LINK_SPEED_12_5G; 7532 } 7533 if (ppd->link_speed_active == 0) { 7534 dd_dev_err(dd, "%s: unexpected remote tx rate %d, using 25Gb\n", 7535 __func__, (int)remote_tx_rate); 7536 ppd->link_speed_active = OPA_LINK_SPEED_25G; 7537 } 7538 7539 /* 7540 * Cache the values of the supported, enabled, and active 7541 * LTP CRC modes to return in 'portinfo' queries. But the bit 7542 * flags that are returned in the portinfo query differ from 7543 * what's in the link_crc_mask, crc_sizes, and crc_val 7544 * variables. Convert these here. 7545 */ 7546 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8; 7547 /* supported crc modes */ 7548 ppd->port_ltp_crc_mode |= 7549 cap_to_port_ltp(ppd->port_crc_mode_enabled) << 4; 7550 /* enabled crc modes */ 7551 ppd->port_ltp_crc_mode |= lcb_to_port_ltp(crc_val); 7552 /* active crc mode */ 7553 7554 /* set up the remote credit return table */ 7555 assign_remote_cm_au_table(dd, vcu); 7556 7557 /* 7558 * The LCB is reset on entry to handle_verify_cap(), so this must 7559 * be applied on every link up. 7560 * 7561 * Adjust LCB error kill enable to kill the link if 7562 * these RBUF errors are seen: 7563 * REPLAY_BUF_MBE_SMASK 7564 * FLIT_INPUT_BUF_MBE_SMASK 7565 */ 7566 if (is_ax(dd)) { /* fixed in B0 */ 7567 reg = read_csr(dd, DC_LCB_CFG_LINK_KILL_EN); 7568 reg |= DC_LCB_CFG_LINK_KILL_EN_REPLAY_BUF_MBE_SMASK 7569 | DC_LCB_CFG_LINK_KILL_EN_FLIT_INPUT_BUF_MBE_SMASK; 7570 write_csr(dd, DC_LCB_CFG_LINK_KILL_EN, reg); 7571 } 7572 7573 /* pull LCB fifos out of reset - all fifo clocks must be stable */ 7574 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0); 7575 7576 /* give 8051 access to the LCB CSRs */ 7577 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */ 7578 set_8051_lcb_access(dd); 7579 7580 /* tell the 8051 to go to LinkUp */ 7581 set_link_state(ppd, HLS_GOING_UP); 7582 } 7583 7584 /** 7585 * apply_link_downgrade_policy - Apply the link width downgrade enabled 7586 * policy against the current active link widths. 7587 * @ppd: info of physical Hfi port 7588 * @refresh_widths: True indicates link downgrade event 7589 * @return: True indicates a successful link downgrade. False indicates 7590 * link downgrade event failed and the link will bounce back to 7591 * default link width. 7592 * 7593 * Called when the enabled policy changes or the active link widths 7594 * change. 7595 * Refresh_widths indicates that a link downgrade occurred. The 7596 * link_downgraded variable is set by refresh_widths and 7597 * determines the success/failure of the policy application. 7598 */ 7599 bool apply_link_downgrade_policy(struct hfi1_pportdata *ppd, 7600 bool refresh_widths) 7601 { 7602 int do_bounce = 0; 7603 int tries; 7604 u16 lwde; 7605 u16 tx, rx; 7606 bool link_downgraded = refresh_widths; 7607 7608 /* use the hls lock to avoid a race with actual link up */ 7609 tries = 0; 7610 retry: 7611 mutex_lock(&ppd->hls_lock); 7612 /* only apply if the link is up */ 7613 if (ppd->host_link_state & HLS_DOWN) { 7614 /* still going up..wait and retry */ 7615 if (ppd->host_link_state & HLS_GOING_UP) { 7616 if (++tries < 1000) { 7617 mutex_unlock(&ppd->hls_lock); 7618 usleep_range(100, 120); /* arbitrary */ 7619 goto retry; 7620 } 7621 dd_dev_err(ppd->dd, 7622 "%s: giving up waiting for link state change\n", 7623 __func__); 7624 } 7625 goto done; 7626 } 7627 7628 lwde = ppd->link_width_downgrade_enabled; 7629 7630 if (refresh_widths) { 7631 get_link_widths(ppd->dd, &tx, &rx); 7632 ppd->link_width_downgrade_tx_active = tx; 7633 ppd->link_width_downgrade_rx_active = rx; 7634 } 7635 7636 if (ppd->link_width_downgrade_tx_active == 0 || 7637 ppd->link_width_downgrade_rx_active == 0) { 7638 /* the 8051 reported a dead link as a downgrade */ 7639 dd_dev_err(ppd->dd, "Link downgrade is really a link down, ignoring\n"); 7640 link_downgraded = false; 7641 } else if (lwde == 0) { 7642 /* downgrade is disabled */ 7643 7644 /* bounce if not at starting active width */ 7645 if ((ppd->link_width_active != 7646 ppd->link_width_downgrade_tx_active) || 7647 (ppd->link_width_active != 7648 ppd->link_width_downgrade_rx_active)) { 7649 dd_dev_err(ppd->dd, 7650 "Link downgrade is disabled and link has downgraded, downing link\n"); 7651 dd_dev_err(ppd->dd, 7652 " original 0x%x, tx active 0x%x, rx active 0x%x\n", 7653 ppd->link_width_active, 7654 ppd->link_width_downgrade_tx_active, 7655 ppd->link_width_downgrade_rx_active); 7656 do_bounce = 1; 7657 link_downgraded = false; 7658 } 7659 } else if ((lwde & ppd->link_width_downgrade_tx_active) == 0 || 7660 (lwde & ppd->link_width_downgrade_rx_active) == 0) { 7661 /* Tx or Rx is outside the enabled policy */ 7662 dd_dev_err(ppd->dd, 7663 "Link is outside of downgrade allowed, downing link\n"); 7664 dd_dev_err(ppd->dd, 7665 " enabled 0x%x, tx active 0x%x, rx active 0x%x\n", 7666 lwde, ppd->link_width_downgrade_tx_active, 7667 ppd->link_width_downgrade_rx_active); 7668 do_bounce = 1; 7669 link_downgraded = false; 7670 } 7671 7672 done: 7673 mutex_unlock(&ppd->hls_lock); 7674 7675 if (do_bounce) { 7676 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_WIDTH_POLICY, 0, 7677 OPA_LINKDOWN_REASON_WIDTH_POLICY); 7678 set_link_state(ppd, HLS_DN_OFFLINE); 7679 start_link(ppd); 7680 } 7681 7682 return link_downgraded; 7683 } 7684 7685 /* 7686 * Handle a link downgrade interrupt from the 8051. 7687 * 7688 * This is a work-queue function outside of the interrupt. 7689 */ 7690 void handle_link_downgrade(struct work_struct *work) 7691 { 7692 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 7693 link_downgrade_work); 7694 7695 dd_dev_info(ppd->dd, "8051: Link width downgrade\n"); 7696 if (apply_link_downgrade_policy(ppd, true)) 7697 update_xmit_counters(ppd, ppd->link_width_downgrade_tx_active); 7698 } 7699 7700 static char *dcc_err_string(char *buf, int buf_len, u64 flags) 7701 { 7702 return flag_string(buf, buf_len, flags, dcc_err_flags, 7703 ARRAY_SIZE(dcc_err_flags)); 7704 } 7705 7706 static char *lcb_err_string(char *buf, int buf_len, u64 flags) 7707 { 7708 return flag_string(buf, buf_len, flags, lcb_err_flags, 7709 ARRAY_SIZE(lcb_err_flags)); 7710 } 7711 7712 static char *dc8051_err_string(char *buf, int buf_len, u64 flags) 7713 { 7714 return flag_string(buf, buf_len, flags, dc8051_err_flags, 7715 ARRAY_SIZE(dc8051_err_flags)); 7716 } 7717 7718 static char *dc8051_info_err_string(char *buf, int buf_len, u64 flags) 7719 { 7720 return flag_string(buf, buf_len, flags, dc8051_info_err_flags, 7721 ARRAY_SIZE(dc8051_info_err_flags)); 7722 } 7723 7724 static char *dc8051_info_host_msg_string(char *buf, int buf_len, u64 flags) 7725 { 7726 return flag_string(buf, buf_len, flags, dc8051_info_host_msg_flags, 7727 ARRAY_SIZE(dc8051_info_host_msg_flags)); 7728 } 7729 7730 static void handle_8051_interrupt(struct hfi1_devdata *dd, u32 unused, u64 reg) 7731 { 7732 struct hfi1_pportdata *ppd = dd->pport; 7733 u64 info, err, host_msg; 7734 int queue_link_down = 0; 7735 char buf[96]; 7736 7737 /* look at the flags */ 7738 if (reg & DC_DC8051_ERR_FLG_SET_BY_8051_SMASK) { 7739 /* 8051 information set by firmware */ 7740 /* read DC8051_DBG_ERR_INFO_SET_BY_8051 for details */ 7741 info = read_csr(dd, DC_DC8051_DBG_ERR_INFO_SET_BY_8051); 7742 err = (info >> DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_SHIFT) 7743 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_ERROR_MASK; 7744 host_msg = (info >> 7745 DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_SHIFT) 7746 & DC_DC8051_DBG_ERR_INFO_SET_BY_8051_HOST_MSG_MASK; 7747 7748 /* 7749 * Handle error flags. 7750 */ 7751 if (err & FAILED_LNI) { 7752 /* 7753 * LNI error indications are cleared by the 8051 7754 * only when starting polling. Only pay attention 7755 * to them when in the states that occur during 7756 * LNI. 7757 */ 7758 if (ppd->host_link_state 7759 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) { 7760 queue_link_down = 1; 7761 dd_dev_info(dd, "Link error: %s\n", 7762 dc8051_info_err_string(buf, 7763 sizeof(buf), 7764 err & 7765 FAILED_LNI)); 7766 } 7767 err &= ~(u64)FAILED_LNI; 7768 } 7769 /* unknown frames can happen durning LNI, just count */ 7770 if (err & UNKNOWN_FRAME) { 7771 ppd->unknown_frame_count++; 7772 err &= ~(u64)UNKNOWN_FRAME; 7773 } 7774 if (err) { 7775 /* report remaining errors, but do not do anything */ 7776 dd_dev_err(dd, "8051 info error: %s\n", 7777 dc8051_info_err_string(buf, sizeof(buf), 7778 err)); 7779 } 7780 7781 /* 7782 * Handle host message flags. 7783 */ 7784 if (host_msg & HOST_REQ_DONE) { 7785 /* 7786 * Presently, the driver does a busy wait for 7787 * host requests to complete. This is only an 7788 * informational message. 7789 * NOTE: The 8051 clears the host message 7790 * information *on the next 8051 command*. 7791 * Therefore, when linkup is achieved, 7792 * this flag will still be set. 7793 */ 7794 host_msg &= ~(u64)HOST_REQ_DONE; 7795 } 7796 if (host_msg & BC_SMA_MSG) { 7797 queue_work(ppd->link_wq, &ppd->sma_message_work); 7798 host_msg &= ~(u64)BC_SMA_MSG; 7799 } 7800 if (host_msg & LINKUP_ACHIEVED) { 7801 dd_dev_info(dd, "8051: Link up\n"); 7802 queue_work(ppd->link_wq, &ppd->link_up_work); 7803 host_msg &= ~(u64)LINKUP_ACHIEVED; 7804 } 7805 if (host_msg & EXT_DEVICE_CFG_REQ) { 7806 handle_8051_request(ppd); 7807 host_msg &= ~(u64)EXT_DEVICE_CFG_REQ; 7808 } 7809 if (host_msg & VERIFY_CAP_FRAME) { 7810 queue_work(ppd->link_wq, &ppd->link_vc_work); 7811 host_msg &= ~(u64)VERIFY_CAP_FRAME; 7812 } 7813 if (host_msg & LINK_GOING_DOWN) { 7814 const char *extra = ""; 7815 /* no downgrade action needed if going down */ 7816 if (host_msg & LINK_WIDTH_DOWNGRADED) { 7817 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED; 7818 extra = " (ignoring downgrade)"; 7819 } 7820 dd_dev_info(dd, "8051: Link down%s\n", extra); 7821 queue_link_down = 1; 7822 host_msg &= ~(u64)LINK_GOING_DOWN; 7823 } 7824 if (host_msg & LINK_WIDTH_DOWNGRADED) { 7825 queue_work(ppd->link_wq, &ppd->link_downgrade_work); 7826 host_msg &= ~(u64)LINK_WIDTH_DOWNGRADED; 7827 } 7828 if (host_msg) { 7829 /* report remaining messages, but do not do anything */ 7830 dd_dev_info(dd, "8051 info host message: %s\n", 7831 dc8051_info_host_msg_string(buf, 7832 sizeof(buf), 7833 host_msg)); 7834 } 7835 7836 reg &= ~DC_DC8051_ERR_FLG_SET_BY_8051_SMASK; 7837 } 7838 if (reg & DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK) { 7839 /* 7840 * Lost the 8051 heartbeat. If this happens, we 7841 * receive constant interrupts about it. Disable 7842 * the interrupt after the first. 7843 */ 7844 dd_dev_err(dd, "Lost 8051 heartbeat\n"); 7845 write_csr(dd, DC_DC8051_ERR_EN, 7846 read_csr(dd, DC_DC8051_ERR_EN) & 7847 ~DC_DC8051_ERR_EN_LOST_8051_HEART_BEAT_SMASK); 7848 7849 reg &= ~DC_DC8051_ERR_FLG_LOST_8051_HEART_BEAT_SMASK; 7850 } 7851 if (reg) { 7852 /* report the error, but do not do anything */ 7853 dd_dev_err(dd, "8051 error: %s\n", 7854 dc8051_err_string(buf, sizeof(buf), reg)); 7855 } 7856 7857 if (queue_link_down) { 7858 /* 7859 * if the link is already going down or disabled, do not 7860 * queue another. If there's a link down entry already 7861 * queued, don't queue another one. 7862 */ 7863 if ((ppd->host_link_state & 7864 (HLS_GOING_OFFLINE | HLS_LINK_COOLDOWN)) || 7865 ppd->link_enabled == 0) { 7866 dd_dev_info(dd, "%s: not queuing link down. host_link_state %x, link_enabled %x\n", 7867 __func__, ppd->host_link_state, 7868 ppd->link_enabled); 7869 } else { 7870 if (xchg(&ppd->is_link_down_queued, 1) == 1) 7871 dd_dev_info(dd, 7872 "%s: link down request already queued\n", 7873 __func__); 7874 else 7875 queue_work(ppd->link_wq, &ppd->link_down_work); 7876 } 7877 } 7878 } 7879 7880 static const char * const fm_config_txt[] = { 7881 [0] = 7882 "BadHeadDist: Distance violation between two head flits", 7883 [1] = 7884 "BadTailDist: Distance violation between two tail flits", 7885 [2] = 7886 "BadCtrlDist: Distance violation between two credit control flits", 7887 [3] = 7888 "BadCrdAck: Credits return for unsupported VL", 7889 [4] = 7890 "UnsupportedVLMarker: Received VL Marker", 7891 [5] = 7892 "BadPreempt: Exceeded the preemption nesting level", 7893 [6] = 7894 "BadControlFlit: Received unsupported control flit", 7895 /* no 7 */ 7896 [8] = 7897 "UnsupportedVLMarker: Received VL Marker for unconfigured or disabled VL", 7898 }; 7899 7900 static const char * const port_rcv_txt[] = { 7901 [1] = 7902 "BadPktLen: Illegal PktLen", 7903 [2] = 7904 "PktLenTooLong: Packet longer than PktLen", 7905 [3] = 7906 "PktLenTooShort: Packet shorter than PktLen", 7907 [4] = 7908 "BadSLID: Illegal SLID (0, using multicast as SLID, does not include security validation of SLID)", 7909 [5] = 7910 "BadDLID: Illegal DLID (0, doesn't match HFI)", 7911 [6] = 7912 "BadL2: Illegal L2 opcode", 7913 [7] = 7914 "BadSC: Unsupported SC", 7915 [9] = 7916 "BadRC: Illegal RC", 7917 [11] = 7918 "PreemptError: Preempting with same VL", 7919 [12] = 7920 "PreemptVL15: Preempting a VL15 packet", 7921 }; 7922 7923 #define OPA_LDR_FMCONFIG_OFFSET 16 7924 #define OPA_LDR_PORTRCV_OFFSET 0 7925 static void handle_dcc_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 7926 { 7927 u64 info, hdr0, hdr1; 7928 const char *extra; 7929 char buf[96]; 7930 struct hfi1_pportdata *ppd = dd->pport; 7931 u8 lcl_reason = 0; 7932 int do_bounce = 0; 7933 7934 if (reg & DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK) { 7935 if (!(dd->err_info_uncorrectable & OPA_EI_STATUS_SMASK)) { 7936 info = read_csr(dd, DCC_ERR_INFO_UNCORRECTABLE); 7937 dd->err_info_uncorrectable = info & OPA_EI_CODE_SMASK; 7938 /* set status bit */ 7939 dd->err_info_uncorrectable |= OPA_EI_STATUS_SMASK; 7940 } 7941 reg &= ~DCC_ERR_FLG_UNCORRECTABLE_ERR_SMASK; 7942 } 7943 7944 if (reg & DCC_ERR_FLG_LINK_ERR_SMASK) { 7945 struct hfi1_pportdata *ppd = dd->pport; 7946 /* this counter saturates at (2^32) - 1 */ 7947 if (ppd->link_downed < (u32)UINT_MAX) 7948 ppd->link_downed++; 7949 reg &= ~DCC_ERR_FLG_LINK_ERR_SMASK; 7950 } 7951 7952 if (reg & DCC_ERR_FLG_FMCONFIG_ERR_SMASK) { 7953 u8 reason_valid = 1; 7954 7955 info = read_csr(dd, DCC_ERR_INFO_FMCONFIG); 7956 if (!(dd->err_info_fmconfig & OPA_EI_STATUS_SMASK)) { 7957 dd->err_info_fmconfig = info & OPA_EI_CODE_SMASK; 7958 /* set status bit */ 7959 dd->err_info_fmconfig |= OPA_EI_STATUS_SMASK; 7960 } 7961 switch (info) { 7962 case 0: 7963 case 1: 7964 case 2: 7965 case 3: 7966 case 4: 7967 case 5: 7968 case 6: 7969 extra = fm_config_txt[info]; 7970 break; 7971 case 8: 7972 extra = fm_config_txt[info]; 7973 if (ppd->port_error_action & 7974 OPA_PI_MASK_FM_CFG_UNSUPPORTED_VL_MARKER) { 7975 do_bounce = 1; 7976 /* 7977 * lcl_reason cannot be derived from info 7978 * for this error 7979 */ 7980 lcl_reason = 7981 OPA_LINKDOWN_REASON_UNSUPPORTED_VL_MARKER; 7982 } 7983 break; 7984 default: 7985 reason_valid = 0; 7986 snprintf(buf, sizeof(buf), "reserved%lld", info); 7987 extra = buf; 7988 break; 7989 } 7990 7991 if (reason_valid && !do_bounce) { 7992 do_bounce = ppd->port_error_action & 7993 (1 << (OPA_LDR_FMCONFIG_OFFSET + info)); 7994 lcl_reason = info + OPA_LINKDOWN_REASON_BAD_HEAD_DIST; 7995 } 7996 7997 /* just report this */ 7998 dd_dev_info_ratelimited(dd, "DCC Error: fmconfig error: %s\n", 7999 extra); 8000 reg &= ~DCC_ERR_FLG_FMCONFIG_ERR_SMASK; 8001 } 8002 8003 if (reg & DCC_ERR_FLG_RCVPORT_ERR_SMASK) { 8004 u8 reason_valid = 1; 8005 8006 info = read_csr(dd, DCC_ERR_INFO_PORTRCV); 8007 hdr0 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR0); 8008 hdr1 = read_csr(dd, DCC_ERR_INFO_PORTRCV_HDR1); 8009 if (!(dd->err_info_rcvport.status_and_code & 8010 OPA_EI_STATUS_SMASK)) { 8011 dd->err_info_rcvport.status_and_code = 8012 info & OPA_EI_CODE_SMASK; 8013 /* set status bit */ 8014 dd->err_info_rcvport.status_and_code |= 8015 OPA_EI_STATUS_SMASK; 8016 /* 8017 * save first 2 flits in the packet that caused 8018 * the error 8019 */ 8020 dd->err_info_rcvport.packet_flit1 = hdr0; 8021 dd->err_info_rcvport.packet_flit2 = hdr1; 8022 } 8023 switch (info) { 8024 case 1: 8025 case 2: 8026 case 3: 8027 case 4: 8028 case 5: 8029 case 6: 8030 case 7: 8031 case 9: 8032 case 11: 8033 case 12: 8034 extra = port_rcv_txt[info]; 8035 break; 8036 default: 8037 reason_valid = 0; 8038 snprintf(buf, sizeof(buf), "reserved%lld", info); 8039 extra = buf; 8040 break; 8041 } 8042 8043 if (reason_valid && !do_bounce) { 8044 do_bounce = ppd->port_error_action & 8045 (1 << (OPA_LDR_PORTRCV_OFFSET + info)); 8046 lcl_reason = info + OPA_LINKDOWN_REASON_RCV_ERROR_0; 8047 } 8048 8049 /* just report this */ 8050 dd_dev_info_ratelimited(dd, "DCC Error: PortRcv error: %s\n" 8051 " hdr0 0x%llx, hdr1 0x%llx\n", 8052 extra, hdr0, hdr1); 8053 8054 reg &= ~DCC_ERR_FLG_RCVPORT_ERR_SMASK; 8055 } 8056 8057 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK) { 8058 /* informative only */ 8059 dd_dev_info_ratelimited(dd, "8051 access to LCB blocked\n"); 8060 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_UC_SMASK; 8061 } 8062 if (reg & DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK) { 8063 /* informative only */ 8064 dd_dev_info_ratelimited(dd, "host access to LCB blocked\n"); 8065 reg &= ~DCC_ERR_FLG_EN_CSR_ACCESS_BLOCKED_HOST_SMASK; 8066 } 8067 8068 if (unlikely(hfi1_dbg_fault_suppress_err(&dd->verbs_dev))) 8069 reg &= ~DCC_ERR_FLG_LATE_EBP_ERR_SMASK; 8070 8071 /* report any remaining errors */ 8072 if (reg) 8073 dd_dev_info_ratelimited(dd, "DCC Error: %s\n", 8074 dcc_err_string(buf, sizeof(buf), reg)); 8075 8076 if (lcl_reason == 0) 8077 lcl_reason = OPA_LINKDOWN_REASON_UNKNOWN; 8078 8079 if (do_bounce) { 8080 dd_dev_info_ratelimited(dd, "%s: PortErrorAction bounce\n", 8081 __func__); 8082 set_link_down_reason(ppd, lcl_reason, 0, lcl_reason); 8083 queue_work(ppd->link_wq, &ppd->link_bounce_work); 8084 } 8085 } 8086 8087 static void handle_lcb_err(struct hfi1_devdata *dd, u32 unused, u64 reg) 8088 { 8089 char buf[96]; 8090 8091 dd_dev_info(dd, "LCB Error: %s\n", 8092 lcb_err_string(buf, sizeof(buf), reg)); 8093 } 8094 8095 /* 8096 * CCE block DC interrupt. Source is < 8. 8097 */ 8098 static void is_dc_int(struct hfi1_devdata *dd, unsigned int source) 8099 { 8100 const struct err_reg_info *eri = &dc_errs[source]; 8101 8102 if (eri->handler) { 8103 interrupt_clear_down(dd, 0, eri); 8104 } else if (source == 3 /* dc_lbm_int */) { 8105 /* 8106 * This indicates that a parity error has occurred on the 8107 * address/control lines presented to the LBM. The error 8108 * is a single pulse, there is no associated error flag, 8109 * and it is non-maskable. This is because if a parity 8110 * error occurs on the request the request is dropped. 8111 * This should never occur, but it is nice to know if it 8112 * ever does. 8113 */ 8114 dd_dev_err(dd, "Parity error in DC LBM block\n"); 8115 } else { 8116 dd_dev_err(dd, "Invalid DC interrupt %u\n", source); 8117 } 8118 } 8119 8120 /* 8121 * TX block send credit interrupt. Source is < 160. 8122 */ 8123 static void is_send_credit_int(struct hfi1_devdata *dd, unsigned int source) 8124 { 8125 sc_group_release_update(dd, source); 8126 } 8127 8128 /* 8129 * TX block SDMA interrupt. Source is < 48. 8130 * 8131 * SDMA interrupts are grouped by type: 8132 * 8133 * 0 - N-1 = SDma 8134 * N - 2N-1 = SDmaProgress 8135 * 2N - 3N-1 = SDmaIdle 8136 */ 8137 static void is_sdma_eng_int(struct hfi1_devdata *dd, unsigned int source) 8138 { 8139 /* what interrupt */ 8140 unsigned int what = source / TXE_NUM_SDMA_ENGINES; 8141 /* which engine */ 8142 unsigned int which = source % TXE_NUM_SDMA_ENGINES; 8143 8144 #ifdef CONFIG_SDMA_VERBOSITY 8145 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", which, 8146 slashstrip(__FILE__), __LINE__, __func__); 8147 sdma_dumpstate(&dd->per_sdma[which]); 8148 #endif 8149 8150 if (likely(what < 3 && which < dd->num_sdma)) { 8151 sdma_engine_interrupt(&dd->per_sdma[which], 1ull << source); 8152 } else { 8153 /* should not happen */ 8154 dd_dev_err(dd, "Invalid SDMA interrupt 0x%x\n", source); 8155 } 8156 } 8157 8158 /** 8159 * is_rcv_avail_int() - User receive context available IRQ handler 8160 * @dd: valid dd 8161 * @source: logical IRQ source (offset from IS_RCVAVAIL_START) 8162 * 8163 * RX block receive available interrupt. Source is < 160. 8164 * 8165 * This is the general interrupt handler for user (PSM) receive contexts, 8166 * and can only be used for non-threaded IRQs. 8167 */ 8168 static void is_rcv_avail_int(struct hfi1_devdata *dd, unsigned int source) 8169 { 8170 struct hfi1_ctxtdata *rcd; 8171 char *err_detail; 8172 8173 if (likely(source < dd->num_rcv_contexts)) { 8174 rcd = hfi1_rcd_get_by_index(dd, source); 8175 if (rcd) { 8176 handle_user_interrupt(rcd); 8177 hfi1_rcd_put(rcd); 8178 return; /* OK */ 8179 } 8180 /* received an interrupt, but no rcd */ 8181 err_detail = "dataless"; 8182 } else { 8183 /* received an interrupt, but are not using that context */ 8184 err_detail = "out of range"; 8185 } 8186 dd_dev_err(dd, "unexpected %s receive available context interrupt %u\n", 8187 err_detail, source); 8188 } 8189 8190 /** 8191 * is_rcv_urgent_int() - User receive context urgent IRQ handler 8192 * @dd: valid dd 8193 * @source: logical IRQ source (offset from IS_RCVURGENT_START) 8194 * 8195 * RX block receive urgent interrupt. Source is < 160. 8196 * 8197 * NOTE: kernel receive contexts specifically do NOT enable this IRQ. 8198 */ 8199 static void is_rcv_urgent_int(struct hfi1_devdata *dd, unsigned int source) 8200 { 8201 struct hfi1_ctxtdata *rcd; 8202 char *err_detail; 8203 8204 if (likely(source < dd->num_rcv_contexts)) { 8205 rcd = hfi1_rcd_get_by_index(dd, source); 8206 if (rcd) { 8207 handle_user_interrupt(rcd); 8208 hfi1_rcd_put(rcd); 8209 return; /* OK */ 8210 } 8211 /* received an interrupt, but no rcd */ 8212 err_detail = "dataless"; 8213 } else { 8214 /* received an interrupt, but are not using that context */ 8215 err_detail = "out of range"; 8216 } 8217 dd_dev_err(dd, "unexpected %s receive urgent context interrupt %u\n", 8218 err_detail, source); 8219 } 8220 8221 /* 8222 * Reserved range interrupt. Should not be called in normal operation. 8223 */ 8224 static void is_reserved_int(struct hfi1_devdata *dd, unsigned int source) 8225 { 8226 char name[64]; 8227 8228 dd_dev_err(dd, "unexpected %s interrupt\n", 8229 is_reserved_name(name, sizeof(name), source)); 8230 } 8231 8232 static const struct is_table is_table[] = { 8233 /* 8234 * start end 8235 * name func interrupt func 8236 */ 8237 { IS_GENERAL_ERR_START, IS_GENERAL_ERR_END, 8238 is_misc_err_name, is_misc_err_int }, 8239 { IS_SDMAENG_ERR_START, IS_SDMAENG_ERR_END, 8240 is_sdma_eng_err_name, is_sdma_eng_err_int }, 8241 { IS_SENDCTXT_ERR_START, IS_SENDCTXT_ERR_END, 8242 is_sendctxt_err_name, is_sendctxt_err_int }, 8243 { IS_SDMA_START, IS_SDMA_IDLE_END, 8244 is_sdma_eng_name, is_sdma_eng_int }, 8245 { IS_VARIOUS_START, IS_VARIOUS_END, 8246 is_various_name, is_various_int }, 8247 { IS_DC_START, IS_DC_END, 8248 is_dc_name, is_dc_int }, 8249 { IS_RCVAVAIL_START, IS_RCVAVAIL_END, 8250 is_rcv_avail_name, is_rcv_avail_int }, 8251 { IS_RCVURGENT_START, IS_RCVURGENT_END, 8252 is_rcv_urgent_name, is_rcv_urgent_int }, 8253 { IS_SENDCREDIT_START, IS_SENDCREDIT_END, 8254 is_send_credit_name, is_send_credit_int}, 8255 { IS_RESERVED_START, IS_RESERVED_END, 8256 is_reserved_name, is_reserved_int}, 8257 }; 8258 8259 /* 8260 * Interrupt source interrupt - called when the given source has an interrupt. 8261 * Source is a bit index into an array of 64-bit integers. 8262 */ 8263 static void is_interrupt(struct hfi1_devdata *dd, unsigned int source) 8264 { 8265 const struct is_table *entry; 8266 8267 /* avoids a double compare by walking the table in-order */ 8268 for (entry = &is_table[0]; entry->is_name; entry++) { 8269 if (source <= entry->end) { 8270 trace_hfi1_interrupt(dd, entry, source); 8271 entry->is_int(dd, source - entry->start); 8272 return; 8273 } 8274 } 8275 /* fell off the end */ 8276 dd_dev_err(dd, "invalid interrupt source %u\n", source); 8277 } 8278 8279 /** 8280 * general_interrupt - General interrupt handler 8281 * @irq: MSIx IRQ vector 8282 * @data: hfi1 devdata 8283 * 8284 * This is able to correctly handle all non-threaded interrupts. Receive 8285 * context DATA IRQs are threaded and are not supported by this handler. 8286 * 8287 */ 8288 irqreturn_t general_interrupt(int irq, void *data) 8289 { 8290 struct hfi1_devdata *dd = data; 8291 u64 regs[CCE_NUM_INT_CSRS]; 8292 u32 bit; 8293 int i; 8294 irqreturn_t handled = IRQ_NONE; 8295 8296 this_cpu_inc(*dd->int_counter); 8297 8298 /* phase 1: scan and clear all handled interrupts */ 8299 for (i = 0; i < CCE_NUM_INT_CSRS; i++) { 8300 if (dd->gi_mask[i] == 0) { 8301 regs[i] = 0; /* used later */ 8302 continue; 8303 } 8304 regs[i] = read_csr(dd, CCE_INT_STATUS + (8 * i)) & 8305 dd->gi_mask[i]; 8306 /* only clear if anything is set */ 8307 if (regs[i]) 8308 write_csr(dd, CCE_INT_CLEAR + (8 * i), regs[i]); 8309 } 8310 8311 /* phase 2: call the appropriate handler */ 8312 for_each_set_bit(bit, (unsigned long *)®s[0], 8313 CCE_NUM_INT_CSRS * 64) { 8314 is_interrupt(dd, bit); 8315 handled = IRQ_HANDLED; 8316 } 8317 8318 return handled; 8319 } 8320 8321 irqreturn_t sdma_interrupt(int irq, void *data) 8322 { 8323 struct sdma_engine *sde = data; 8324 struct hfi1_devdata *dd = sde->dd; 8325 u64 status; 8326 8327 #ifdef CONFIG_SDMA_VERBOSITY 8328 dd_dev_err(dd, "CONFIG SDMA(%u) %s:%d %s()\n", sde->this_idx, 8329 slashstrip(__FILE__), __LINE__, __func__); 8330 sdma_dumpstate(sde); 8331 #endif 8332 8333 this_cpu_inc(*dd->int_counter); 8334 8335 /* This read_csr is really bad in the hot path */ 8336 status = read_csr(dd, 8337 CCE_INT_STATUS + (8 * (IS_SDMA_START / 64))) 8338 & sde->imask; 8339 if (likely(status)) { 8340 /* clear the interrupt(s) */ 8341 write_csr(dd, 8342 CCE_INT_CLEAR + (8 * (IS_SDMA_START / 64)), 8343 status); 8344 8345 /* handle the interrupt(s) */ 8346 sdma_engine_interrupt(sde, status); 8347 } else { 8348 dd_dev_info_ratelimited(dd, "SDMA engine %u interrupt, but no status bits set\n", 8349 sde->this_idx); 8350 } 8351 return IRQ_HANDLED; 8352 } 8353 8354 /* 8355 * Clear the receive interrupt. Use a read of the interrupt clear CSR 8356 * to insure that the write completed. This does NOT guarantee that 8357 * queued DMA writes to memory from the chip are pushed. 8358 */ 8359 static inline void clear_recv_intr(struct hfi1_ctxtdata *rcd) 8360 { 8361 struct hfi1_devdata *dd = rcd->dd; 8362 u32 addr = CCE_INT_CLEAR + (8 * rcd->ireg); 8363 8364 write_csr(dd, addr, rcd->imask); 8365 /* force the above write on the chip and get a value back */ 8366 (void)read_csr(dd, addr); 8367 } 8368 8369 /* force the receive interrupt */ 8370 void force_recv_intr(struct hfi1_ctxtdata *rcd) 8371 { 8372 write_csr(rcd->dd, CCE_INT_FORCE + (8 * rcd->ireg), rcd->imask); 8373 } 8374 8375 /* 8376 * Return non-zero if a packet is present. 8377 * 8378 * This routine is called when rechecking for packets after the RcvAvail 8379 * interrupt has been cleared down. First, do a quick check of memory for 8380 * a packet present. If not found, use an expensive CSR read of the context 8381 * tail to determine the actual tail. The CSR read is necessary because there 8382 * is no method to push pending DMAs to memory other than an interrupt and we 8383 * are trying to determine if we need to force an interrupt. 8384 */ 8385 static inline int check_packet_present(struct hfi1_ctxtdata *rcd) 8386 { 8387 u32 tail; 8388 8389 if (hfi1_packet_present(rcd)) 8390 return 1; 8391 8392 /* fall back to a CSR read, correct indpendent of DMA_RTAIL */ 8393 tail = (u32)read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL); 8394 return hfi1_rcd_head(rcd) != tail; 8395 } 8396 8397 /* 8398 * Common code for receive contexts interrupt handlers. 8399 * Update traces, increment kernel IRQ counter and 8400 * setup ASPM when needed. 8401 */ 8402 static void receive_interrupt_common(struct hfi1_ctxtdata *rcd) 8403 { 8404 struct hfi1_devdata *dd = rcd->dd; 8405 8406 trace_hfi1_receive_interrupt(dd, rcd); 8407 this_cpu_inc(*dd->int_counter); 8408 aspm_ctx_disable(rcd); 8409 } 8410 8411 /* 8412 * __hfi1_rcd_eoi_intr() - Make HW issue receive interrupt 8413 * when there are packets present in the queue. When calling 8414 * with interrupts enabled please use hfi1_rcd_eoi_intr. 8415 * 8416 * @rcd: valid receive context 8417 */ 8418 static void __hfi1_rcd_eoi_intr(struct hfi1_ctxtdata *rcd) 8419 { 8420 if (!rcd->rcvhdrq) 8421 return; 8422 clear_recv_intr(rcd); 8423 if (check_packet_present(rcd)) 8424 force_recv_intr(rcd); 8425 } 8426 8427 /** 8428 * hfi1_rcd_eoi_intr() - End of Interrupt processing action 8429 * 8430 * @rcd: Ptr to hfi1_ctxtdata of receive context 8431 * 8432 * Hold IRQs so we can safely clear the interrupt and 8433 * recheck for a packet that may have arrived after the previous 8434 * check and the interrupt clear. If a packet arrived, force another 8435 * interrupt. This routine can be called at the end of receive packet 8436 * processing in interrupt service routines, interrupt service thread 8437 * and softirqs 8438 */ 8439 static void hfi1_rcd_eoi_intr(struct hfi1_ctxtdata *rcd) 8440 { 8441 unsigned long flags; 8442 8443 local_irq_save(flags); 8444 __hfi1_rcd_eoi_intr(rcd); 8445 local_irq_restore(flags); 8446 } 8447 8448 /** 8449 * hfi1_netdev_rx_napi - napi poll function to move eoi inline 8450 * @napi: pointer to napi object 8451 * @budget: netdev budget 8452 */ 8453 int hfi1_netdev_rx_napi(struct napi_struct *napi, int budget) 8454 { 8455 struct hfi1_netdev_rxq *rxq = container_of(napi, 8456 struct hfi1_netdev_rxq, napi); 8457 struct hfi1_ctxtdata *rcd = rxq->rcd; 8458 int work_done = 0; 8459 8460 work_done = rcd->do_interrupt(rcd, budget); 8461 8462 if (work_done < budget) { 8463 napi_complete_done(napi, work_done); 8464 hfi1_rcd_eoi_intr(rcd); 8465 } 8466 8467 return work_done; 8468 } 8469 8470 /* Receive packet napi handler for netdevs VNIC and AIP */ 8471 irqreturn_t receive_context_interrupt_napi(int irq, void *data) 8472 { 8473 struct hfi1_ctxtdata *rcd = data; 8474 8475 receive_interrupt_common(rcd); 8476 8477 if (likely(rcd->napi)) { 8478 if (likely(napi_schedule_prep(rcd->napi))) 8479 __napi_schedule_irqoff(rcd->napi); 8480 else 8481 __hfi1_rcd_eoi_intr(rcd); 8482 } else { 8483 WARN_ONCE(1, "Napi IRQ handler without napi set up ctxt=%d\n", 8484 rcd->ctxt); 8485 __hfi1_rcd_eoi_intr(rcd); 8486 } 8487 8488 return IRQ_HANDLED; 8489 } 8490 8491 /* 8492 * Receive packet IRQ handler. This routine expects to be on its own IRQ. 8493 * This routine will try to handle packets immediately (latency), but if 8494 * it finds too many, it will invoke the thread handler (bandwitdh). The 8495 * chip receive interrupt is *not* cleared down until this or the thread (if 8496 * invoked) is finished. The intent is to avoid extra interrupts while we 8497 * are processing packets anyway. 8498 */ 8499 irqreturn_t receive_context_interrupt(int irq, void *data) 8500 { 8501 struct hfi1_ctxtdata *rcd = data; 8502 int disposition; 8503 8504 receive_interrupt_common(rcd); 8505 8506 /* receive interrupt remains blocked while processing packets */ 8507 disposition = rcd->do_interrupt(rcd, 0); 8508 8509 /* 8510 * Too many packets were seen while processing packets in this 8511 * IRQ handler. Invoke the handler thread. The receive interrupt 8512 * remains blocked. 8513 */ 8514 if (disposition == RCV_PKT_LIMIT) 8515 return IRQ_WAKE_THREAD; 8516 8517 __hfi1_rcd_eoi_intr(rcd); 8518 return IRQ_HANDLED; 8519 } 8520 8521 /* 8522 * Receive packet thread handler. This expects to be invoked with the 8523 * receive interrupt still blocked. 8524 */ 8525 irqreturn_t receive_context_thread(int irq, void *data) 8526 { 8527 struct hfi1_ctxtdata *rcd = data; 8528 8529 /* receive interrupt is still blocked from the IRQ handler */ 8530 (void)rcd->do_interrupt(rcd, 1); 8531 8532 hfi1_rcd_eoi_intr(rcd); 8533 8534 return IRQ_HANDLED; 8535 } 8536 8537 /* ========================================================================= */ 8538 8539 u32 read_physical_state(struct hfi1_devdata *dd) 8540 { 8541 u64 reg; 8542 8543 reg = read_csr(dd, DC_DC8051_STS_CUR_STATE); 8544 return (reg >> DC_DC8051_STS_CUR_STATE_PORT_SHIFT) 8545 & DC_DC8051_STS_CUR_STATE_PORT_MASK; 8546 } 8547 8548 u32 read_logical_state(struct hfi1_devdata *dd) 8549 { 8550 u64 reg; 8551 8552 reg = read_csr(dd, DCC_CFG_PORT_CONFIG); 8553 return (reg >> DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT) 8554 & DCC_CFG_PORT_CONFIG_LINK_STATE_MASK; 8555 } 8556 8557 static void set_logical_state(struct hfi1_devdata *dd, u32 chip_lstate) 8558 { 8559 u64 reg; 8560 8561 reg = read_csr(dd, DCC_CFG_PORT_CONFIG); 8562 /* clear current state, set new state */ 8563 reg &= ~DCC_CFG_PORT_CONFIG_LINK_STATE_SMASK; 8564 reg |= (u64)chip_lstate << DCC_CFG_PORT_CONFIG_LINK_STATE_SHIFT; 8565 write_csr(dd, DCC_CFG_PORT_CONFIG, reg); 8566 } 8567 8568 /* 8569 * Use the 8051 to read a LCB CSR. 8570 */ 8571 static int read_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 *data) 8572 { 8573 u32 regno; 8574 int ret; 8575 8576 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) { 8577 if (acquire_lcb_access(dd, 0) == 0) { 8578 *data = read_csr(dd, addr); 8579 release_lcb_access(dd, 0); 8580 return 0; 8581 } 8582 return -EBUSY; 8583 } 8584 8585 /* register is an index of LCB registers: (offset - base) / 8 */ 8586 regno = (addr - DC_LCB_CFG_RUN) >> 3; 8587 ret = do_8051_command(dd, HCMD_READ_LCB_CSR, regno, data); 8588 if (ret != HCMD_SUCCESS) 8589 return -EBUSY; 8590 return 0; 8591 } 8592 8593 /* 8594 * Provide a cache for some of the LCB registers in case the LCB is 8595 * unavailable. 8596 * (The LCB is unavailable in certain link states, for example.) 8597 */ 8598 struct lcb_datum { 8599 u32 off; 8600 u64 val; 8601 }; 8602 8603 static struct lcb_datum lcb_cache[] = { 8604 { DC_LCB_ERR_INFO_RX_REPLAY_CNT, 0}, 8605 { DC_LCB_ERR_INFO_SEQ_CRC_CNT, 0 }, 8606 { DC_LCB_ERR_INFO_REINIT_FROM_PEER_CNT, 0 }, 8607 }; 8608 8609 static void update_lcb_cache(struct hfi1_devdata *dd) 8610 { 8611 int i; 8612 int ret; 8613 u64 val; 8614 8615 for (i = 0; i < ARRAY_SIZE(lcb_cache); i++) { 8616 ret = read_lcb_csr(dd, lcb_cache[i].off, &val); 8617 8618 /* Update if we get good data */ 8619 if (likely(ret != -EBUSY)) 8620 lcb_cache[i].val = val; 8621 } 8622 } 8623 8624 static int read_lcb_cache(u32 off, u64 *val) 8625 { 8626 int i; 8627 8628 for (i = 0; i < ARRAY_SIZE(lcb_cache); i++) { 8629 if (lcb_cache[i].off == off) { 8630 *val = lcb_cache[i].val; 8631 return 0; 8632 } 8633 } 8634 8635 pr_warn("%s bad offset 0x%x\n", __func__, off); 8636 return -1; 8637 } 8638 8639 /* 8640 * Read an LCB CSR. Access may not be in host control, so check. 8641 * Return 0 on success, -EBUSY on failure. 8642 */ 8643 int read_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 *data) 8644 { 8645 struct hfi1_pportdata *ppd = dd->pport; 8646 8647 /* if up, go through the 8051 for the value */ 8648 if (ppd->host_link_state & HLS_UP) 8649 return read_lcb_via_8051(dd, addr, data); 8650 /* if going up or down, check the cache, otherwise, no access */ 8651 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE)) { 8652 if (read_lcb_cache(addr, data)) 8653 return -EBUSY; 8654 return 0; 8655 } 8656 8657 /* otherwise, host has access */ 8658 *data = read_csr(dd, addr); 8659 return 0; 8660 } 8661 8662 /* 8663 * Use the 8051 to write a LCB CSR. 8664 */ 8665 static int write_lcb_via_8051(struct hfi1_devdata *dd, u32 addr, u64 data) 8666 { 8667 u32 regno; 8668 int ret; 8669 8670 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR || 8671 (dd->dc8051_ver < dc8051_ver(0, 20, 0))) { 8672 if (acquire_lcb_access(dd, 0) == 0) { 8673 write_csr(dd, addr, data); 8674 release_lcb_access(dd, 0); 8675 return 0; 8676 } 8677 return -EBUSY; 8678 } 8679 8680 /* register is an index of LCB registers: (offset - base) / 8 */ 8681 regno = (addr - DC_LCB_CFG_RUN) >> 3; 8682 ret = do_8051_command(dd, HCMD_WRITE_LCB_CSR, regno, &data); 8683 if (ret != HCMD_SUCCESS) 8684 return -EBUSY; 8685 return 0; 8686 } 8687 8688 /* 8689 * Write an LCB CSR. Access may not be in host control, so check. 8690 * Return 0 on success, -EBUSY on failure. 8691 */ 8692 int write_lcb_csr(struct hfi1_devdata *dd, u32 addr, u64 data) 8693 { 8694 struct hfi1_pportdata *ppd = dd->pport; 8695 8696 /* if up, go through the 8051 for the value */ 8697 if (ppd->host_link_state & HLS_UP) 8698 return write_lcb_via_8051(dd, addr, data); 8699 /* if going up or down, no access */ 8700 if (ppd->host_link_state & (HLS_GOING_UP | HLS_GOING_OFFLINE)) 8701 return -EBUSY; 8702 /* otherwise, host has access */ 8703 write_csr(dd, addr, data); 8704 return 0; 8705 } 8706 8707 /* 8708 * Returns: 8709 * < 0 = Linux error, not able to get access 8710 * > 0 = 8051 command RETURN_CODE 8711 */ 8712 static int do_8051_command(struct hfi1_devdata *dd, u32 type, u64 in_data, 8713 u64 *out_data) 8714 { 8715 u64 reg, completed; 8716 int return_code; 8717 unsigned long timeout; 8718 8719 hfi1_cdbg(DC8051, "type %d, data 0x%012llx", type, in_data); 8720 8721 mutex_lock(&dd->dc8051_lock); 8722 8723 /* We can't send any commands to the 8051 if it's in reset */ 8724 if (dd->dc_shutdown) { 8725 return_code = -ENODEV; 8726 goto fail; 8727 } 8728 8729 /* 8730 * If an 8051 host command timed out previously, then the 8051 is 8731 * stuck. 8732 * 8733 * On first timeout, attempt to reset and restart the entire DC 8734 * block (including 8051). (Is this too big of a hammer?) 8735 * 8736 * If the 8051 times out a second time, the reset did not bring it 8737 * back to healthy life. In that case, fail any subsequent commands. 8738 */ 8739 if (dd->dc8051_timed_out) { 8740 if (dd->dc8051_timed_out > 1) { 8741 dd_dev_err(dd, 8742 "Previous 8051 host command timed out, skipping command %u\n", 8743 type); 8744 return_code = -ENXIO; 8745 goto fail; 8746 } 8747 _dc_shutdown(dd); 8748 _dc_start(dd); 8749 } 8750 8751 /* 8752 * If there is no timeout, then the 8051 command interface is 8753 * waiting for a command. 8754 */ 8755 8756 /* 8757 * When writing a LCB CSR, out_data contains the full value to 8758 * be written, while in_data contains the relative LCB 8759 * address in 7:0. Do the work here, rather than the caller, 8760 * of distrubting the write data to where it needs to go: 8761 * 8762 * Write data 8763 * 39:00 -> in_data[47:8] 8764 * 47:40 -> DC8051_CFG_EXT_DEV_0.RETURN_CODE 8765 * 63:48 -> DC8051_CFG_EXT_DEV_0.RSP_DATA 8766 */ 8767 if (type == HCMD_WRITE_LCB_CSR) { 8768 in_data |= ((*out_data) & 0xffffffffffull) << 8; 8769 /* must preserve COMPLETED - it is tied to hardware */ 8770 reg = read_csr(dd, DC_DC8051_CFG_EXT_DEV_0); 8771 reg &= DC_DC8051_CFG_EXT_DEV_0_COMPLETED_SMASK; 8772 reg |= ((((*out_data) >> 40) & 0xff) << 8773 DC_DC8051_CFG_EXT_DEV_0_RETURN_CODE_SHIFT) 8774 | ((((*out_data) >> 48) & 0xffff) << 8775 DC_DC8051_CFG_EXT_DEV_0_RSP_DATA_SHIFT); 8776 write_csr(dd, DC_DC8051_CFG_EXT_DEV_0, reg); 8777 } 8778 8779 /* 8780 * Do two writes: the first to stabilize the type and req_data, the 8781 * second to activate. 8782 */ 8783 reg = ((u64)type & DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_MASK) 8784 << DC_DC8051_CFG_HOST_CMD_0_REQ_TYPE_SHIFT 8785 | (in_data & DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_MASK) 8786 << DC_DC8051_CFG_HOST_CMD_0_REQ_DATA_SHIFT; 8787 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg); 8788 reg |= DC_DC8051_CFG_HOST_CMD_0_REQ_NEW_SMASK; 8789 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, reg); 8790 8791 /* wait for completion, alternate: interrupt */ 8792 timeout = jiffies + msecs_to_jiffies(DC8051_COMMAND_TIMEOUT); 8793 while (1) { 8794 reg = read_csr(dd, DC_DC8051_CFG_HOST_CMD_1); 8795 completed = reg & DC_DC8051_CFG_HOST_CMD_1_COMPLETED_SMASK; 8796 if (completed) 8797 break; 8798 if (time_after(jiffies, timeout)) { 8799 dd->dc8051_timed_out++; 8800 dd_dev_err(dd, "8051 host command %u timeout\n", type); 8801 if (out_data) 8802 *out_data = 0; 8803 return_code = -ETIMEDOUT; 8804 goto fail; 8805 } 8806 udelay(2); 8807 } 8808 8809 if (out_data) { 8810 *out_data = (reg >> DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_SHIFT) 8811 & DC_DC8051_CFG_HOST_CMD_1_RSP_DATA_MASK; 8812 if (type == HCMD_READ_LCB_CSR) { 8813 /* top 16 bits are in a different register */ 8814 *out_data |= (read_csr(dd, DC_DC8051_CFG_EXT_DEV_1) 8815 & DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SMASK) 8816 << (48 8817 - DC_DC8051_CFG_EXT_DEV_1_REQ_DATA_SHIFT); 8818 } 8819 } 8820 return_code = (reg >> DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_SHIFT) 8821 & DC_DC8051_CFG_HOST_CMD_1_RETURN_CODE_MASK; 8822 dd->dc8051_timed_out = 0; 8823 /* 8824 * Clear command for next user. 8825 */ 8826 write_csr(dd, DC_DC8051_CFG_HOST_CMD_0, 0); 8827 8828 fail: 8829 mutex_unlock(&dd->dc8051_lock); 8830 return return_code; 8831 } 8832 8833 static int set_physical_link_state(struct hfi1_devdata *dd, u64 state) 8834 { 8835 return do_8051_command(dd, HCMD_CHANGE_PHY_STATE, state, NULL); 8836 } 8837 8838 int load_8051_config(struct hfi1_devdata *dd, u8 field_id, 8839 u8 lane_id, u32 config_data) 8840 { 8841 u64 data; 8842 int ret; 8843 8844 data = (u64)field_id << LOAD_DATA_FIELD_ID_SHIFT 8845 | (u64)lane_id << LOAD_DATA_LANE_ID_SHIFT 8846 | (u64)config_data << LOAD_DATA_DATA_SHIFT; 8847 ret = do_8051_command(dd, HCMD_LOAD_CONFIG_DATA, data, NULL); 8848 if (ret != HCMD_SUCCESS) { 8849 dd_dev_err(dd, 8850 "load 8051 config: field id %d, lane %d, err %d\n", 8851 (int)field_id, (int)lane_id, ret); 8852 } 8853 return ret; 8854 } 8855 8856 /* 8857 * Read the 8051 firmware "registers". Use the RAM directly. Always 8858 * set the result, even on error. 8859 * Return 0 on success, -errno on failure 8860 */ 8861 int read_8051_config(struct hfi1_devdata *dd, u8 field_id, u8 lane_id, 8862 u32 *result) 8863 { 8864 u64 big_data; 8865 u32 addr; 8866 int ret; 8867 8868 /* address start depends on the lane_id */ 8869 if (lane_id < 4) 8870 addr = (4 * NUM_GENERAL_FIELDS) 8871 + (lane_id * 4 * NUM_LANE_FIELDS); 8872 else 8873 addr = 0; 8874 addr += field_id * 4; 8875 8876 /* read is in 8-byte chunks, hardware will truncate the address down */ 8877 ret = read_8051_data(dd, addr, 8, &big_data); 8878 8879 if (ret == 0) { 8880 /* extract the 4 bytes we want */ 8881 if (addr & 0x4) 8882 *result = (u32)(big_data >> 32); 8883 else 8884 *result = (u32)big_data; 8885 } else { 8886 *result = 0; 8887 dd_dev_err(dd, "%s: direct read failed, lane %d, field %d!\n", 8888 __func__, lane_id, field_id); 8889 } 8890 8891 return ret; 8892 } 8893 8894 static int write_vc_local_phy(struct hfi1_devdata *dd, u8 power_management, 8895 u8 continuous) 8896 { 8897 u32 frame; 8898 8899 frame = continuous << CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT 8900 | power_management << POWER_MANAGEMENT_SHIFT; 8901 return load_8051_config(dd, VERIFY_CAP_LOCAL_PHY, 8902 GENERAL_CONFIG, frame); 8903 } 8904 8905 static int write_vc_local_fabric(struct hfi1_devdata *dd, u8 vau, u8 z, u8 vcu, 8906 u16 vl15buf, u8 crc_sizes) 8907 { 8908 u32 frame; 8909 8910 frame = (u32)vau << VAU_SHIFT 8911 | (u32)z << Z_SHIFT 8912 | (u32)vcu << VCU_SHIFT 8913 | (u32)vl15buf << VL15BUF_SHIFT 8914 | (u32)crc_sizes << CRC_SIZES_SHIFT; 8915 return load_8051_config(dd, VERIFY_CAP_LOCAL_FABRIC, 8916 GENERAL_CONFIG, frame); 8917 } 8918 8919 static void read_vc_local_link_mode(struct hfi1_devdata *dd, u8 *misc_bits, 8920 u8 *flag_bits, u16 *link_widths) 8921 { 8922 u32 frame; 8923 8924 read_8051_config(dd, VERIFY_CAP_LOCAL_LINK_MODE, GENERAL_CONFIG, 8925 &frame); 8926 *misc_bits = (frame >> MISC_CONFIG_BITS_SHIFT) & MISC_CONFIG_BITS_MASK; 8927 *flag_bits = (frame >> LOCAL_FLAG_BITS_SHIFT) & LOCAL_FLAG_BITS_MASK; 8928 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK; 8929 } 8930 8931 static int write_vc_local_link_mode(struct hfi1_devdata *dd, 8932 u8 misc_bits, 8933 u8 flag_bits, 8934 u16 link_widths) 8935 { 8936 u32 frame; 8937 8938 frame = (u32)misc_bits << MISC_CONFIG_BITS_SHIFT 8939 | (u32)flag_bits << LOCAL_FLAG_BITS_SHIFT 8940 | (u32)link_widths << LINK_WIDTH_SHIFT; 8941 return load_8051_config(dd, VERIFY_CAP_LOCAL_LINK_MODE, GENERAL_CONFIG, 8942 frame); 8943 } 8944 8945 static int write_local_device_id(struct hfi1_devdata *dd, u16 device_id, 8946 u8 device_rev) 8947 { 8948 u32 frame; 8949 8950 frame = ((u32)device_id << LOCAL_DEVICE_ID_SHIFT) 8951 | ((u32)device_rev << LOCAL_DEVICE_REV_SHIFT); 8952 return load_8051_config(dd, LOCAL_DEVICE_ID, GENERAL_CONFIG, frame); 8953 } 8954 8955 static void read_remote_device_id(struct hfi1_devdata *dd, u16 *device_id, 8956 u8 *device_rev) 8957 { 8958 u32 frame; 8959 8960 read_8051_config(dd, REMOTE_DEVICE_ID, GENERAL_CONFIG, &frame); 8961 *device_id = (frame >> REMOTE_DEVICE_ID_SHIFT) & REMOTE_DEVICE_ID_MASK; 8962 *device_rev = (frame >> REMOTE_DEVICE_REV_SHIFT) 8963 & REMOTE_DEVICE_REV_MASK; 8964 } 8965 8966 int write_host_interface_version(struct hfi1_devdata *dd, u8 version) 8967 { 8968 u32 frame; 8969 u32 mask; 8970 8971 mask = (HOST_INTERFACE_VERSION_MASK << HOST_INTERFACE_VERSION_SHIFT); 8972 read_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG, &frame); 8973 /* Clear, then set field */ 8974 frame &= ~mask; 8975 frame |= ((u32)version << HOST_INTERFACE_VERSION_SHIFT); 8976 return load_8051_config(dd, RESERVED_REGISTERS, GENERAL_CONFIG, 8977 frame); 8978 } 8979 8980 void read_misc_status(struct hfi1_devdata *dd, u8 *ver_major, u8 *ver_minor, 8981 u8 *ver_patch) 8982 { 8983 u32 frame; 8984 8985 read_8051_config(dd, MISC_STATUS, GENERAL_CONFIG, &frame); 8986 *ver_major = (frame >> STS_FM_VERSION_MAJOR_SHIFT) & 8987 STS_FM_VERSION_MAJOR_MASK; 8988 *ver_minor = (frame >> STS_FM_VERSION_MINOR_SHIFT) & 8989 STS_FM_VERSION_MINOR_MASK; 8990 8991 read_8051_config(dd, VERSION_PATCH, GENERAL_CONFIG, &frame); 8992 *ver_patch = (frame >> STS_FM_VERSION_PATCH_SHIFT) & 8993 STS_FM_VERSION_PATCH_MASK; 8994 } 8995 8996 static void read_vc_remote_phy(struct hfi1_devdata *dd, u8 *power_management, 8997 u8 *continuous) 8998 { 8999 u32 frame; 9000 9001 read_8051_config(dd, VERIFY_CAP_REMOTE_PHY, GENERAL_CONFIG, &frame); 9002 *power_management = (frame >> POWER_MANAGEMENT_SHIFT) 9003 & POWER_MANAGEMENT_MASK; 9004 *continuous = (frame >> CONTINIOUS_REMOTE_UPDATE_SUPPORT_SHIFT) 9005 & CONTINIOUS_REMOTE_UPDATE_SUPPORT_MASK; 9006 } 9007 9008 static void read_vc_remote_fabric(struct hfi1_devdata *dd, u8 *vau, u8 *z, 9009 u8 *vcu, u16 *vl15buf, u8 *crc_sizes) 9010 { 9011 u32 frame; 9012 9013 read_8051_config(dd, VERIFY_CAP_REMOTE_FABRIC, GENERAL_CONFIG, &frame); 9014 *vau = (frame >> VAU_SHIFT) & VAU_MASK; 9015 *z = (frame >> Z_SHIFT) & Z_MASK; 9016 *vcu = (frame >> VCU_SHIFT) & VCU_MASK; 9017 *vl15buf = (frame >> VL15BUF_SHIFT) & VL15BUF_MASK; 9018 *crc_sizes = (frame >> CRC_SIZES_SHIFT) & CRC_SIZES_MASK; 9019 } 9020 9021 static void read_vc_remote_link_width(struct hfi1_devdata *dd, 9022 u8 *remote_tx_rate, 9023 u16 *link_widths) 9024 { 9025 u32 frame; 9026 9027 read_8051_config(dd, VERIFY_CAP_REMOTE_LINK_WIDTH, GENERAL_CONFIG, 9028 &frame); 9029 *remote_tx_rate = (frame >> REMOTE_TX_RATE_SHIFT) 9030 & REMOTE_TX_RATE_MASK; 9031 *link_widths = (frame >> LINK_WIDTH_SHIFT) & LINK_WIDTH_MASK; 9032 } 9033 9034 static void read_local_lni(struct hfi1_devdata *dd, u8 *enable_lane_rx) 9035 { 9036 u32 frame; 9037 9038 read_8051_config(dd, LOCAL_LNI_INFO, GENERAL_CONFIG, &frame); 9039 *enable_lane_rx = (frame >> ENABLE_LANE_RX_SHIFT) & ENABLE_LANE_RX_MASK; 9040 } 9041 9042 static void read_last_local_state(struct hfi1_devdata *dd, u32 *lls) 9043 { 9044 read_8051_config(dd, LAST_LOCAL_STATE_COMPLETE, GENERAL_CONFIG, lls); 9045 } 9046 9047 static void read_last_remote_state(struct hfi1_devdata *dd, u32 *lrs) 9048 { 9049 read_8051_config(dd, LAST_REMOTE_STATE_COMPLETE, GENERAL_CONFIG, lrs); 9050 } 9051 9052 void hfi1_read_link_quality(struct hfi1_devdata *dd, u8 *link_quality) 9053 { 9054 u32 frame; 9055 int ret; 9056 9057 *link_quality = 0; 9058 if (dd->pport->host_link_state & HLS_UP) { 9059 ret = read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG, 9060 &frame); 9061 if (ret == 0) 9062 *link_quality = (frame >> LINK_QUALITY_SHIFT) 9063 & LINK_QUALITY_MASK; 9064 } 9065 } 9066 9067 static void read_planned_down_reason_code(struct hfi1_devdata *dd, u8 *pdrrc) 9068 { 9069 u32 frame; 9070 9071 read_8051_config(dd, LINK_QUALITY_INFO, GENERAL_CONFIG, &frame); 9072 *pdrrc = (frame >> DOWN_REMOTE_REASON_SHIFT) & DOWN_REMOTE_REASON_MASK; 9073 } 9074 9075 static void read_link_down_reason(struct hfi1_devdata *dd, u8 *ldr) 9076 { 9077 u32 frame; 9078 9079 read_8051_config(dd, LINK_DOWN_REASON, GENERAL_CONFIG, &frame); 9080 *ldr = (frame & 0xff); 9081 } 9082 9083 static int read_tx_settings(struct hfi1_devdata *dd, 9084 u8 *enable_lane_tx, 9085 u8 *tx_polarity_inversion, 9086 u8 *rx_polarity_inversion, 9087 u8 *max_rate) 9088 { 9089 u32 frame; 9090 int ret; 9091 9092 ret = read_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, &frame); 9093 *enable_lane_tx = (frame >> ENABLE_LANE_TX_SHIFT) 9094 & ENABLE_LANE_TX_MASK; 9095 *tx_polarity_inversion = (frame >> TX_POLARITY_INVERSION_SHIFT) 9096 & TX_POLARITY_INVERSION_MASK; 9097 *rx_polarity_inversion = (frame >> RX_POLARITY_INVERSION_SHIFT) 9098 & RX_POLARITY_INVERSION_MASK; 9099 *max_rate = (frame >> MAX_RATE_SHIFT) & MAX_RATE_MASK; 9100 return ret; 9101 } 9102 9103 static int write_tx_settings(struct hfi1_devdata *dd, 9104 u8 enable_lane_tx, 9105 u8 tx_polarity_inversion, 9106 u8 rx_polarity_inversion, 9107 u8 max_rate) 9108 { 9109 u32 frame; 9110 9111 /* no need to mask, all variable sizes match field widths */ 9112 frame = enable_lane_tx << ENABLE_LANE_TX_SHIFT 9113 | tx_polarity_inversion << TX_POLARITY_INVERSION_SHIFT 9114 | rx_polarity_inversion << RX_POLARITY_INVERSION_SHIFT 9115 | max_rate << MAX_RATE_SHIFT; 9116 return load_8051_config(dd, TX_SETTINGS, GENERAL_CONFIG, frame); 9117 } 9118 9119 /* 9120 * Read an idle LCB message. 9121 * 9122 * Returns 0 on success, -EINVAL on error 9123 */ 9124 static int read_idle_message(struct hfi1_devdata *dd, u64 type, u64 *data_out) 9125 { 9126 int ret; 9127 9128 ret = do_8051_command(dd, HCMD_READ_LCB_IDLE_MSG, type, data_out); 9129 if (ret != HCMD_SUCCESS) { 9130 dd_dev_err(dd, "read idle message: type %d, err %d\n", 9131 (u32)type, ret); 9132 return -EINVAL; 9133 } 9134 dd_dev_info(dd, "%s: read idle message 0x%llx\n", __func__, *data_out); 9135 /* return only the payload as we already know the type */ 9136 *data_out >>= IDLE_PAYLOAD_SHIFT; 9137 return 0; 9138 } 9139 9140 /* 9141 * Read an idle SMA message. To be done in response to a notification from 9142 * the 8051. 9143 * 9144 * Returns 0 on success, -EINVAL on error 9145 */ 9146 static int read_idle_sma(struct hfi1_devdata *dd, u64 *data) 9147 { 9148 return read_idle_message(dd, (u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT, 9149 data); 9150 } 9151 9152 /* 9153 * Send an idle LCB message. 9154 * 9155 * Returns 0 on success, -EINVAL on error 9156 */ 9157 static int send_idle_message(struct hfi1_devdata *dd, u64 data) 9158 { 9159 int ret; 9160 9161 dd_dev_info(dd, "%s: sending idle message 0x%llx\n", __func__, data); 9162 ret = do_8051_command(dd, HCMD_SEND_LCB_IDLE_MSG, data, NULL); 9163 if (ret != HCMD_SUCCESS) { 9164 dd_dev_err(dd, "send idle message: data 0x%llx, err %d\n", 9165 data, ret); 9166 return -EINVAL; 9167 } 9168 return 0; 9169 } 9170 9171 /* 9172 * Send an idle SMA message. 9173 * 9174 * Returns 0 on success, -EINVAL on error 9175 */ 9176 int send_idle_sma(struct hfi1_devdata *dd, u64 message) 9177 { 9178 u64 data; 9179 9180 data = ((message & IDLE_PAYLOAD_MASK) << IDLE_PAYLOAD_SHIFT) | 9181 ((u64)IDLE_SMA << IDLE_MSG_TYPE_SHIFT); 9182 return send_idle_message(dd, data); 9183 } 9184 9185 /* 9186 * Initialize the LCB then do a quick link up. This may or may not be 9187 * in loopback. 9188 * 9189 * return 0 on success, -errno on error 9190 */ 9191 static int do_quick_linkup(struct hfi1_devdata *dd) 9192 { 9193 int ret; 9194 9195 lcb_shutdown(dd, 0); 9196 9197 if (loopback) { 9198 /* LCB_CFG_LOOPBACK.VAL = 2 */ 9199 /* LCB_CFG_LANE_WIDTH.VAL = 0 */ 9200 write_csr(dd, DC_LCB_CFG_LOOPBACK, 9201 IB_PACKET_TYPE << DC_LCB_CFG_LOOPBACK_VAL_SHIFT); 9202 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0); 9203 } 9204 9205 /* start the LCBs */ 9206 /* LCB_CFG_TX_FIFOS_RESET.VAL = 0 */ 9207 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0); 9208 9209 /* simulator only loopback steps */ 9210 if (loopback && dd->icode == ICODE_FUNCTIONAL_SIMULATOR) { 9211 /* LCB_CFG_RUN.EN = 1 */ 9212 write_csr(dd, DC_LCB_CFG_RUN, 9213 1ull << DC_LCB_CFG_RUN_EN_SHIFT); 9214 9215 ret = wait_link_transfer_active(dd, 10); 9216 if (ret) 9217 return ret; 9218 9219 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 9220 1ull << DC_LCB_CFG_ALLOW_LINK_UP_VAL_SHIFT); 9221 } 9222 9223 if (!loopback) { 9224 /* 9225 * When doing quick linkup and not in loopback, both 9226 * sides must be done with LCB set-up before either 9227 * starts the quick linkup. Put a delay here so that 9228 * both sides can be started and have a chance to be 9229 * done with LCB set up before resuming. 9230 */ 9231 dd_dev_err(dd, 9232 "Pausing for peer to be finished with LCB set up\n"); 9233 msleep(5000); 9234 dd_dev_err(dd, "Continuing with quick linkup\n"); 9235 } 9236 9237 write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */ 9238 set_8051_lcb_access(dd); 9239 9240 /* 9241 * State "quick" LinkUp request sets the physical link state to 9242 * LinkUp without a verify capability sequence. 9243 * This state is in simulator v37 and later. 9244 */ 9245 ret = set_physical_link_state(dd, PLS_QUICK_LINKUP); 9246 if (ret != HCMD_SUCCESS) { 9247 dd_dev_err(dd, 9248 "%s: set physical link state to quick LinkUp failed with return %d\n", 9249 __func__, ret); 9250 9251 set_host_lcb_access(dd); 9252 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */ 9253 9254 if (ret >= 0) 9255 ret = -EINVAL; 9256 return ret; 9257 } 9258 9259 return 0; /* success */ 9260 } 9261 9262 /* 9263 * Do all special steps to set up loopback. 9264 */ 9265 static int init_loopback(struct hfi1_devdata *dd) 9266 { 9267 dd_dev_info(dd, "Entering loopback mode\n"); 9268 9269 /* all loopbacks should disable self GUID check */ 9270 write_csr(dd, DC_DC8051_CFG_MODE, 9271 (read_csr(dd, DC_DC8051_CFG_MODE) | DISABLE_SELF_GUID_CHECK)); 9272 9273 /* 9274 * The simulator has only one loopback option - LCB. Switch 9275 * to that option, which includes quick link up. 9276 * 9277 * Accept all valid loopback values. 9278 */ 9279 if ((dd->icode == ICODE_FUNCTIONAL_SIMULATOR) && 9280 (loopback == LOOPBACK_SERDES || loopback == LOOPBACK_LCB || 9281 loopback == LOOPBACK_CABLE)) { 9282 loopback = LOOPBACK_LCB; 9283 quick_linkup = 1; 9284 return 0; 9285 } 9286 9287 /* 9288 * SerDes loopback init sequence is handled in set_local_link_attributes 9289 */ 9290 if (loopback == LOOPBACK_SERDES) 9291 return 0; 9292 9293 /* LCB loopback - handled at poll time */ 9294 if (loopback == LOOPBACK_LCB) { 9295 quick_linkup = 1; /* LCB is always quick linkup */ 9296 9297 /* not supported in emulation due to emulation RTL changes */ 9298 if (dd->icode == ICODE_FPGA_EMULATION) { 9299 dd_dev_err(dd, 9300 "LCB loopback not supported in emulation\n"); 9301 return -EINVAL; 9302 } 9303 return 0; 9304 } 9305 9306 /* external cable loopback requires no extra steps */ 9307 if (loopback == LOOPBACK_CABLE) 9308 return 0; 9309 9310 dd_dev_err(dd, "Invalid loopback mode %d\n", loopback); 9311 return -EINVAL; 9312 } 9313 9314 /* 9315 * Translate from the OPA_LINK_WIDTH handed to us by the FM to bits 9316 * used in the Verify Capability link width attribute. 9317 */ 9318 static u16 opa_to_vc_link_widths(u16 opa_widths) 9319 { 9320 int i; 9321 u16 result = 0; 9322 9323 static const struct link_bits { 9324 u16 from; 9325 u16 to; 9326 } opa_link_xlate[] = { 9327 { OPA_LINK_WIDTH_1X, 1 << (1 - 1) }, 9328 { OPA_LINK_WIDTH_2X, 1 << (2 - 1) }, 9329 { OPA_LINK_WIDTH_3X, 1 << (3 - 1) }, 9330 { OPA_LINK_WIDTH_4X, 1 << (4 - 1) }, 9331 }; 9332 9333 for (i = 0; i < ARRAY_SIZE(opa_link_xlate); i++) { 9334 if (opa_widths & opa_link_xlate[i].from) 9335 result |= opa_link_xlate[i].to; 9336 } 9337 return result; 9338 } 9339 9340 /* 9341 * Set link attributes before moving to polling. 9342 */ 9343 static int set_local_link_attributes(struct hfi1_pportdata *ppd) 9344 { 9345 struct hfi1_devdata *dd = ppd->dd; 9346 u8 enable_lane_tx; 9347 u8 tx_polarity_inversion; 9348 u8 rx_polarity_inversion; 9349 int ret; 9350 u32 misc_bits = 0; 9351 /* reset our fabric serdes to clear any lingering problems */ 9352 fabric_serdes_reset(dd); 9353 9354 /* set the local tx rate - need to read-modify-write */ 9355 ret = read_tx_settings(dd, &enable_lane_tx, &tx_polarity_inversion, 9356 &rx_polarity_inversion, &ppd->local_tx_rate); 9357 if (ret) 9358 goto set_local_link_attributes_fail; 9359 9360 if (dd->dc8051_ver < dc8051_ver(0, 20, 0)) { 9361 /* set the tx rate to the fastest enabled */ 9362 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G) 9363 ppd->local_tx_rate = 1; 9364 else 9365 ppd->local_tx_rate = 0; 9366 } else { 9367 /* set the tx rate to all enabled */ 9368 ppd->local_tx_rate = 0; 9369 if (ppd->link_speed_enabled & OPA_LINK_SPEED_25G) 9370 ppd->local_tx_rate |= 2; 9371 if (ppd->link_speed_enabled & OPA_LINK_SPEED_12_5G) 9372 ppd->local_tx_rate |= 1; 9373 } 9374 9375 enable_lane_tx = 0xF; /* enable all four lanes */ 9376 ret = write_tx_settings(dd, enable_lane_tx, tx_polarity_inversion, 9377 rx_polarity_inversion, ppd->local_tx_rate); 9378 if (ret != HCMD_SUCCESS) 9379 goto set_local_link_attributes_fail; 9380 9381 ret = write_host_interface_version(dd, HOST_INTERFACE_VERSION); 9382 if (ret != HCMD_SUCCESS) { 9383 dd_dev_err(dd, 9384 "Failed to set host interface version, return 0x%x\n", 9385 ret); 9386 goto set_local_link_attributes_fail; 9387 } 9388 9389 /* 9390 * DC supports continuous updates. 9391 */ 9392 ret = write_vc_local_phy(dd, 9393 0 /* no power management */, 9394 1 /* continuous updates */); 9395 if (ret != HCMD_SUCCESS) 9396 goto set_local_link_attributes_fail; 9397 9398 /* z=1 in the next call: AU of 0 is not supported by the hardware */ 9399 ret = write_vc_local_fabric(dd, dd->vau, 1, dd->vcu, dd->vl15_init, 9400 ppd->port_crc_mode_enabled); 9401 if (ret != HCMD_SUCCESS) 9402 goto set_local_link_attributes_fail; 9403 9404 /* 9405 * SerDes loopback init sequence requires 9406 * setting bit 0 of MISC_CONFIG_BITS 9407 */ 9408 if (loopback == LOOPBACK_SERDES) 9409 misc_bits |= 1 << LOOPBACK_SERDES_CONFIG_BIT_MASK_SHIFT; 9410 9411 /* 9412 * An external device configuration request is used to reset the LCB 9413 * to retry to obtain operational lanes when the first attempt is 9414 * unsuccesful. 9415 */ 9416 if (dd->dc8051_ver >= dc8051_ver(1, 25, 0)) 9417 misc_bits |= 1 << EXT_CFG_LCB_RESET_SUPPORTED_SHIFT; 9418 9419 ret = write_vc_local_link_mode(dd, misc_bits, 0, 9420 opa_to_vc_link_widths( 9421 ppd->link_width_enabled)); 9422 if (ret != HCMD_SUCCESS) 9423 goto set_local_link_attributes_fail; 9424 9425 /* let peer know who we are */ 9426 ret = write_local_device_id(dd, dd->pcidev->device, dd->minrev); 9427 if (ret == HCMD_SUCCESS) 9428 return 0; 9429 9430 set_local_link_attributes_fail: 9431 dd_dev_err(dd, 9432 "Failed to set local link attributes, return 0x%x\n", 9433 ret); 9434 return ret; 9435 } 9436 9437 /* 9438 * Call this to start the link. 9439 * Do not do anything if the link is disabled. 9440 * Returns 0 if link is disabled, moved to polling, or the driver is not ready. 9441 */ 9442 int start_link(struct hfi1_pportdata *ppd) 9443 { 9444 /* 9445 * Tune the SerDes to a ballpark setting for optimal signal and bit 9446 * error rate. Needs to be done before starting the link. 9447 */ 9448 tune_serdes(ppd); 9449 9450 if (!ppd->driver_link_ready) { 9451 dd_dev_info(ppd->dd, 9452 "%s: stopping link start because driver is not ready\n", 9453 __func__); 9454 return 0; 9455 } 9456 9457 /* 9458 * FULL_MGMT_P_KEY is cleared from the pkey table, so that the 9459 * pkey table can be configured properly if the HFI unit is connected 9460 * to switch port with MgmtAllowed=NO 9461 */ 9462 clear_full_mgmt_pkey(ppd); 9463 9464 return set_link_state(ppd, HLS_DN_POLL); 9465 } 9466 9467 static void wait_for_qsfp_init(struct hfi1_pportdata *ppd) 9468 { 9469 struct hfi1_devdata *dd = ppd->dd; 9470 u64 mask; 9471 unsigned long timeout; 9472 9473 /* 9474 * Some QSFP cables have a quirk that asserts the IntN line as a side 9475 * effect of power up on plug-in. We ignore this false positive 9476 * interrupt until the module has finished powering up by waiting for 9477 * a minimum timeout of the module inrush initialization time of 9478 * 500 ms (SFF 8679 Table 5-6) to ensure the voltage rails in the 9479 * module have stabilized. 9480 */ 9481 msleep(500); 9482 9483 /* 9484 * Check for QSFP interrupt for t_init (SFF 8679 Table 8-1) 9485 */ 9486 timeout = jiffies + msecs_to_jiffies(2000); 9487 while (1) { 9488 mask = read_csr(dd, dd->hfi1_id ? 9489 ASIC_QSFP2_IN : ASIC_QSFP1_IN); 9490 if (!(mask & QSFP_HFI0_INT_N)) 9491 break; 9492 if (time_after(jiffies, timeout)) { 9493 dd_dev_info(dd, "%s: No IntN detected, reset complete\n", 9494 __func__); 9495 break; 9496 } 9497 udelay(2); 9498 } 9499 } 9500 9501 static void set_qsfp_int_n(struct hfi1_pportdata *ppd, u8 enable) 9502 { 9503 struct hfi1_devdata *dd = ppd->dd; 9504 u64 mask; 9505 9506 mask = read_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK); 9507 if (enable) { 9508 /* 9509 * Clear the status register to avoid an immediate interrupt 9510 * when we re-enable the IntN pin 9511 */ 9512 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR, 9513 QSFP_HFI0_INT_N); 9514 mask |= (u64)QSFP_HFI0_INT_N; 9515 } else { 9516 mask &= ~(u64)QSFP_HFI0_INT_N; 9517 } 9518 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK, mask); 9519 } 9520 9521 int reset_qsfp(struct hfi1_pportdata *ppd) 9522 { 9523 struct hfi1_devdata *dd = ppd->dd; 9524 u64 mask, qsfp_mask; 9525 9526 /* Disable INT_N from triggering QSFP interrupts */ 9527 set_qsfp_int_n(ppd, 0); 9528 9529 /* Reset the QSFP */ 9530 mask = (u64)QSFP_HFI0_RESET_N; 9531 9532 qsfp_mask = read_csr(dd, 9533 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT); 9534 qsfp_mask &= ~mask; 9535 write_csr(dd, 9536 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask); 9537 9538 udelay(10); 9539 9540 qsfp_mask |= mask; 9541 write_csr(dd, 9542 dd->hfi1_id ? ASIC_QSFP2_OUT : ASIC_QSFP1_OUT, qsfp_mask); 9543 9544 wait_for_qsfp_init(ppd); 9545 9546 /* 9547 * Allow INT_N to trigger the QSFP interrupt to watch 9548 * for alarms and warnings 9549 */ 9550 set_qsfp_int_n(ppd, 1); 9551 9552 /* 9553 * After the reset, AOC transmitters are enabled by default. They need 9554 * to be turned off to complete the QSFP setup before they can be 9555 * enabled again. 9556 */ 9557 return set_qsfp_tx(ppd, 0); 9558 } 9559 9560 static int handle_qsfp_error_conditions(struct hfi1_pportdata *ppd, 9561 u8 *qsfp_interrupt_status) 9562 { 9563 struct hfi1_devdata *dd = ppd->dd; 9564 9565 if ((qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_ALARM) || 9566 (qsfp_interrupt_status[0] & QSFP_HIGH_TEMP_WARNING)) 9567 dd_dev_err(dd, "%s: QSFP cable temperature too high\n", 9568 __func__); 9569 9570 if ((qsfp_interrupt_status[0] & QSFP_LOW_TEMP_ALARM) || 9571 (qsfp_interrupt_status[0] & QSFP_LOW_TEMP_WARNING)) 9572 dd_dev_err(dd, "%s: QSFP cable temperature too low\n", 9573 __func__); 9574 9575 /* 9576 * The remaining alarms/warnings don't matter if the link is down. 9577 */ 9578 if (ppd->host_link_state & HLS_DOWN) 9579 return 0; 9580 9581 if ((qsfp_interrupt_status[1] & QSFP_HIGH_VCC_ALARM) || 9582 (qsfp_interrupt_status[1] & QSFP_HIGH_VCC_WARNING)) 9583 dd_dev_err(dd, "%s: QSFP supply voltage too high\n", 9584 __func__); 9585 9586 if ((qsfp_interrupt_status[1] & QSFP_LOW_VCC_ALARM) || 9587 (qsfp_interrupt_status[1] & QSFP_LOW_VCC_WARNING)) 9588 dd_dev_err(dd, "%s: QSFP supply voltage too low\n", 9589 __func__); 9590 9591 /* Byte 2 is vendor specific */ 9592 9593 if ((qsfp_interrupt_status[3] & QSFP_HIGH_POWER_ALARM) || 9594 (qsfp_interrupt_status[3] & QSFP_HIGH_POWER_WARNING)) 9595 dd_dev_err(dd, "%s: Cable RX channel 1/2 power too high\n", 9596 __func__); 9597 9598 if ((qsfp_interrupt_status[3] & QSFP_LOW_POWER_ALARM) || 9599 (qsfp_interrupt_status[3] & QSFP_LOW_POWER_WARNING)) 9600 dd_dev_err(dd, "%s: Cable RX channel 1/2 power too low\n", 9601 __func__); 9602 9603 if ((qsfp_interrupt_status[4] & QSFP_HIGH_POWER_ALARM) || 9604 (qsfp_interrupt_status[4] & QSFP_HIGH_POWER_WARNING)) 9605 dd_dev_err(dd, "%s: Cable RX channel 3/4 power too high\n", 9606 __func__); 9607 9608 if ((qsfp_interrupt_status[4] & QSFP_LOW_POWER_ALARM) || 9609 (qsfp_interrupt_status[4] & QSFP_LOW_POWER_WARNING)) 9610 dd_dev_err(dd, "%s: Cable RX channel 3/4 power too low\n", 9611 __func__); 9612 9613 if ((qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_ALARM) || 9614 (qsfp_interrupt_status[5] & QSFP_HIGH_BIAS_WARNING)) 9615 dd_dev_err(dd, "%s: Cable TX channel 1/2 bias too high\n", 9616 __func__); 9617 9618 if ((qsfp_interrupt_status[5] & QSFP_LOW_BIAS_ALARM) || 9619 (qsfp_interrupt_status[5] & QSFP_LOW_BIAS_WARNING)) 9620 dd_dev_err(dd, "%s: Cable TX channel 1/2 bias too low\n", 9621 __func__); 9622 9623 if ((qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_ALARM) || 9624 (qsfp_interrupt_status[6] & QSFP_HIGH_BIAS_WARNING)) 9625 dd_dev_err(dd, "%s: Cable TX channel 3/4 bias too high\n", 9626 __func__); 9627 9628 if ((qsfp_interrupt_status[6] & QSFP_LOW_BIAS_ALARM) || 9629 (qsfp_interrupt_status[6] & QSFP_LOW_BIAS_WARNING)) 9630 dd_dev_err(dd, "%s: Cable TX channel 3/4 bias too low\n", 9631 __func__); 9632 9633 if ((qsfp_interrupt_status[7] & QSFP_HIGH_POWER_ALARM) || 9634 (qsfp_interrupt_status[7] & QSFP_HIGH_POWER_WARNING)) 9635 dd_dev_err(dd, "%s: Cable TX channel 1/2 power too high\n", 9636 __func__); 9637 9638 if ((qsfp_interrupt_status[7] & QSFP_LOW_POWER_ALARM) || 9639 (qsfp_interrupt_status[7] & QSFP_LOW_POWER_WARNING)) 9640 dd_dev_err(dd, "%s: Cable TX channel 1/2 power too low\n", 9641 __func__); 9642 9643 if ((qsfp_interrupt_status[8] & QSFP_HIGH_POWER_ALARM) || 9644 (qsfp_interrupt_status[8] & QSFP_HIGH_POWER_WARNING)) 9645 dd_dev_err(dd, "%s: Cable TX channel 3/4 power too high\n", 9646 __func__); 9647 9648 if ((qsfp_interrupt_status[8] & QSFP_LOW_POWER_ALARM) || 9649 (qsfp_interrupt_status[8] & QSFP_LOW_POWER_WARNING)) 9650 dd_dev_err(dd, "%s: Cable TX channel 3/4 power too low\n", 9651 __func__); 9652 9653 /* Bytes 9-10 and 11-12 are reserved */ 9654 /* Bytes 13-15 are vendor specific */ 9655 9656 return 0; 9657 } 9658 9659 /* This routine will only be scheduled if the QSFP module present is asserted */ 9660 void qsfp_event(struct work_struct *work) 9661 { 9662 struct qsfp_data *qd; 9663 struct hfi1_pportdata *ppd; 9664 struct hfi1_devdata *dd; 9665 9666 qd = container_of(work, struct qsfp_data, qsfp_work); 9667 ppd = qd->ppd; 9668 dd = ppd->dd; 9669 9670 /* Sanity check */ 9671 if (!qsfp_mod_present(ppd)) 9672 return; 9673 9674 if (ppd->host_link_state == HLS_DN_DISABLE) { 9675 dd_dev_info(ppd->dd, 9676 "%s: stopping link start because link is disabled\n", 9677 __func__); 9678 return; 9679 } 9680 9681 /* 9682 * Turn DC back on after cable has been re-inserted. Up until 9683 * now, the DC has been in reset to save power. 9684 */ 9685 dc_start(dd); 9686 9687 if (qd->cache_refresh_required) { 9688 set_qsfp_int_n(ppd, 0); 9689 9690 wait_for_qsfp_init(ppd); 9691 9692 /* 9693 * Allow INT_N to trigger the QSFP interrupt to watch 9694 * for alarms and warnings 9695 */ 9696 set_qsfp_int_n(ppd, 1); 9697 9698 start_link(ppd); 9699 } 9700 9701 if (qd->check_interrupt_flags) { 9702 u8 qsfp_interrupt_status[16] = {0,}; 9703 9704 if (one_qsfp_read(ppd, dd->hfi1_id, 6, 9705 &qsfp_interrupt_status[0], 16) != 16) { 9706 dd_dev_info(dd, 9707 "%s: Failed to read status of QSFP module\n", 9708 __func__); 9709 } else { 9710 unsigned long flags; 9711 9712 handle_qsfp_error_conditions( 9713 ppd, qsfp_interrupt_status); 9714 spin_lock_irqsave(&ppd->qsfp_info.qsfp_lock, flags); 9715 ppd->qsfp_info.check_interrupt_flags = 0; 9716 spin_unlock_irqrestore(&ppd->qsfp_info.qsfp_lock, 9717 flags); 9718 } 9719 } 9720 } 9721 9722 void init_qsfp_int(struct hfi1_devdata *dd) 9723 { 9724 struct hfi1_pportdata *ppd = dd->pport; 9725 u64 qsfp_mask; 9726 9727 qsfp_mask = (u64)(QSFP_HFI0_INT_N | QSFP_HFI0_MODPRST_N); 9728 /* Clear current status to avoid spurious interrupts */ 9729 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_CLEAR : ASIC_QSFP1_CLEAR, 9730 qsfp_mask); 9731 write_csr(dd, dd->hfi1_id ? ASIC_QSFP2_MASK : ASIC_QSFP1_MASK, 9732 qsfp_mask); 9733 9734 set_qsfp_int_n(ppd, 0); 9735 9736 /* Handle active low nature of INT_N and MODPRST_N pins */ 9737 if (qsfp_mod_present(ppd)) 9738 qsfp_mask &= ~(u64)QSFP_HFI0_MODPRST_N; 9739 write_csr(dd, 9740 dd->hfi1_id ? ASIC_QSFP2_INVERT : ASIC_QSFP1_INVERT, 9741 qsfp_mask); 9742 9743 /* Enable the appropriate QSFP IRQ source */ 9744 if (!dd->hfi1_id) 9745 set_intr_bits(dd, QSFP1_INT, QSFP1_INT, true); 9746 else 9747 set_intr_bits(dd, QSFP2_INT, QSFP2_INT, true); 9748 } 9749 9750 /* 9751 * Do a one-time initialize of the LCB block. 9752 */ 9753 static void init_lcb(struct hfi1_devdata *dd) 9754 { 9755 /* simulator does not correctly handle LCB cclk loopback, skip */ 9756 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) 9757 return; 9758 9759 /* the DC has been reset earlier in the driver load */ 9760 9761 /* set LCB for cclk loopback on the port */ 9762 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x01); 9763 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0x00); 9764 write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0x00); 9765 write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110); 9766 write_csr(dd, DC_LCB_CFG_CLK_CNTR, 0x08); 9767 write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x02); 9768 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0x00); 9769 } 9770 9771 /* 9772 * Perform a test read on the QSFP. Return 0 on success, -ERRNO 9773 * on error. 9774 */ 9775 static int test_qsfp_read(struct hfi1_pportdata *ppd) 9776 { 9777 int ret; 9778 u8 status; 9779 9780 /* 9781 * Report success if not a QSFP or, if it is a QSFP, but the cable is 9782 * not present 9783 */ 9784 if (ppd->port_type != PORT_TYPE_QSFP || !qsfp_mod_present(ppd)) 9785 return 0; 9786 9787 /* read byte 2, the status byte */ 9788 ret = one_qsfp_read(ppd, ppd->dd->hfi1_id, 2, &status, 1); 9789 if (ret < 0) 9790 return ret; 9791 if (ret != 1) 9792 return -EIO; 9793 9794 return 0; /* success */ 9795 } 9796 9797 /* 9798 * Values for QSFP retry. 9799 * 9800 * Give up after 10s (20 x 500ms). The overall timeout was empirically 9801 * arrived at from experience on a large cluster. 9802 */ 9803 #define MAX_QSFP_RETRIES 20 9804 #define QSFP_RETRY_WAIT 500 /* msec */ 9805 9806 /* 9807 * Try a QSFP read. If it fails, schedule a retry for later. 9808 * Called on first link activation after driver load. 9809 */ 9810 static void try_start_link(struct hfi1_pportdata *ppd) 9811 { 9812 if (test_qsfp_read(ppd)) { 9813 /* read failed */ 9814 if (ppd->qsfp_retry_count >= MAX_QSFP_RETRIES) { 9815 dd_dev_err(ppd->dd, "QSFP not responding, giving up\n"); 9816 return; 9817 } 9818 dd_dev_info(ppd->dd, 9819 "QSFP not responding, waiting and retrying %d\n", 9820 (int)ppd->qsfp_retry_count); 9821 ppd->qsfp_retry_count++; 9822 queue_delayed_work(ppd->link_wq, &ppd->start_link_work, 9823 msecs_to_jiffies(QSFP_RETRY_WAIT)); 9824 return; 9825 } 9826 ppd->qsfp_retry_count = 0; 9827 9828 start_link(ppd); 9829 } 9830 9831 /* 9832 * Workqueue function to start the link after a delay. 9833 */ 9834 void handle_start_link(struct work_struct *work) 9835 { 9836 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata, 9837 start_link_work.work); 9838 try_start_link(ppd); 9839 } 9840 9841 int bringup_serdes(struct hfi1_pportdata *ppd) 9842 { 9843 struct hfi1_devdata *dd = ppd->dd; 9844 u64 guid; 9845 int ret; 9846 9847 if (HFI1_CAP_IS_KSET(EXTENDED_PSN)) 9848 add_rcvctrl(dd, RCV_CTRL_RCV_EXTENDED_PSN_ENABLE_SMASK); 9849 9850 guid = ppd->guids[HFI1_PORT_GUID_INDEX]; 9851 if (!guid) { 9852 if (dd->base_guid) 9853 guid = dd->base_guid + ppd->port - 1; 9854 ppd->guids[HFI1_PORT_GUID_INDEX] = guid; 9855 } 9856 9857 /* Set linkinit_reason on power up per OPA spec */ 9858 ppd->linkinit_reason = OPA_LINKINIT_REASON_LINKUP; 9859 9860 /* one-time init of the LCB */ 9861 init_lcb(dd); 9862 9863 if (loopback) { 9864 ret = init_loopback(dd); 9865 if (ret < 0) 9866 return ret; 9867 } 9868 9869 get_port_type(ppd); 9870 if (ppd->port_type == PORT_TYPE_QSFP) { 9871 set_qsfp_int_n(ppd, 0); 9872 wait_for_qsfp_init(ppd); 9873 set_qsfp_int_n(ppd, 1); 9874 } 9875 9876 try_start_link(ppd); 9877 return 0; 9878 } 9879 9880 void hfi1_quiet_serdes(struct hfi1_pportdata *ppd) 9881 { 9882 struct hfi1_devdata *dd = ppd->dd; 9883 9884 /* 9885 * Shut down the link and keep it down. First turn off that the 9886 * driver wants to allow the link to be up (driver_link_ready). 9887 * Then make sure the link is not automatically restarted 9888 * (link_enabled). Cancel any pending restart. And finally 9889 * go offline. 9890 */ 9891 ppd->driver_link_ready = 0; 9892 ppd->link_enabled = 0; 9893 9894 ppd->qsfp_retry_count = MAX_QSFP_RETRIES; /* prevent more retries */ 9895 flush_delayed_work(&ppd->start_link_work); 9896 cancel_delayed_work_sync(&ppd->start_link_work); 9897 9898 ppd->offline_disabled_reason = 9899 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_REBOOT); 9900 set_link_down_reason(ppd, OPA_LINKDOWN_REASON_REBOOT, 0, 9901 OPA_LINKDOWN_REASON_REBOOT); 9902 set_link_state(ppd, HLS_DN_OFFLINE); 9903 9904 /* disable the port */ 9905 clear_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 9906 cancel_work_sync(&ppd->freeze_work); 9907 } 9908 9909 static inline int init_cpu_counters(struct hfi1_devdata *dd) 9910 { 9911 struct hfi1_pportdata *ppd; 9912 int i; 9913 9914 ppd = (struct hfi1_pportdata *)(dd + 1); 9915 for (i = 0; i < dd->num_pports; i++, ppd++) { 9916 ppd->ibport_data.rvp.rc_acks = NULL; 9917 ppd->ibport_data.rvp.rc_qacks = NULL; 9918 ppd->ibport_data.rvp.rc_acks = alloc_percpu(u64); 9919 ppd->ibport_data.rvp.rc_qacks = alloc_percpu(u64); 9920 ppd->ibport_data.rvp.rc_delayed_comp = alloc_percpu(u64); 9921 if (!ppd->ibport_data.rvp.rc_acks || 9922 !ppd->ibport_data.rvp.rc_delayed_comp || 9923 !ppd->ibport_data.rvp.rc_qacks) 9924 return -ENOMEM; 9925 } 9926 9927 return 0; 9928 } 9929 9930 /* 9931 * index is the index into the receive array 9932 */ 9933 void hfi1_put_tid(struct hfi1_devdata *dd, u32 index, 9934 u32 type, unsigned long pa, u16 order) 9935 { 9936 u64 reg; 9937 9938 if (!(dd->flags & HFI1_PRESENT)) 9939 goto done; 9940 9941 if (type == PT_INVALID || type == PT_INVALID_FLUSH) { 9942 pa = 0; 9943 order = 0; 9944 } else if (type > PT_INVALID) { 9945 dd_dev_err(dd, 9946 "unexpected receive array type %u for index %u, not handled\n", 9947 type, index); 9948 goto done; 9949 } 9950 trace_hfi1_put_tid(dd, index, type, pa, order); 9951 9952 #define RT_ADDR_SHIFT 12 /* 4KB kernel address boundary */ 9953 reg = RCV_ARRAY_RT_WRITE_ENABLE_SMASK 9954 | (u64)order << RCV_ARRAY_RT_BUF_SIZE_SHIFT 9955 | ((pa >> RT_ADDR_SHIFT) & RCV_ARRAY_RT_ADDR_MASK) 9956 << RCV_ARRAY_RT_ADDR_SHIFT; 9957 trace_hfi1_write_rcvarray(dd->rcvarray_wc + (index * 8), reg); 9958 writeq(reg, dd->rcvarray_wc + (index * 8)); 9959 9960 if (type == PT_EAGER || type == PT_INVALID_FLUSH || (index & 3) == 3) 9961 /* 9962 * Eager entries are written and flushed 9963 * 9964 * Expected entries are flushed every 4 writes 9965 */ 9966 flush_wc(); 9967 done: 9968 return; 9969 } 9970 9971 void hfi1_clear_tids(struct hfi1_ctxtdata *rcd) 9972 { 9973 struct hfi1_devdata *dd = rcd->dd; 9974 u32 i; 9975 9976 /* this could be optimized */ 9977 for (i = rcd->eager_base; i < rcd->eager_base + 9978 rcd->egrbufs.alloced; i++) 9979 hfi1_put_tid(dd, i, PT_INVALID, 0, 0); 9980 9981 for (i = rcd->expected_base; 9982 i < rcd->expected_base + rcd->expected_count; i++) 9983 hfi1_put_tid(dd, i, PT_INVALID, 0, 0); 9984 } 9985 9986 static const char * const ib_cfg_name_strings[] = { 9987 "HFI1_IB_CFG_LIDLMC", 9988 "HFI1_IB_CFG_LWID_DG_ENB", 9989 "HFI1_IB_CFG_LWID_ENB", 9990 "HFI1_IB_CFG_LWID", 9991 "HFI1_IB_CFG_SPD_ENB", 9992 "HFI1_IB_CFG_SPD", 9993 "HFI1_IB_CFG_RXPOL_ENB", 9994 "HFI1_IB_CFG_LREV_ENB", 9995 "HFI1_IB_CFG_LINKLATENCY", 9996 "HFI1_IB_CFG_HRTBT", 9997 "HFI1_IB_CFG_OP_VLS", 9998 "HFI1_IB_CFG_VL_HIGH_CAP", 9999 "HFI1_IB_CFG_VL_LOW_CAP", 10000 "HFI1_IB_CFG_OVERRUN_THRESH", 10001 "HFI1_IB_CFG_PHYERR_THRESH", 10002 "HFI1_IB_CFG_LINKDEFAULT", 10003 "HFI1_IB_CFG_PKEYS", 10004 "HFI1_IB_CFG_MTU", 10005 "HFI1_IB_CFG_LSTATE", 10006 "HFI1_IB_CFG_VL_HIGH_LIMIT", 10007 "HFI1_IB_CFG_PMA_TICKS", 10008 "HFI1_IB_CFG_PORT" 10009 }; 10010 10011 static const char *ib_cfg_name(int which) 10012 { 10013 if (which < 0 || which >= ARRAY_SIZE(ib_cfg_name_strings)) 10014 return "invalid"; 10015 return ib_cfg_name_strings[which]; 10016 } 10017 10018 int hfi1_get_ib_cfg(struct hfi1_pportdata *ppd, int which) 10019 { 10020 struct hfi1_devdata *dd = ppd->dd; 10021 int val = 0; 10022 10023 switch (which) { 10024 case HFI1_IB_CFG_LWID_ENB: /* allowed Link-width */ 10025 val = ppd->link_width_enabled; 10026 break; 10027 case HFI1_IB_CFG_LWID: /* currently active Link-width */ 10028 val = ppd->link_width_active; 10029 break; 10030 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */ 10031 val = ppd->link_speed_enabled; 10032 break; 10033 case HFI1_IB_CFG_SPD: /* current Link speed */ 10034 val = ppd->link_speed_active; 10035 break; 10036 10037 case HFI1_IB_CFG_RXPOL_ENB: /* Auto-RX-polarity enable */ 10038 case HFI1_IB_CFG_LREV_ENB: /* Auto-Lane-reversal enable */ 10039 case HFI1_IB_CFG_LINKLATENCY: 10040 goto unimplemented; 10041 10042 case HFI1_IB_CFG_OP_VLS: 10043 val = ppd->actual_vls_operational; 10044 break; 10045 case HFI1_IB_CFG_VL_HIGH_CAP: /* VL arb high priority table size */ 10046 val = VL_ARB_HIGH_PRIO_TABLE_SIZE; 10047 break; 10048 case HFI1_IB_CFG_VL_LOW_CAP: /* VL arb low priority table size */ 10049 val = VL_ARB_LOW_PRIO_TABLE_SIZE; 10050 break; 10051 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */ 10052 val = ppd->overrun_threshold; 10053 break; 10054 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */ 10055 val = ppd->phy_error_threshold; 10056 break; 10057 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */ 10058 val = HLS_DEFAULT; 10059 break; 10060 10061 case HFI1_IB_CFG_HRTBT: /* Heartbeat off/enable/auto */ 10062 case HFI1_IB_CFG_PMA_TICKS: 10063 default: 10064 unimplemented: 10065 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL)) 10066 dd_dev_info( 10067 dd, 10068 "%s: which %s: not implemented\n", 10069 __func__, 10070 ib_cfg_name(which)); 10071 break; 10072 } 10073 10074 return val; 10075 } 10076 10077 /* 10078 * The largest MAD packet size. 10079 */ 10080 #define MAX_MAD_PACKET 2048 10081 10082 /* 10083 * Return the maximum header bytes that can go on the _wire_ 10084 * for this device. This count includes the ICRC which is 10085 * not part of the packet held in memory but it is appended 10086 * by the HW. 10087 * This is dependent on the device's receive header entry size. 10088 * HFI allows this to be set per-receive context, but the 10089 * driver presently enforces a global value. 10090 */ 10091 u32 lrh_max_header_bytes(struct hfi1_devdata *dd) 10092 { 10093 /* 10094 * The maximum non-payload (MTU) bytes in LRH.PktLen are 10095 * the Receive Header Entry Size minus the PBC (or RHF) size 10096 * plus one DW for the ICRC appended by HW. 10097 * 10098 * dd->rcd[0].rcvhdrqentsize is in DW. 10099 * We use rcd[0] as all context will have the same value. Also, 10100 * the first kernel context would have been allocated by now so 10101 * we are guaranteed a valid value. 10102 */ 10103 return (get_hdrqentsize(dd->rcd[0]) - 2/*PBC/RHF*/ + 1/*ICRC*/) << 2; 10104 } 10105 10106 /* 10107 * Set Send Length 10108 * @ppd: per port data 10109 * 10110 * Set the MTU by limiting how many DWs may be sent. The SendLenCheck* 10111 * registers compare against LRH.PktLen, so use the max bytes included 10112 * in the LRH. 10113 * 10114 * This routine changes all VL values except VL15, which it maintains at 10115 * the same value. 10116 */ 10117 static void set_send_length(struct hfi1_pportdata *ppd) 10118 { 10119 struct hfi1_devdata *dd = ppd->dd; 10120 u32 max_hb = lrh_max_header_bytes(dd), dcmtu; 10121 u32 maxvlmtu = dd->vld[15].mtu; 10122 u64 len1 = 0, len2 = (((dd->vld[15].mtu + max_hb) >> 2) 10123 & SEND_LEN_CHECK1_LEN_VL15_MASK) << 10124 SEND_LEN_CHECK1_LEN_VL15_SHIFT; 10125 int i, j; 10126 u32 thres; 10127 10128 for (i = 0; i < ppd->vls_supported; i++) { 10129 if (dd->vld[i].mtu > maxvlmtu) 10130 maxvlmtu = dd->vld[i].mtu; 10131 if (i <= 3) 10132 len1 |= (((dd->vld[i].mtu + max_hb) >> 2) 10133 & SEND_LEN_CHECK0_LEN_VL0_MASK) << 10134 ((i % 4) * SEND_LEN_CHECK0_LEN_VL1_SHIFT); 10135 else 10136 len2 |= (((dd->vld[i].mtu + max_hb) >> 2) 10137 & SEND_LEN_CHECK1_LEN_VL4_MASK) << 10138 ((i % 4) * SEND_LEN_CHECK1_LEN_VL5_SHIFT); 10139 } 10140 write_csr(dd, SEND_LEN_CHECK0, len1); 10141 write_csr(dd, SEND_LEN_CHECK1, len2); 10142 /* adjust kernel credit return thresholds based on new MTUs */ 10143 /* all kernel receive contexts have the same hdrqentsize */ 10144 for (i = 0; i < ppd->vls_supported; i++) { 10145 thres = min(sc_percent_to_threshold(dd->vld[i].sc, 50), 10146 sc_mtu_to_threshold(dd->vld[i].sc, 10147 dd->vld[i].mtu, 10148 get_hdrqentsize(dd->rcd[0]))); 10149 for (j = 0; j < INIT_SC_PER_VL; j++) 10150 sc_set_cr_threshold( 10151 pio_select_send_context_vl(dd, j, i), 10152 thres); 10153 } 10154 thres = min(sc_percent_to_threshold(dd->vld[15].sc, 50), 10155 sc_mtu_to_threshold(dd->vld[15].sc, 10156 dd->vld[15].mtu, 10157 dd->rcd[0]->rcvhdrqentsize)); 10158 sc_set_cr_threshold(dd->vld[15].sc, thres); 10159 10160 /* Adjust maximum MTU for the port in DC */ 10161 dcmtu = maxvlmtu == 10240 ? DCC_CFG_PORT_MTU_CAP_10240 : 10162 (ilog2(maxvlmtu >> 8) + 1); 10163 len1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG); 10164 len1 &= ~DCC_CFG_PORT_CONFIG_MTU_CAP_SMASK; 10165 len1 |= ((u64)dcmtu & DCC_CFG_PORT_CONFIG_MTU_CAP_MASK) << 10166 DCC_CFG_PORT_CONFIG_MTU_CAP_SHIFT; 10167 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG, len1); 10168 } 10169 10170 static void set_lidlmc(struct hfi1_pportdata *ppd) 10171 { 10172 int i; 10173 u64 sreg = 0; 10174 struct hfi1_devdata *dd = ppd->dd; 10175 u32 mask = ~((1U << ppd->lmc) - 1); 10176 u64 c1 = read_csr(ppd->dd, DCC_CFG_PORT_CONFIG1); 10177 u32 lid; 10178 10179 /* 10180 * Program 0 in CSR if port lid is extended. This prevents 10181 * 9B packets being sent out for large lids. 10182 */ 10183 lid = (ppd->lid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) ? 0 : ppd->lid; 10184 c1 &= ~(DCC_CFG_PORT_CONFIG1_TARGET_DLID_SMASK 10185 | DCC_CFG_PORT_CONFIG1_DLID_MASK_SMASK); 10186 c1 |= ((lid & DCC_CFG_PORT_CONFIG1_TARGET_DLID_MASK) 10187 << DCC_CFG_PORT_CONFIG1_TARGET_DLID_SHIFT) | 10188 ((mask & DCC_CFG_PORT_CONFIG1_DLID_MASK_MASK) 10189 << DCC_CFG_PORT_CONFIG1_DLID_MASK_SHIFT); 10190 write_csr(ppd->dd, DCC_CFG_PORT_CONFIG1, c1); 10191 10192 /* 10193 * Iterate over all the send contexts and set their SLID check 10194 */ 10195 sreg = ((mask & SEND_CTXT_CHECK_SLID_MASK_MASK) << 10196 SEND_CTXT_CHECK_SLID_MASK_SHIFT) | 10197 (((lid & mask) & SEND_CTXT_CHECK_SLID_VALUE_MASK) << 10198 SEND_CTXT_CHECK_SLID_VALUE_SHIFT); 10199 10200 for (i = 0; i < chip_send_contexts(dd); i++) { 10201 hfi1_cdbg(LINKVERB, "SendContext[%d].SLID_CHECK = 0x%x", 10202 i, (u32)sreg); 10203 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, sreg); 10204 } 10205 10206 /* Now we have to do the same thing for the sdma engines */ 10207 sdma_update_lmc(dd, mask, lid); 10208 } 10209 10210 static const char *state_completed_string(u32 completed) 10211 { 10212 static const char * const state_completed[] = { 10213 "EstablishComm", 10214 "OptimizeEQ", 10215 "VerifyCap" 10216 }; 10217 10218 if (completed < ARRAY_SIZE(state_completed)) 10219 return state_completed[completed]; 10220 10221 return "unknown"; 10222 } 10223 10224 static const char all_lanes_dead_timeout_expired[] = 10225 "All lanes were inactive – was the interconnect media removed?"; 10226 static const char tx_out_of_policy[] = 10227 "Passing lanes on local port do not meet the local link width policy"; 10228 static const char no_state_complete[] = 10229 "State timeout occurred before link partner completed the state"; 10230 static const char * const state_complete_reasons[] = { 10231 [0x00] = "Reason unknown", 10232 [0x01] = "Link was halted by driver, refer to LinkDownReason", 10233 [0x02] = "Link partner reported failure", 10234 [0x10] = "Unable to achieve frame sync on any lane", 10235 [0x11] = 10236 "Unable to find a common bit rate with the link partner", 10237 [0x12] = 10238 "Unable to achieve frame sync on sufficient lanes to meet the local link width policy", 10239 [0x13] = 10240 "Unable to identify preset equalization on sufficient lanes to meet the local link width policy", 10241 [0x14] = no_state_complete, 10242 [0x15] = 10243 "State timeout occurred before link partner identified equalization presets", 10244 [0x16] = 10245 "Link partner completed the EstablishComm state, but the passing lanes do not meet the local link width policy", 10246 [0x17] = tx_out_of_policy, 10247 [0x20] = all_lanes_dead_timeout_expired, 10248 [0x21] = 10249 "Unable to achieve acceptable BER on sufficient lanes to meet the local link width policy", 10250 [0x22] = no_state_complete, 10251 [0x23] = 10252 "Link partner completed the OptimizeEq state, but the passing lanes do not meet the local link width policy", 10253 [0x24] = tx_out_of_policy, 10254 [0x30] = all_lanes_dead_timeout_expired, 10255 [0x31] = 10256 "State timeout occurred waiting for host to process received frames", 10257 [0x32] = no_state_complete, 10258 [0x33] = 10259 "Link partner completed the VerifyCap state, but the passing lanes do not meet the local link width policy", 10260 [0x34] = tx_out_of_policy, 10261 [0x35] = "Negotiated link width is mutually exclusive", 10262 [0x36] = 10263 "Timed out before receiving verifycap frames in VerifyCap.Exchange", 10264 [0x37] = "Unable to resolve secure data exchange", 10265 }; 10266 10267 static const char *state_complete_reason_code_string(struct hfi1_pportdata *ppd, 10268 u32 code) 10269 { 10270 const char *str = NULL; 10271 10272 if (code < ARRAY_SIZE(state_complete_reasons)) 10273 str = state_complete_reasons[code]; 10274 10275 if (str) 10276 return str; 10277 return "Reserved"; 10278 } 10279 10280 /* describe the given last state complete frame */ 10281 static void decode_state_complete(struct hfi1_pportdata *ppd, u32 frame, 10282 const char *prefix) 10283 { 10284 struct hfi1_devdata *dd = ppd->dd; 10285 u32 success; 10286 u32 state; 10287 u32 reason; 10288 u32 lanes; 10289 10290 /* 10291 * Decode frame: 10292 * [ 0: 0] - success 10293 * [ 3: 1] - state 10294 * [ 7: 4] - next state timeout 10295 * [15: 8] - reason code 10296 * [31:16] - lanes 10297 */ 10298 success = frame & 0x1; 10299 state = (frame >> 1) & 0x7; 10300 reason = (frame >> 8) & 0xff; 10301 lanes = (frame >> 16) & 0xffff; 10302 10303 dd_dev_err(dd, "Last %s LNI state complete frame 0x%08x:\n", 10304 prefix, frame); 10305 dd_dev_err(dd, " last reported state state: %s (0x%x)\n", 10306 state_completed_string(state), state); 10307 dd_dev_err(dd, " state successfully completed: %s\n", 10308 success ? "yes" : "no"); 10309 dd_dev_err(dd, " fail reason 0x%x: %s\n", 10310 reason, state_complete_reason_code_string(ppd, reason)); 10311 dd_dev_err(dd, " passing lane mask: 0x%x", lanes); 10312 } 10313 10314 /* 10315 * Read the last state complete frames and explain them. This routine 10316 * expects to be called if the link went down during link negotiation 10317 * and initialization (LNI). That is, anywhere between polling and link up. 10318 */ 10319 static void check_lni_states(struct hfi1_pportdata *ppd) 10320 { 10321 u32 last_local_state; 10322 u32 last_remote_state; 10323 10324 read_last_local_state(ppd->dd, &last_local_state); 10325 read_last_remote_state(ppd->dd, &last_remote_state); 10326 10327 /* 10328 * Don't report anything if there is nothing to report. A value of 10329 * 0 means the link was taken down while polling and there was no 10330 * training in-process. 10331 */ 10332 if (last_local_state == 0 && last_remote_state == 0) 10333 return; 10334 10335 decode_state_complete(ppd, last_local_state, "transmitted"); 10336 decode_state_complete(ppd, last_remote_state, "received"); 10337 } 10338 10339 /* wait for wait_ms for LINK_TRANSFER_ACTIVE to go to 1 */ 10340 static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms) 10341 { 10342 u64 reg; 10343 unsigned long timeout; 10344 10345 /* watch LCB_STS_LINK_TRANSFER_ACTIVE */ 10346 timeout = jiffies + msecs_to_jiffies(wait_ms); 10347 while (1) { 10348 reg = read_csr(dd, DC_LCB_STS_LINK_TRANSFER_ACTIVE); 10349 if (reg) 10350 break; 10351 if (time_after(jiffies, timeout)) { 10352 dd_dev_err(dd, 10353 "timeout waiting for LINK_TRANSFER_ACTIVE\n"); 10354 return -ETIMEDOUT; 10355 } 10356 udelay(2); 10357 } 10358 return 0; 10359 } 10360 10361 /* called when the logical link state is not down as it should be */ 10362 static void force_logical_link_state_down(struct hfi1_pportdata *ppd) 10363 { 10364 struct hfi1_devdata *dd = ppd->dd; 10365 10366 /* 10367 * Bring link up in LCB loopback 10368 */ 10369 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 1); 10370 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK, 10371 DC_LCB_CFG_IGNORE_LOST_RCLK_EN_SMASK); 10372 10373 write_csr(dd, DC_LCB_CFG_LANE_WIDTH, 0); 10374 write_csr(dd, DC_LCB_CFG_REINIT_AS_SLAVE, 0); 10375 write_csr(dd, DC_LCB_CFG_CNT_FOR_SKIP_STALL, 0x110); 10376 write_csr(dd, DC_LCB_CFG_LOOPBACK, 0x2); 10377 10378 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 0); 10379 (void)read_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET); 10380 udelay(3); 10381 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 1); 10382 write_csr(dd, DC_LCB_CFG_RUN, 1ull << DC_LCB_CFG_RUN_EN_SHIFT); 10383 10384 wait_link_transfer_active(dd, 100); 10385 10386 /* 10387 * Bring the link down again. 10388 */ 10389 write_csr(dd, DC_LCB_CFG_TX_FIFOS_RESET, 1); 10390 write_csr(dd, DC_LCB_CFG_ALLOW_LINK_UP, 0); 10391 write_csr(dd, DC_LCB_CFG_IGNORE_LOST_RCLK, 0); 10392 10393 dd_dev_info(ppd->dd, "logical state forced to LINK_DOWN\n"); 10394 } 10395 10396 /* 10397 * Helper for set_link_state(). Do not call except from that routine. 10398 * Expects ppd->hls_mutex to be held. 10399 * 10400 * @rem_reason value to be sent to the neighbor 10401 * 10402 * LinkDownReasons only set if transition succeeds. 10403 */ 10404 static int goto_offline(struct hfi1_pportdata *ppd, u8 rem_reason) 10405 { 10406 struct hfi1_devdata *dd = ppd->dd; 10407 u32 previous_state; 10408 int offline_state_ret; 10409 int ret; 10410 10411 update_lcb_cache(dd); 10412 10413 previous_state = ppd->host_link_state; 10414 ppd->host_link_state = HLS_GOING_OFFLINE; 10415 10416 /* start offline transition */ 10417 ret = set_physical_link_state(dd, (rem_reason << 8) | PLS_OFFLINE); 10418 10419 if (ret != HCMD_SUCCESS) { 10420 dd_dev_err(dd, 10421 "Failed to transition to Offline link state, return %d\n", 10422 ret); 10423 return -EINVAL; 10424 } 10425 if (ppd->offline_disabled_reason == 10426 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE)) 10427 ppd->offline_disabled_reason = 10428 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_TRANSIENT); 10429 10430 offline_state_ret = wait_phys_link_offline_substates(ppd, 10000); 10431 if (offline_state_ret < 0) 10432 return offline_state_ret; 10433 10434 /* Disabling AOC transmitters */ 10435 if (ppd->port_type == PORT_TYPE_QSFP && 10436 ppd->qsfp_info.limiting_active && 10437 qsfp_mod_present(ppd)) { 10438 int ret; 10439 10440 ret = acquire_chip_resource(dd, qsfp_resource(dd), QSFP_WAIT); 10441 if (ret == 0) { 10442 set_qsfp_tx(ppd, 0); 10443 release_chip_resource(dd, qsfp_resource(dd)); 10444 } else { 10445 /* not fatal, but should warn */ 10446 dd_dev_err(dd, 10447 "Unable to acquire lock to turn off QSFP TX\n"); 10448 } 10449 } 10450 10451 /* 10452 * Wait for the offline.Quiet transition if it hasn't happened yet. It 10453 * can take a while for the link to go down. 10454 */ 10455 if (offline_state_ret != PLS_OFFLINE_QUIET) { 10456 ret = wait_physical_linkstate(ppd, PLS_OFFLINE, 30000); 10457 if (ret < 0) 10458 return ret; 10459 } 10460 10461 /* 10462 * Now in charge of LCB - must be after the physical state is 10463 * offline.quiet and before host_link_state is changed. 10464 */ 10465 set_host_lcb_access(dd); 10466 write_csr(dd, DC_LCB_ERR_EN, ~0ull); /* watch LCB errors */ 10467 10468 /* make sure the logical state is also down */ 10469 ret = wait_logical_linkstate(ppd, IB_PORT_DOWN, 1000); 10470 if (ret) 10471 force_logical_link_state_down(ppd); 10472 10473 ppd->host_link_state = HLS_LINK_COOLDOWN; /* LCB access allowed */ 10474 update_statusp(ppd, IB_PORT_DOWN); 10475 10476 /* 10477 * The LNI has a mandatory wait time after the physical state 10478 * moves to Offline.Quiet. The wait time may be different 10479 * depending on how the link went down. The 8051 firmware 10480 * will observe the needed wait time and only move to ready 10481 * when that is completed. The largest of the quiet timeouts 10482 * is 6s, so wait that long and then at least 0.5s more for 10483 * other transitions, and another 0.5s for a buffer. 10484 */ 10485 ret = wait_fm_ready(dd, 7000); 10486 if (ret) { 10487 dd_dev_err(dd, 10488 "After going offline, timed out waiting for the 8051 to become ready to accept host requests\n"); 10489 /* state is really offline, so make it so */ 10490 ppd->host_link_state = HLS_DN_OFFLINE; 10491 return ret; 10492 } 10493 10494 /* 10495 * The state is now offline and the 8051 is ready to accept host 10496 * requests. 10497 * - change our state 10498 * - notify others if we were previously in a linkup state 10499 */ 10500 ppd->host_link_state = HLS_DN_OFFLINE; 10501 if (previous_state & HLS_UP) { 10502 /* went down while link was up */ 10503 handle_linkup_change(dd, 0); 10504 } else if (previous_state 10505 & (HLS_DN_POLL | HLS_VERIFY_CAP | HLS_GOING_UP)) { 10506 /* went down while attempting link up */ 10507 check_lni_states(ppd); 10508 10509 /* The QSFP doesn't need to be reset on LNI failure */ 10510 ppd->qsfp_info.reset_needed = 0; 10511 } 10512 10513 /* the active link width (downgrade) is 0 on link down */ 10514 ppd->link_width_active = 0; 10515 ppd->link_width_downgrade_tx_active = 0; 10516 ppd->link_width_downgrade_rx_active = 0; 10517 ppd->current_egress_rate = 0; 10518 return 0; 10519 } 10520 10521 /* return the link state name */ 10522 static const char *link_state_name(u32 state) 10523 { 10524 const char *name; 10525 int n = ilog2(state); 10526 static const char * const names[] = { 10527 [__HLS_UP_INIT_BP] = "INIT", 10528 [__HLS_UP_ARMED_BP] = "ARMED", 10529 [__HLS_UP_ACTIVE_BP] = "ACTIVE", 10530 [__HLS_DN_DOWNDEF_BP] = "DOWNDEF", 10531 [__HLS_DN_POLL_BP] = "POLL", 10532 [__HLS_DN_DISABLE_BP] = "DISABLE", 10533 [__HLS_DN_OFFLINE_BP] = "OFFLINE", 10534 [__HLS_VERIFY_CAP_BP] = "VERIFY_CAP", 10535 [__HLS_GOING_UP_BP] = "GOING_UP", 10536 [__HLS_GOING_OFFLINE_BP] = "GOING_OFFLINE", 10537 [__HLS_LINK_COOLDOWN_BP] = "LINK_COOLDOWN" 10538 }; 10539 10540 name = n < ARRAY_SIZE(names) ? names[n] : NULL; 10541 return name ? name : "unknown"; 10542 } 10543 10544 /* return the link state reason name */ 10545 static const char *link_state_reason_name(struct hfi1_pportdata *ppd, u32 state) 10546 { 10547 if (state == HLS_UP_INIT) { 10548 switch (ppd->linkinit_reason) { 10549 case OPA_LINKINIT_REASON_LINKUP: 10550 return "(LINKUP)"; 10551 case OPA_LINKINIT_REASON_FLAPPING: 10552 return "(FLAPPING)"; 10553 case OPA_LINKINIT_OUTSIDE_POLICY: 10554 return "(OUTSIDE_POLICY)"; 10555 case OPA_LINKINIT_QUARANTINED: 10556 return "(QUARANTINED)"; 10557 case OPA_LINKINIT_INSUFIC_CAPABILITY: 10558 return "(INSUFIC_CAPABILITY)"; 10559 default: 10560 break; 10561 } 10562 } 10563 return ""; 10564 } 10565 10566 /* 10567 * driver_pstate - convert the driver's notion of a port's 10568 * state (an HLS_*) into a physical state (a {IB,OPA}_PORTPHYSSTATE_*). 10569 * Return -1 (converted to a u32) to indicate error. 10570 */ 10571 u32 driver_pstate(struct hfi1_pportdata *ppd) 10572 { 10573 switch (ppd->host_link_state) { 10574 case HLS_UP_INIT: 10575 case HLS_UP_ARMED: 10576 case HLS_UP_ACTIVE: 10577 return IB_PORTPHYSSTATE_LINKUP; 10578 case HLS_DN_POLL: 10579 return IB_PORTPHYSSTATE_POLLING; 10580 case HLS_DN_DISABLE: 10581 return IB_PORTPHYSSTATE_DISABLED; 10582 case HLS_DN_OFFLINE: 10583 return OPA_PORTPHYSSTATE_OFFLINE; 10584 case HLS_VERIFY_CAP: 10585 return IB_PORTPHYSSTATE_TRAINING; 10586 case HLS_GOING_UP: 10587 return IB_PORTPHYSSTATE_TRAINING; 10588 case HLS_GOING_OFFLINE: 10589 return OPA_PORTPHYSSTATE_OFFLINE; 10590 case HLS_LINK_COOLDOWN: 10591 return OPA_PORTPHYSSTATE_OFFLINE; 10592 case HLS_DN_DOWNDEF: 10593 default: 10594 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n", 10595 ppd->host_link_state); 10596 return -1; 10597 } 10598 } 10599 10600 /* 10601 * driver_lstate - convert the driver's notion of a port's 10602 * state (an HLS_*) into a logical state (a IB_PORT_*). Return -1 10603 * (converted to a u32) to indicate error. 10604 */ 10605 u32 driver_lstate(struct hfi1_pportdata *ppd) 10606 { 10607 if (ppd->host_link_state && (ppd->host_link_state & HLS_DOWN)) 10608 return IB_PORT_DOWN; 10609 10610 switch (ppd->host_link_state & HLS_UP) { 10611 case HLS_UP_INIT: 10612 return IB_PORT_INIT; 10613 case HLS_UP_ARMED: 10614 return IB_PORT_ARMED; 10615 case HLS_UP_ACTIVE: 10616 return IB_PORT_ACTIVE; 10617 default: 10618 dd_dev_err(ppd->dd, "invalid host_link_state 0x%x\n", 10619 ppd->host_link_state); 10620 return -1; 10621 } 10622 } 10623 10624 void set_link_down_reason(struct hfi1_pportdata *ppd, u8 lcl_reason, 10625 u8 neigh_reason, u8 rem_reason) 10626 { 10627 if (ppd->local_link_down_reason.latest == 0 && 10628 ppd->neigh_link_down_reason.latest == 0) { 10629 ppd->local_link_down_reason.latest = lcl_reason; 10630 ppd->neigh_link_down_reason.latest = neigh_reason; 10631 ppd->remote_link_down_reason = rem_reason; 10632 } 10633 } 10634 10635 /** 10636 * data_vls_operational() - Verify if data VL BCT credits and MTU 10637 * are both set. 10638 * @ppd: pointer to hfi1_pportdata structure 10639 * 10640 * Return: true - Ok, false -otherwise. 10641 */ 10642 static inline bool data_vls_operational(struct hfi1_pportdata *ppd) 10643 { 10644 int i; 10645 u64 reg; 10646 10647 if (!ppd->actual_vls_operational) 10648 return false; 10649 10650 for (i = 0; i < ppd->vls_supported; i++) { 10651 reg = read_csr(ppd->dd, SEND_CM_CREDIT_VL + (8 * i)); 10652 if ((reg && !ppd->dd->vld[i].mtu) || 10653 (!reg && ppd->dd->vld[i].mtu)) 10654 return false; 10655 } 10656 10657 return true; 10658 } 10659 10660 /* 10661 * Change the physical and/or logical link state. 10662 * 10663 * Do not call this routine while inside an interrupt. It contains 10664 * calls to routines that can take multiple seconds to finish. 10665 * 10666 * Returns 0 on success, -errno on failure. 10667 */ 10668 int set_link_state(struct hfi1_pportdata *ppd, u32 state) 10669 { 10670 struct hfi1_devdata *dd = ppd->dd; 10671 struct ib_event event = {.device = NULL}; 10672 int ret1, ret = 0; 10673 int orig_new_state, poll_bounce; 10674 10675 mutex_lock(&ppd->hls_lock); 10676 10677 orig_new_state = state; 10678 if (state == HLS_DN_DOWNDEF) 10679 state = HLS_DEFAULT; 10680 10681 /* interpret poll -> poll as a link bounce */ 10682 poll_bounce = ppd->host_link_state == HLS_DN_POLL && 10683 state == HLS_DN_POLL; 10684 10685 dd_dev_info(dd, "%s: current %s, new %s %s%s\n", __func__, 10686 link_state_name(ppd->host_link_state), 10687 link_state_name(orig_new_state), 10688 poll_bounce ? "(bounce) " : "", 10689 link_state_reason_name(ppd, state)); 10690 10691 /* 10692 * If we're going to a (HLS_*) link state that implies the logical 10693 * link state is neither of (IB_PORT_ARMED, IB_PORT_ACTIVE), then 10694 * reset is_sm_config_started to 0. 10695 */ 10696 if (!(state & (HLS_UP_ARMED | HLS_UP_ACTIVE))) 10697 ppd->is_sm_config_started = 0; 10698 10699 /* 10700 * Do nothing if the states match. Let a poll to poll link bounce 10701 * go through. 10702 */ 10703 if (ppd->host_link_state == state && !poll_bounce) 10704 goto done; 10705 10706 switch (state) { 10707 case HLS_UP_INIT: 10708 if (ppd->host_link_state == HLS_DN_POLL && 10709 (quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR)) { 10710 /* 10711 * Quick link up jumps from polling to here. 10712 * 10713 * Whether in normal or loopback mode, the 10714 * simulator jumps from polling to link up. 10715 * Accept that here. 10716 */ 10717 /* OK */ 10718 } else if (ppd->host_link_state != HLS_GOING_UP) { 10719 goto unexpected; 10720 } 10721 10722 /* 10723 * Wait for Link_Up physical state. 10724 * Physical and Logical states should already be 10725 * be transitioned to LinkUp and LinkInit respectively. 10726 */ 10727 ret = wait_physical_linkstate(ppd, PLS_LINKUP, 1000); 10728 if (ret) { 10729 dd_dev_err(dd, 10730 "%s: physical state did not change to LINK-UP\n", 10731 __func__); 10732 break; 10733 } 10734 10735 ret = wait_logical_linkstate(ppd, IB_PORT_INIT, 1000); 10736 if (ret) { 10737 dd_dev_err(dd, 10738 "%s: logical state did not change to INIT\n", 10739 __func__); 10740 break; 10741 } 10742 10743 /* clear old transient LINKINIT_REASON code */ 10744 if (ppd->linkinit_reason >= OPA_LINKINIT_REASON_CLEAR) 10745 ppd->linkinit_reason = 10746 OPA_LINKINIT_REASON_LINKUP; 10747 10748 /* enable the port */ 10749 add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK); 10750 10751 handle_linkup_change(dd, 1); 10752 pio_kernel_linkup(dd); 10753 10754 /* 10755 * After link up, a new link width will have been set. 10756 * Update the xmit counters with regards to the new 10757 * link width. 10758 */ 10759 update_xmit_counters(ppd, ppd->link_width_active); 10760 10761 ppd->host_link_state = HLS_UP_INIT; 10762 update_statusp(ppd, IB_PORT_INIT); 10763 break; 10764 case HLS_UP_ARMED: 10765 if (ppd->host_link_state != HLS_UP_INIT) 10766 goto unexpected; 10767 10768 if (!data_vls_operational(ppd)) { 10769 dd_dev_err(dd, 10770 "%s: Invalid data VL credits or mtu\n", 10771 __func__); 10772 ret = -EINVAL; 10773 break; 10774 } 10775 10776 set_logical_state(dd, LSTATE_ARMED); 10777 ret = wait_logical_linkstate(ppd, IB_PORT_ARMED, 1000); 10778 if (ret) { 10779 dd_dev_err(dd, 10780 "%s: logical state did not change to ARMED\n", 10781 __func__); 10782 break; 10783 } 10784 ppd->host_link_state = HLS_UP_ARMED; 10785 update_statusp(ppd, IB_PORT_ARMED); 10786 /* 10787 * The simulator does not currently implement SMA messages, 10788 * so neighbor_normal is not set. Set it here when we first 10789 * move to Armed. 10790 */ 10791 if (dd->icode == ICODE_FUNCTIONAL_SIMULATOR) 10792 ppd->neighbor_normal = 1; 10793 break; 10794 case HLS_UP_ACTIVE: 10795 if (ppd->host_link_state != HLS_UP_ARMED) 10796 goto unexpected; 10797 10798 set_logical_state(dd, LSTATE_ACTIVE); 10799 ret = wait_logical_linkstate(ppd, IB_PORT_ACTIVE, 1000); 10800 if (ret) { 10801 dd_dev_err(dd, 10802 "%s: logical state did not change to ACTIVE\n", 10803 __func__); 10804 } else { 10805 /* tell all engines to go running */ 10806 sdma_all_running(dd); 10807 ppd->host_link_state = HLS_UP_ACTIVE; 10808 update_statusp(ppd, IB_PORT_ACTIVE); 10809 10810 /* Signal the IB layer that the port has went active */ 10811 event.device = &dd->verbs_dev.rdi.ibdev; 10812 event.element.port_num = ppd->port; 10813 event.event = IB_EVENT_PORT_ACTIVE; 10814 } 10815 break; 10816 case HLS_DN_POLL: 10817 if ((ppd->host_link_state == HLS_DN_DISABLE || 10818 ppd->host_link_state == HLS_DN_OFFLINE) && 10819 dd->dc_shutdown) 10820 dc_start(dd); 10821 /* Hand LED control to the DC */ 10822 write_csr(dd, DCC_CFG_LED_CNTRL, 0); 10823 10824 if (ppd->host_link_state != HLS_DN_OFFLINE) { 10825 u8 tmp = ppd->link_enabled; 10826 10827 ret = goto_offline(ppd, ppd->remote_link_down_reason); 10828 if (ret) { 10829 ppd->link_enabled = tmp; 10830 break; 10831 } 10832 ppd->remote_link_down_reason = 0; 10833 10834 if (ppd->driver_link_ready) 10835 ppd->link_enabled = 1; 10836 } 10837 10838 set_all_slowpath(ppd->dd); 10839 ret = set_local_link_attributes(ppd); 10840 if (ret) 10841 break; 10842 10843 ppd->port_error_action = 0; 10844 10845 if (quick_linkup) { 10846 /* quick linkup does not go into polling */ 10847 ret = do_quick_linkup(dd); 10848 } else { 10849 ret1 = set_physical_link_state(dd, PLS_POLLING); 10850 if (!ret1) 10851 ret1 = wait_phys_link_out_of_offline(ppd, 10852 3000); 10853 if (ret1 != HCMD_SUCCESS) { 10854 dd_dev_err(dd, 10855 "Failed to transition to Polling link state, return 0x%x\n", 10856 ret1); 10857 ret = -EINVAL; 10858 } 10859 } 10860 10861 /* 10862 * Change the host link state after requesting DC8051 to 10863 * change its physical state so that we can ignore any 10864 * interrupt with stale LNI(XX) error, which will not be 10865 * cleared until DC8051 transitions to Polling state. 10866 */ 10867 ppd->host_link_state = HLS_DN_POLL; 10868 ppd->offline_disabled_reason = 10869 HFI1_ODR_MASK(OPA_LINKDOWN_REASON_NONE); 10870 /* 10871 * If an error occurred above, go back to offline. The 10872 * caller may reschedule another attempt. 10873 */ 10874 if (ret) 10875 goto_offline(ppd, 0); 10876 else 10877 log_physical_state(ppd, PLS_POLLING); 10878 break; 10879 case HLS_DN_DISABLE: 10880 /* link is disabled */ 10881 ppd->link_enabled = 0; 10882 10883 /* allow any state to transition to disabled */ 10884 10885 /* must transition to offline first */ 10886 if (ppd->host_link_state != HLS_DN_OFFLINE) { 10887 ret = goto_offline(ppd, ppd->remote_link_down_reason); 10888 if (ret) 10889 break; 10890 ppd->remote_link_down_reason = 0; 10891 } 10892 10893 if (!dd->dc_shutdown) { 10894 ret1 = set_physical_link_state(dd, PLS_DISABLED); 10895 if (ret1 != HCMD_SUCCESS) { 10896 dd_dev_err(dd, 10897 "Failed to transition to Disabled link state, return 0x%x\n", 10898 ret1); 10899 ret = -EINVAL; 10900 break; 10901 } 10902 ret = wait_physical_linkstate(ppd, PLS_DISABLED, 10000); 10903 if (ret) { 10904 dd_dev_err(dd, 10905 "%s: physical state did not change to DISABLED\n", 10906 __func__); 10907 break; 10908 } 10909 dc_shutdown(dd); 10910 } 10911 ppd->host_link_state = HLS_DN_DISABLE; 10912 break; 10913 case HLS_DN_OFFLINE: 10914 if (ppd->host_link_state == HLS_DN_DISABLE) 10915 dc_start(dd); 10916 10917 /* allow any state to transition to offline */ 10918 ret = goto_offline(ppd, ppd->remote_link_down_reason); 10919 if (!ret) 10920 ppd->remote_link_down_reason = 0; 10921 break; 10922 case HLS_VERIFY_CAP: 10923 if (ppd->host_link_state != HLS_DN_POLL) 10924 goto unexpected; 10925 ppd->host_link_state = HLS_VERIFY_CAP; 10926 log_physical_state(ppd, PLS_CONFIGPHY_VERIFYCAP); 10927 break; 10928 case HLS_GOING_UP: 10929 if (ppd->host_link_state != HLS_VERIFY_CAP) 10930 goto unexpected; 10931 10932 ret1 = set_physical_link_state(dd, PLS_LINKUP); 10933 if (ret1 != HCMD_SUCCESS) { 10934 dd_dev_err(dd, 10935 "Failed to transition to link up state, return 0x%x\n", 10936 ret1); 10937 ret = -EINVAL; 10938 break; 10939 } 10940 ppd->host_link_state = HLS_GOING_UP; 10941 break; 10942 10943 case HLS_GOING_OFFLINE: /* transient within goto_offline() */ 10944 case HLS_LINK_COOLDOWN: /* transient within goto_offline() */ 10945 default: 10946 dd_dev_info(dd, "%s: state 0x%x: not supported\n", 10947 __func__, state); 10948 ret = -EINVAL; 10949 break; 10950 } 10951 10952 goto done; 10953 10954 unexpected: 10955 dd_dev_err(dd, "%s: unexpected state transition from %s to %s\n", 10956 __func__, link_state_name(ppd->host_link_state), 10957 link_state_name(state)); 10958 ret = -EINVAL; 10959 10960 done: 10961 mutex_unlock(&ppd->hls_lock); 10962 10963 if (event.device) 10964 ib_dispatch_event(&event); 10965 10966 return ret; 10967 } 10968 10969 int hfi1_set_ib_cfg(struct hfi1_pportdata *ppd, int which, u32 val) 10970 { 10971 u64 reg; 10972 int ret = 0; 10973 10974 switch (which) { 10975 case HFI1_IB_CFG_LIDLMC: 10976 set_lidlmc(ppd); 10977 break; 10978 case HFI1_IB_CFG_VL_HIGH_LIMIT: 10979 /* 10980 * The VL Arbitrator high limit is sent in units of 4k 10981 * bytes, while HFI stores it in units of 64 bytes. 10982 */ 10983 val *= 4096 / 64; 10984 reg = ((u64)val & SEND_HIGH_PRIORITY_LIMIT_LIMIT_MASK) 10985 << SEND_HIGH_PRIORITY_LIMIT_LIMIT_SHIFT; 10986 write_csr(ppd->dd, SEND_HIGH_PRIORITY_LIMIT, reg); 10987 break; 10988 case HFI1_IB_CFG_LINKDEFAULT: /* IB link default (sleep/poll) */ 10989 /* HFI only supports POLL as the default link down state */ 10990 if (val != HLS_DN_POLL) 10991 ret = -EINVAL; 10992 break; 10993 case HFI1_IB_CFG_OP_VLS: 10994 if (ppd->vls_operational != val) { 10995 ppd->vls_operational = val; 10996 if (!ppd->port) 10997 ret = -EINVAL; 10998 } 10999 break; 11000 /* 11001 * For link width, link width downgrade, and speed enable, always AND 11002 * the setting with what is actually supported. This has two benefits. 11003 * First, enabled can't have unsupported values, no matter what the 11004 * SM or FM might want. Second, the ALL_SUPPORTED wildcards that mean 11005 * "fill in with your supported value" have all the bits in the 11006 * field set, so simply ANDing with supported has the desired result. 11007 */ 11008 case HFI1_IB_CFG_LWID_ENB: /* set allowed Link-width */ 11009 ppd->link_width_enabled = val & ppd->link_width_supported; 11010 break; 11011 case HFI1_IB_CFG_LWID_DG_ENB: /* set allowed link width downgrade */ 11012 ppd->link_width_downgrade_enabled = 11013 val & ppd->link_width_downgrade_supported; 11014 break; 11015 case HFI1_IB_CFG_SPD_ENB: /* allowed Link speeds */ 11016 ppd->link_speed_enabled = val & ppd->link_speed_supported; 11017 break; 11018 case HFI1_IB_CFG_OVERRUN_THRESH: /* IB overrun threshold */ 11019 /* 11020 * HFI does not follow IB specs, save this value 11021 * so we can report it, if asked. 11022 */ 11023 ppd->overrun_threshold = val; 11024 break; 11025 case HFI1_IB_CFG_PHYERR_THRESH: /* IB PHY error threshold */ 11026 /* 11027 * HFI does not follow IB specs, save this value 11028 * so we can report it, if asked. 11029 */ 11030 ppd->phy_error_threshold = val; 11031 break; 11032 11033 case HFI1_IB_CFG_MTU: 11034 set_send_length(ppd); 11035 break; 11036 11037 case HFI1_IB_CFG_PKEYS: 11038 if (HFI1_CAP_IS_KSET(PKEY_CHECK)) 11039 set_partition_keys(ppd); 11040 break; 11041 11042 default: 11043 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL)) 11044 dd_dev_info(ppd->dd, 11045 "%s: which %s, val 0x%x: not implemented\n", 11046 __func__, ib_cfg_name(which), val); 11047 break; 11048 } 11049 return ret; 11050 } 11051 11052 /* begin functions related to vl arbitration table caching */ 11053 static void init_vl_arb_caches(struct hfi1_pportdata *ppd) 11054 { 11055 int i; 11056 11057 BUILD_BUG_ON(VL_ARB_TABLE_SIZE != 11058 VL_ARB_LOW_PRIO_TABLE_SIZE); 11059 BUILD_BUG_ON(VL_ARB_TABLE_SIZE != 11060 VL_ARB_HIGH_PRIO_TABLE_SIZE); 11061 11062 /* 11063 * Note that we always return values directly from the 11064 * 'vl_arb_cache' (and do no CSR reads) in response to a 11065 * 'Get(VLArbTable)'. This is obviously correct after a 11066 * 'Set(VLArbTable)', since the cache will then be up to 11067 * date. But it's also correct prior to any 'Set(VLArbTable)' 11068 * since then both the cache, and the relevant h/w registers 11069 * will be zeroed. 11070 */ 11071 11072 for (i = 0; i < MAX_PRIO_TABLE; i++) 11073 spin_lock_init(&ppd->vl_arb_cache[i].lock); 11074 } 11075 11076 /* 11077 * vl_arb_lock_cache 11078 * 11079 * All other vl_arb_* functions should be called only after locking 11080 * the cache. 11081 */ 11082 static inline struct vl_arb_cache * 11083 vl_arb_lock_cache(struct hfi1_pportdata *ppd, int idx) 11084 { 11085 if (idx != LO_PRIO_TABLE && idx != HI_PRIO_TABLE) 11086 return NULL; 11087 spin_lock(&ppd->vl_arb_cache[idx].lock); 11088 return &ppd->vl_arb_cache[idx]; 11089 } 11090 11091 static inline void vl_arb_unlock_cache(struct hfi1_pportdata *ppd, int idx) 11092 { 11093 spin_unlock(&ppd->vl_arb_cache[idx].lock); 11094 } 11095 11096 static void vl_arb_get_cache(struct vl_arb_cache *cache, 11097 struct ib_vl_weight_elem *vl) 11098 { 11099 memcpy(vl, cache->table, VL_ARB_TABLE_SIZE * sizeof(*vl)); 11100 } 11101 11102 static void vl_arb_set_cache(struct vl_arb_cache *cache, 11103 struct ib_vl_weight_elem *vl) 11104 { 11105 memcpy(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl)); 11106 } 11107 11108 static int vl_arb_match_cache(struct vl_arb_cache *cache, 11109 struct ib_vl_weight_elem *vl) 11110 { 11111 return !memcmp(cache->table, vl, VL_ARB_TABLE_SIZE * sizeof(*vl)); 11112 } 11113 11114 /* end functions related to vl arbitration table caching */ 11115 11116 static int set_vl_weights(struct hfi1_pportdata *ppd, u32 target, 11117 u32 size, struct ib_vl_weight_elem *vl) 11118 { 11119 struct hfi1_devdata *dd = ppd->dd; 11120 u64 reg; 11121 unsigned int i, is_up = 0; 11122 int drain, ret = 0; 11123 11124 mutex_lock(&ppd->hls_lock); 11125 11126 if (ppd->host_link_state & HLS_UP) 11127 is_up = 1; 11128 11129 drain = !is_ax(dd) && is_up; 11130 11131 if (drain) 11132 /* 11133 * Before adjusting VL arbitration weights, empty per-VL 11134 * FIFOs, otherwise a packet whose VL weight is being 11135 * set to 0 could get stuck in a FIFO with no chance to 11136 * egress. 11137 */ 11138 ret = stop_drain_data_vls(dd); 11139 11140 if (ret) { 11141 dd_dev_err( 11142 dd, 11143 "%s: cannot stop/drain VLs - refusing to change VL arbitration weights\n", 11144 __func__); 11145 goto err; 11146 } 11147 11148 for (i = 0; i < size; i++, vl++) { 11149 /* 11150 * NOTE: The low priority shift and mask are used here, but 11151 * they are the same for both the low and high registers. 11152 */ 11153 reg = (((u64)vl->vl & SEND_LOW_PRIORITY_LIST_VL_MASK) 11154 << SEND_LOW_PRIORITY_LIST_VL_SHIFT) 11155 | (((u64)vl->weight 11156 & SEND_LOW_PRIORITY_LIST_WEIGHT_MASK) 11157 << SEND_LOW_PRIORITY_LIST_WEIGHT_SHIFT); 11158 write_csr(dd, target + (i * 8), reg); 11159 } 11160 pio_send_control(dd, PSC_GLOBAL_VLARB_ENABLE); 11161 11162 if (drain) 11163 open_fill_data_vls(dd); /* reopen all VLs */ 11164 11165 err: 11166 mutex_unlock(&ppd->hls_lock); 11167 11168 return ret; 11169 } 11170 11171 /* 11172 * Read one credit merge VL register. 11173 */ 11174 static void read_one_cm_vl(struct hfi1_devdata *dd, u32 csr, 11175 struct vl_limit *vll) 11176 { 11177 u64 reg = read_csr(dd, csr); 11178 11179 vll->dedicated = cpu_to_be16( 11180 (reg >> SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT) 11181 & SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_MASK); 11182 vll->shared = cpu_to_be16( 11183 (reg >> SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT) 11184 & SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_MASK); 11185 } 11186 11187 /* 11188 * Read the current credit merge limits. 11189 */ 11190 static int get_buffer_control(struct hfi1_devdata *dd, 11191 struct buffer_control *bc, u16 *overall_limit) 11192 { 11193 u64 reg; 11194 int i; 11195 11196 /* not all entries are filled in */ 11197 memset(bc, 0, sizeof(*bc)); 11198 11199 /* OPA and HFI have a 1-1 mapping */ 11200 for (i = 0; i < TXE_NUM_DATA_VL; i++) 11201 read_one_cm_vl(dd, SEND_CM_CREDIT_VL + (8 * i), &bc->vl[i]); 11202 11203 /* NOTE: assumes that VL* and VL15 CSRs are bit-wise identical */ 11204 read_one_cm_vl(dd, SEND_CM_CREDIT_VL15, &bc->vl[15]); 11205 11206 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT); 11207 bc->overall_shared_limit = cpu_to_be16( 11208 (reg >> SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT) 11209 & SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_MASK); 11210 if (overall_limit) 11211 *overall_limit = (reg 11212 >> SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT) 11213 & SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_MASK; 11214 return sizeof(struct buffer_control); 11215 } 11216 11217 static int get_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp) 11218 { 11219 u64 reg; 11220 int i; 11221 11222 /* each register contains 16 SC->VLnt mappings, 4 bits each */ 11223 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_15_0); 11224 for (i = 0; i < sizeof(u64); i++) { 11225 u8 byte = *(((u8 *)®) + i); 11226 11227 dp->vlnt[2 * i] = byte & 0xf; 11228 dp->vlnt[(2 * i) + 1] = (byte & 0xf0) >> 4; 11229 } 11230 11231 reg = read_csr(dd, DCC_CFG_SC_VL_TABLE_31_16); 11232 for (i = 0; i < sizeof(u64); i++) { 11233 u8 byte = *(((u8 *)®) + i); 11234 11235 dp->vlnt[16 + (2 * i)] = byte & 0xf; 11236 dp->vlnt[16 + (2 * i) + 1] = (byte & 0xf0) >> 4; 11237 } 11238 return sizeof(struct sc2vlnt); 11239 } 11240 11241 static void get_vlarb_preempt(struct hfi1_devdata *dd, u32 nelems, 11242 struct ib_vl_weight_elem *vl) 11243 { 11244 unsigned int i; 11245 11246 for (i = 0; i < nelems; i++, vl++) { 11247 vl->vl = 0xf; 11248 vl->weight = 0; 11249 } 11250 } 11251 11252 static void set_sc2vlnt(struct hfi1_devdata *dd, struct sc2vlnt *dp) 11253 { 11254 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0, 11255 DC_SC_VL_VAL(15_0, 11256 0, dp->vlnt[0] & 0xf, 11257 1, dp->vlnt[1] & 0xf, 11258 2, dp->vlnt[2] & 0xf, 11259 3, dp->vlnt[3] & 0xf, 11260 4, dp->vlnt[4] & 0xf, 11261 5, dp->vlnt[5] & 0xf, 11262 6, dp->vlnt[6] & 0xf, 11263 7, dp->vlnt[7] & 0xf, 11264 8, dp->vlnt[8] & 0xf, 11265 9, dp->vlnt[9] & 0xf, 11266 10, dp->vlnt[10] & 0xf, 11267 11, dp->vlnt[11] & 0xf, 11268 12, dp->vlnt[12] & 0xf, 11269 13, dp->vlnt[13] & 0xf, 11270 14, dp->vlnt[14] & 0xf, 11271 15, dp->vlnt[15] & 0xf)); 11272 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16, 11273 DC_SC_VL_VAL(31_16, 11274 16, dp->vlnt[16] & 0xf, 11275 17, dp->vlnt[17] & 0xf, 11276 18, dp->vlnt[18] & 0xf, 11277 19, dp->vlnt[19] & 0xf, 11278 20, dp->vlnt[20] & 0xf, 11279 21, dp->vlnt[21] & 0xf, 11280 22, dp->vlnt[22] & 0xf, 11281 23, dp->vlnt[23] & 0xf, 11282 24, dp->vlnt[24] & 0xf, 11283 25, dp->vlnt[25] & 0xf, 11284 26, dp->vlnt[26] & 0xf, 11285 27, dp->vlnt[27] & 0xf, 11286 28, dp->vlnt[28] & 0xf, 11287 29, dp->vlnt[29] & 0xf, 11288 30, dp->vlnt[30] & 0xf, 11289 31, dp->vlnt[31] & 0xf)); 11290 } 11291 11292 static void nonzero_msg(struct hfi1_devdata *dd, int idx, const char *what, 11293 u16 limit) 11294 { 11295 if (limit != 0) 11296 dd_dev_info(dd, "Invalid %s limit %d on VL %d, ignoring\n", 11297 what, (int)limit, idx); 11298 } 11299 11300 /* change only the shared limit portion of SendCmGLobalCredit */ 11301 static void set_global_shared(struct hfi1_devdata *dd, u16 limit) 11302 { 11303 u64 reg; 11304 11305 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT); 11306 reg &= ~SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SMASK; 11307 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_SHARED_LIMIT_SHIFT; 11308 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg); 11309 } 11310 11311 /* change only the total credit limit portion of SendCmGLobalCredit */ 11312 static void set_global_limit(struct hfi1_devdata *dd, u16 limit) 11313 { 11314 u64 reg; 11315 11316 reg = read_csr(dd, SEND_CM_GLOBAL_CREDIT); 11317 reg &= ~SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SMASK; 11318 reg |= (u64)limit << SEND_CM_GLOBAL_CREDIT_TOTAL_CREDIT_LIMIT_SHIFT; 11319 write_csr(dd, SEND_CM_GLOBAL_CREDIT, reg); 11320 } 11321 11322 /* set the given per-VL shared limit */ 11323 static void set_vl_shared(struct hfi1_devdata *dd, int vl, u16 limit) 11324 { 11325 u64 reg; 11326 u32 addr; 11327 11328 if (vl < TXE_NUM_DATA_VL) 11329 addr = SEND_CM_CREDIT_VL + (8 * vl); 11330 else 11331 addr = SEND_CM_CREDIT_VL15; 11332 11333 reg = read_csr(dd, addr); 11334 reg &= ~SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SMASK; 11335 reg |= (u64)limit << SEND_CM_CREDIT_VL_SHARED_LIMIT_VL_SHIFT; 11336 write_csr(dd, addr, reg); 11337 } 11338 11339 /* set the given per-VL dedicated limit */ 11340 static void set_vl_dedicated(struct hfi1_devdata *dd, int vl, u16 limit) 11341 { 11342 u64 reg; 11343 u32 addr; 11344 11345 if (vl < TXE_NUM_DATA_VL) 11346 addr = SEND_CM_CREDIT_VL + (8 * vl); 11347 else 11348 addr = SEND_CM_CREDIT_VL15; 11349 11350 reg = read_csr(dd, addr); 11351 reg &= ~SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SMASK; 11352 reg |= (u64)limit << SEND_CM_CREDIT_VL_DEDICATED_LIMIT_VL_SHIFT; 11353 write_csr(dd, addr, reg); 11354 } 11355 11356 /* spin until the given per-VL status mask bits clear */ 11357 static void wait_for_vl_status_clear(struct hfi1_devdata *dd, u64 mask, 11358 const char *which) 11359 { 11360 unsigned long timeout; 11361 u64 reg; 11362 11363 timeout = jiffies + msecs_to_jiffies(VL_STATUS_CLEAR_TIMEOUT); 11364 while (1) { 11365 reg = read_csr(dd, SEND_CM_CREDIT_USED_STATUS) & mask; 11366 11367 if (reg == 0) 11368 return; /* success */ 11369 if (time_after(jiffies, timeout)) 11370 break; /* timed out */ 11371 udelay(1); 11372 } 11373 11374 dd_dev_err(dd, 11375 "%s credit change status not clearing after %dms, mask 0x%llx, not clear 0x%llx\n", 11376 which, VL_STATUS_CLEAR_TIMEOUT, mask, reg); 11377 /* 11378 * If this occurs, it is likely there was a credit loss on the link. 11379 * The only recovery from that is a link bounce. 11380 */ 11381 dd_dev_err(dd, 11382 "Continuing anyway. A credit loss may occur. Suggest a link bounce\n"); 11383 } 11384 11385 /* 11386 * The number of credits on the VLs may be changed while everything 11387 * is "live", but the following algorithm must be followed due to 11388 * how the hardware is actually implemented. In particular, 11389 * Return_Credit_Status[] is the only correct status check. 11390 * 11391 * if (reducing Global_Shared_Credit_Limit or any shared limit changing) 11392 * set Global_Shared_Credit_Limit = 0 11393 * use_all_vl = 1 11394 * mask0 = all VLs that are changing either dedicated or shared limits 11395 * set Shared_Limit[mask0] = 0 11396 * spin until Return_Credit_Status[use_all_vl ? all VL : mask0] == 0 11397 * if (changing any dedicated limit) 11398 * mask1 = all VLs that are lowering dedicated limits 11399 * lower Dedicated_Limit[mask1] 11400 * spin until Return_Credit_Status[mask1] == 0 11401 * raise Dedicated_Limits 11402 * raise Shared_Limits 11403 * raise Global_Shared_Credit_Limit 11404 * 11405 * lower = if the new limit is lower, set the limit to the new value 11406 * raise = if the new limit is higher than the current value (may be changed 11407 * earlier in the algorithm), set the new limit to the new value 11408 */ 11409 int set_buffer_control(struct hfi1_pportdata *ppd, 11410 struct buffer_control *new_bc) 11411 { 11412 struct hfi1_devdata *dd = ppd->dd; 11413 u64 changing_mask, ld_mask, stat_mask; 11414 int change_count; 11415 int i, use_all_mask; 11416 int this_shared_changing; 11417 int vl_count = 0, ret; 11418 /* 11419 * A0: add the variable any_shared_limit_changing below and in the 11420 * algorithm above. If removing A0 support, it can be removed. 11421 */ 11422 int any_shared_limit_changing; 11423 struct buffer_control cur_bc; 11424 u8 changing[OPA_MAX_VLS]; 11425 u8 lowering_dedicated[OPA_MAX_VLS]; 11426 u16 cur_total; 11427 u32 new_total = 0; 11428 const u64 all_mask = 11429 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK 11430 | SEND_CM_CREDIT_USED_STATUS_VL1_RETURN_CREDIT_STATUS_SMASK 11431 | SEND_CM_CREDIT_USED_STATUS_VL2_RETURN_CREDIT_STATUS_SMASK 11432 | SEND_CM_CREDIT_USED_STATUS_VL3_RETURN_CREDIT_STATUS_SMASK 11433 | SEND_CM_CREDIT_USED_STATUS_VL4_RETURN_CREDIT_STATUS_SMASK 11434 | SEND_CM_CREDIT_USED_STATUS_VL5_RETURN_CREDIT_STATUS_SMASK 11435 | SEND_CM_CREDIT_USED_STATUS_VL6_RETURN_CREDIT_STATUS_SMASK 11436 | SEND_CM_CREDIT_USED_STATUS_VL7_RETURN_CREDIT_STATUS_SMASK 11437 | SEND_CM_CREDIT_USED_STATUS_VL15_RETURN_CREDIT_STATUS_SMASK; 11438 11439 #define valid_vl(idx) ((idx) < TXE_NUM_DATA_VL || (idx) == 15) 11440 #define NUM_USABLE_VLS 16 /* look at VL15 and less */ 11441 11442 /* find the new total credits, do sanity check on unused VLs */ 11443 for (i = 0; i < OPA_MAX_VLS; i++) { 11444 if (valid_vl(i)) { 11445 new_total += be16_to_cpu(new_bc->vl[i].dedicated); 11446 continue; 11447 } 11448 nonzero_msg(dd, i, "dedicated", 11449 be16_to_cpu(new_bc->vl[i].dedicated)); 11450 nonzero_msg(dd, i, "shared", 11451 be16_to_cpu(new_bc->vl[i].shared)); 11452 new_bc->vl[i].dedicated = 0; 11453 new_bc->vl[i].shared = 0; 11454 } 11455 new_total += be16_to_cpu(new_bc->overall_shared_limit); 11456 11457 /* fetch the current values */ 11458 get_buffer_control(dd, &cur_bc, &cur_total); 11459 11460 /* 11461 * Create the masks we will use. 11462 */ 11463 memset(changing, 0, sizeof(changing)); 11464 memset(lowering_dedicated, 0, sizeof(lowering_dedicated)); 11465 /* 11466 * NOTE: Assumes that the individual VL bits are adjacent and in 11467 * increasing order 11468 */ 11469 stat_mask = 11470 SEND_CM_CREDIT_USED_STATUS_VL0_RETURN_CREDIT_STATUS_SMASK; 11471 changing_mask = 0; 11472 ld_mask = 0; 11473 change_count = 0; 11474 any_shared_limit_changing = 0; 11475 for (i = 0; i < NUM_USABLE_VLS; i++, stat_mask <<= 1) { 11476 if (!valid_vl(i)) 11477 continue; 11478 this_shared_changing = new_bc->vl[i].shared 11479 != cur_bc.vl[i].shared; 11480 if (this_shared_changing) 11481 any_shared_limit_changing = 1; 11482 if (new_bc->vl[i].dedicated != cur_bc.vl[i].dedicated || 11483 this_shared_changing) { 11484 changing[i] = 1; 11485 changing_mask |= stat_mask; 11486 change_count++; 11487 } 11488 if (be16_to_cpu(new_bc->vl[i].dedicated) < 11489 be16_to_cpu(cur_bc.vl[i].dedicated)) { 11490 lowering_dedicated[i] = 1; 11491 ld_mask |= stat_mask; 11492 } 11493 } 11494 11495 /* bracket the credit change with a total adjustment */ 11496 if (new_total > cur_total) 11497 set_global_limit(dd, new_total); 11498 11499 /* 11500 * Start the credit change algorithm. 11501 */ 11502 use_all_mask = 0; 11503 if ((be16_to_cpu(new_bc->overall_shared_limit) < 11504 be16_to_cpu(cur_bc.overall_shared_limit)) || 11505 (is_ax(dd) && any_shared_limit_changing)) { 11506 set_global_shared(dd, 0); 11507 cur_bc.overall_shared_limit = 0; 11508 use_all_mask = 1; 11509 } 11510 11511 for (i = 0; i < NUM_USABLE_VLS; i++) { 11512 if (!valid_vl(i)) 11513 continue; 11514 11515 if (changing[i]) { 11516 set_vl_shared(dd, i, 0); 11517 cur_bc.vl[i].shared = 0; 11518 } 11519 } 11520 11521 wait_for_vl_status_clear(dd, use_all_mask ? all_mask : changing_mask, 11522 "shared"); 11523 11524 if (change_count > 0) { 11525 for (i = 0; i < NUM_USABLE_VLS; i++) { 11526 if (!valid_vl(i)) 11527 continue; 11528 11529 if (lowering_dedicated[i]) { 11530 set_vl_dedicated(dd, i, 11531 be16_to_cpu(new_bc-> 11532 vl[i].dedicated)); 11533 cur_bc.vl[i].dedicated = 11534 new_bc->vl[i].dedicated; 11535 } 11536 } 11537 11538 wait_for_vl_status_clear(dd, ld_mask, "dedicated"); 11539 11540 /* now raise all dedicated that are going up */ 11541 for (i = 0; i < NUM_USABLE_VLS; i++) { 11542 if (!valid_vl(i)) 11543 continue; 11544 11545 if (be16_to_cpu(new_bc->vl[i].dedicated) > 11546 be16_to_cpu(cur_bc.vl[i].dedicated)) 11547 set_vl_dedicated(dd, i, 11548 be16_to_cpu(new_bc-> 11549 vl[i].dedicated)); 11550 } 11551 } 11552 11553 /* next raise all shared that are going up */ 11554 for (i = 0; i < NUM_USABLE_VLS; i++) { 11555 if (!valid_vl(i)) 11556 continue; 11557 11558 if (be16_to_cpu(new_bc->vl[i].shared) > 11559 be16_to_cpu(cur_bc.vl[i].shared)) 11560 set_vl_shared(dd, i, be16_to_cpu(new_bc->vl[i].shared)); 11561 } 11562 11563 /* finally raise the global shared */ 11564 if (be16_to_cpu(new_bc->overall_shared_limit) > 11565 be16_to_cpu(cur_bc.overall_shared_limit)) 11566 set_global_shared(dd, 11567 be16_to_cpu(new_bc->overall_shared_limit)); 11568 11569 /* bracket the credit change with a total adjustment */ 11570 if (new_total < cur_total) 11571 set_global_limit(dd, new_total); 11572 11573 /* 11574 * Determine the actual number of operational VLS using the number of 11575 * dedicated and shared credits for each VL. 11576 */ 11577 if (change_count > 0) { 11578 for (i = 0; i < TXE_NUM_DATA_VL; i++) 11579 if (be16_to_cpu(new_bc->vl[i].dedicated) > 0 || 11580 be16_to_cpu(new_bc->vl[i].shared) > 0) 11581 vl_count++; 11582 ppd->actual_vls_operational = vl_count; 11583 ret = sdma_map_init(dd, ppd->port - 1, vl_count ? 11584 ppd->actual_vls_operational : 11585 ppd->vls_operational, 11586 NULL); 11587 if (ret == 0) 11588 ret = pio_map_init(dd, ppd->port - 1, vl_count ? 11589 ppd->actual_vls_operational : 11590 ppd->vls_operational, NULL); 11591 if (ret) 11592 return ret; 11593 } 11594 return 0; 11595 } 11596 11597 /* 11598 * Read the given fabric manager table. Return the size of the 11599 * table (in bytes) on success, and a negative error code on 11600 * failure. 11601 */ 11602 int fm_get_table(struct hfi1_pportdata *ppd, int which, void *t) 11603 11604 { 11605 int size; 11606 struct vl_arb_cache *vlc; 11607 11608 switch (which) { 11609 case FM_TBL_VL_HIGH_ARB: 11610 size = 256; 11611 /* 11612 * OPA specifies 128 elements (of 2 bytes each), though 11613 * HFI supports only 16 elements in h/w. 11614 */ 11615 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE); 11616 vl_arb_get_cache(vlc, t); 11617 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE); 11618 break; 11619 case FM_TBL_VL_LOW_ARB: 11620 size = 256; 11621 /* 11622 * OPA specifies 128 elements (of 2 bytes each), though 11623 * HFI supports only 16 elements in h/w. 11624 */ 11625 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE); 11626 vl_arb_get_cache(vlc, t); 11627 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE); 11628 break; 11629 case FM_TBL_BUFFER_CONTROL: 11630 size = get_buffer_control(ppd->dd, t, NULL); 11631 break; 11632 case FM_TBL_SC2VLNT: 11633 size = get_sc2vlnt(ppd->dd, t); 11634 break; 11635 case FM_TBL_VL_PREEMPT_ELEMS: 11636 size = 256; 11637 /* OPA specifies 128 elements, of 2 bytes each */ 11638 get_vlarb_preempt(ppd->dd, OPA_MAX_VLS, t); 11639 break; 11640 case FM_TBL_VL_PREEMPT_MATRIX: 11641 size = 256; 11642 /* 11643 * OPA specifies that this is the same size as the VL 11644 * arbitration tables (i.e., 256 bytes). 11645 */ 11646 break; 11647 default: 11648 return -EINVAL; 11649 } 11650 return size; 11651 } 11652 11653 /* 11654 * Write the given fabric manager table. 11655 */ 11656 int fm_set_table(struct hfi1_pportdata *ppd, int which, void *t) 11657 { 11658 int ret = 0; 11659 struct vl_arb_cache *vlc; 11660 11661 switch (which) { 11662 case FM_TBL_VL_HIGH_ARB: 11663 vlc = vl_arb_lock_cache(ppd, HI_PRIO_TABLE); 11664 if (vl_arb_match_cache(vlc, t)) { 11665 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE); 11666 break; 11667 } 11668 vl_arb_set_cache(vlc, t); 11669 vl_arb_unlock_cache(ppd, HI_PRIO_TABLE); 11670 ret = set_vl_weights(ppd, SEND_HIGH_PRIORITY_LIST, 11671 VL_ARB_HIGH_PRIO_TABLE_SIZE, t); 11672 break; 11673 case FM_TBL_VL_LOW_ARB: 11674 vlc = vl_arb_lock_cache(ppd, LO_PRIO_TABLE); 11675 if (vl_arb_match_cache(vlc, t)) { 11676 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE); 11677 break; 11678 } 11679 vl_arb_set_cache(vlc, t); 11680 vl_arb_unlock_cache(ppd, LO_PRIO_TABLE); 11681 ret = set_vl_weights(ppd, SEND_LOW_PRIORITY_LIST, 11682 VL_ARB_LOW_PRIO_TABLE_SIZE, t); 11683 break; 11684 case FM_TBL_BUFFER_CONTROL: 11685 ret = set_buffer_control(ppd, t); 11686 break; 11687 case FM_TBL_SC2VLNT: 11688 set_sc2vlnt(ppd->dd, t); 11689 break; 11690 default: 11691 ret = -EINVAL; 11692 } 11693 return ret; 11694 } 11695 11696 /* 11697 * Disable all data VLs. 11698 * 11699 * Return 0 if disabled, non-zero if the VLs cannot be disabled. 11700 */ 11701 static int disable_data_vls(struct hfi1_devdata *dd) 11702 { 11703 if (is_ax(dd)) 11704 return 1; 11705 11706 pio_send_control(dd, PSC_DATA_VL_DISABLE); 11707 11708 return 0; 11709 } 11710 11711 /* 11712 * open_fill_data_vls() - the counterpart to stop_drain_data_vls(). 11713 * Just re-enables all data VLs (the "fill" part happens 11714 * automatically - the name was chosen for symmetry with 11715 * stop_drain_data_vls()). 11716 * 11717 * Return 0 if successful, non-zero if the VLs cannot be enabled. 11718 */ 11719 int open_fill_data_vls(struct hfi1_devdata *dd) 11720 { 11721 if (is_ax(dd)) 11722 return 1; 11723 11724 pio_send_control(dd, PSC_DATA_VL_ENABLE); 11725 11726 return 0; 11727 } 11728 11729 /* 11730 * drain_data_vls() - assumes that disable_data_vls() has been called, 11731 * wait for occupancy (of per-VL FIFOs) for all contexts, and SDMA 11732 * engines to drop to 0. 11733 */ 11734 static void drain_data_vls(struct hfi1_devdata *dd) 11735 { 11736 sc_wait(dd); 11737 sdma_wait(dd); 11738 pause_for_credit_return(dd); 11739 } 11740 11741 /* 11742 * stop_drain_data_vls() - disable, then drain all per-VL fifos. 11743 * 11744 * Use open_fill_data_vls() to resume using data VLs. This pair is 11745 * meant to be used like this: 11746 * 11747 * stop_drain_data_vls(dd); 11748 * // do things with per-VL resources 11749 * open_fill_data_vls(dd); 11750 */ 11751 int stop_drain_data_vls(struct hfi1_devdata *dd) 11752 { 11753 int ret; 11754 11755 ret = disable_data_vls(dd); 11756 if (ret == 0) 11757 drain_data_vls(dd); 11758 11759 return ret; 11760 } 11761 11762 /* 11763 * Convert a nanosecond time to a cclock count. No matter how slow 11764 * the cclock, a non-zero ns will always have a non-zero result. 11765 */ 11766 u32 ns_to_cclock(struct hfi1_devdata *dd, u32 ns) 11767 { 11768 u32 cclocks; 11769 11770 if (dd->icode == ICODE_FPGA_EMULATION) 11771 cclocks = (ns * 1000) / FPGA_CCLOCK_PS; 11772 else /* simulation pretends to be ASIC */ 11773 cclocks = (ns * 1000) / ASIC_CCLOCK_PS; 11774 if (ns && !cclocks) /* if ns nonzero, must be at least 1 */ 11775 cclocks = 1; 11776 return cclocks; 11777 } 11778 11779 /* 11780 * Convert a cclock count to nanoseconds. Not matter how slow 11781 * the cclock, a non-zero cclocks will always have a non-zero result. 11782 */ 11783 u32 cclock_to_ns(struct hfi1_devdata *dd, u32 cclocks) 11784 { 11785 u32 ns; 11786 11787 if (dd->icode == ICODE_FPGA_EMULATION) 11788 ns = (cclocks * FPGA_CCLOCK_PS) / 1000; 11789 else /* simulation pretends to be ASIC */ 11790 ns = (cclocks * ASIC_CCLOCK_PS) / 1000; 11791 if (cclocks && !ns) 11792 ns = 1; 11793 return ns; 11794 } 11795 11796 /* 11797 * Dynamically adjust the receive interrupt timeout for a context based on 11798 * incoming packet rate. 11799 * 11800 * NOTE: Dynamic adjustment does not allow rcv_intr_count to be zero. 11801 */ 11802 static void adjust_rcv_timeout(struct hfi1_ctxtdata *rcd, u32 npkts) 11803 { 11804 struct hfi1_devdata *dd = rcd->dd; 11805 u32 timeout = rcd->rcvavail_timeout; 11806 11807 /* 11808 * This algorithm doubles or halves the timeout depending on whether 11809 * the number of packets received in this interrupt were less than or 11810 * greater equal the interrupt count. 11811 * 11812 * The calculations below do not allow a steady state to be achieved. 11813 * Only at the endpoints it is possible to have an unchanging 11814 * timeout. 11815 */ 11816 if (npkts < rcv_intr_count) { 11817 /* 11818 * Not enough packets arrived before the timeout, adjust 11819 * timeout downward. 11820 */ 11821 if (timeout < 2) /* already at minimum? */ 11822 return; 11823 timeout >>= 1; 11824 } else { 11825 /* 11826 * More than enough packets arrived before the timeout, adjust 11827 * timeout upward. 11828 */ 11829 if (timeout >= dd->rcv_intr_timeout_csr) /* already at max? */ 11830 return; 11831 timeout = min(timeout << 1, dd->rcv_intr_timeout_csr); 11832 } 11833 11834 rcd->rcvavail_timeout = timeout; 11835 /* 11836 * timeout cannot be larger than rcv_intr_timeout_csr which has already 11837 * been verified to be in range 11838 */ 11839 write_kctxt_csr(dd, rcd->ctxt, RCV_AVAIL_TIME_OUT, 11840 (u64)timeout << 11841 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT); 11842 } 11843 11844 void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd, 11845 u32 intr_adjust, u32 npkts) 11846 { 11847 struct hfi1_devdata *dd = rcd->dd; 11848 u64 reg; 11849 u32 ctxt = rcd->ctxt; 11850 11851 /* 11852 * Need to write timeout register before updating RcvHdrHead to ensure 11853 * that a new value is used when the HW decides to restart counting. 11854 */ 11855 if (intr_adjust) 11856 adjust_rcv_timeout(rcd, npkts); 11857 if (updegr) { 11858 reg = (egrhd & RCV_EGR_INDEX_HEAD_HEAD_MASK) 11859 << RCV_EGR_INDEX_HEAD_HEAD_SHIFT; 11860 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, reg); 11861 } 11862 reg = ((u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT) | 11863 (((u64)hd & RCV_HDR_HEAD_HEAD_MASK) 11864 << RCV_HDR_HEAD_HEAD_SHIFT); 11865 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg); 11866 } 11867 11868 u32 hdrqempty(struct hfi1_ctxtdata *rcd) 11869 { 11870 u32 head, tail; 11871 11872 head = (read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD) 11873 & RCV_HDR_HEAD_HEAD_SMASK) >> RCV_HDR_HEAD_HEAD_SHIFT; 11874 11875 if (hfi1_rcvhdrtail_kvaddr(rcd)) 11876 tail = get_rcvhdrtail(rcd); 11877 else 11878 tail = read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL); 11879 11880 return head == tail; 11881 } 11882 11883 /* 11884 * Context Control and Receive Array encoding for buffer size: 11885 * 0x0 invalid 11886 * 0x1 4 KB 11887 * 0x2 8 KB 11888 * 0x3 16 KB 11889 * 0x4 32 KB 11890 * 0x5 64 KB 11891 * 0x6 128 KB 11892 * 0x7 256 KB 11893 * 0x8 512 KB (Receive Array only) 11894 * 0x9 1 MB (Receive Array only) 11895 * 0xa 2 MB (Receive Array only) 11896 * 11897 * 0xB-0xF - reserved (Receive Array only) 11898 * 11899 * 11900 * This routine assumes that the value has already been sanity checked. 11901 */ 11902 static u32 encoded_size(u32 size) 11903 { 11904 switch (size) { 11905 case 4 * 1024: return 0x1; 11906 case 8 * 1024: return 0x2; 11907 case 16 * 1024: return 0x3; 11908 case 32 * 1024: return 0x4; 11909 case 64 * 1024: return 0x5; 11910 case 128 * 1024: return 0x6; 11911 case 256 * 1024: return 0x7; 11912 case 512 * 1024: return 0x8; 11913 case 1 * 1024 * 1024: return 0x9; 11914 case 2 * 1024 * 1024: return 0xa; 11915 } 11916 return 0x1; /* if invalid, go with the minimum size */ 11917 } 11918 11919 /** 11920 * encode_rcv_header_entry_size - return chip specific encoding for size 11921 * @size: size in dwords 11922 * 11923 * Convert a receive header entry size that to the encoding used in the CSR. 11924 * 11925 * Return a zero if the given size is invalid, otherwise the encoding. 11926 */ 11927 u8 encode_rcv_header_entry_size(u8 size) 11928 { 11929 /* there are only 3 valid receive header entry sizes */ 11930 if (size == 2) 11931 return 1; 11932 if (size == 16) 11933 return 2; 11934 if (size == 32) 11935 return 4; 11936 return 0; /* invalid */ 11937 } 11938 11939 /** 11940 * hfi1_validate_rcvhdrcnt - validate hdrcnt 11941 * @dd: the device data 11942 * @thecnt: the header count 11943 */ 11944 int hfi1_validate_rcvhdrcnt(struct hfi1_devdata *dd, uint thecnt) 11945 { 11946 if (thecnt <= HFI1_MIN_HDRQ_EGRBUF_CNT) { 11947 dd_dev_err(dd, "Receive header queue count too small\n"); 11948 return -EINVAL; 11949 } 11950 11951 if (thecnt > HFI1_MAX_HDRQ_EGRBUF_CNT) { 11952 dd_dev_err(dd, 11953 "Receive header queue count cannot be greater than %u\n", 11954 HFI1_MAX_HDRQ_EGRBUF_CNT); 11955 return -EINVAL; 11956 } 11957 11958 if (thecnt % HDRQ_INCREMENT) { 11959 dd_dev_err(dd, "Receive header queue count %d must be divisible by %lu\n", 11960 thecnt, HDRQ_INCREMENT); 11961 return -EINVAL; 11962 } 11963 11964 return 0; 11965 } 11966 11967 /** 11968 * set_hdrq_regs - set header queue registers for context 11969 * @dd: the device data 11970 * @ctxt: the context 11971 * @entsize: the dword entry size 11972 * @hdrcnt: the number of header entries 11973 */ 11974 void set_hdrq_regs(struct hfi1_devdata *dd, u8 ctxt, u8 entsize, u16 hdrcnt) 11975 { 11976 u64 reg; 11977 11978 reg = (((u64)hdrcnt >> HDRQ_SIZE_SHIFT) & RCV_HDR_CNT_CNT_MASK) << 11979 RCV_HDR_CNT_CNT_SHIFT; 11980 write_kctxt_csr(dd, ctxt, RCV_HDR_CNT, reg); 11981 reg = ((u64)encode_rcv_header_entry_size(entsize) & 11982 RCV_HDR_ENT_SIZE_ENT_SIZE_MASK) << 11983 RCV_HDR_ENT_SIZE_ENT_SIZE_SHIFT; 11984 write_kctxt_csr(dd, ctxt, RCV_HDR_ENT_SIZE, reg); 11985 reg = ((u64)DEFAULT_RCVHDRSIZE & RCV_HDR_SIZE_HDR_SIZE_MASK) << 11986 RCV_HDR_SIZE_HDR_SIZE_SHIFT; 11987 write_kctxt_csr(dd, ctxt, RCV_HDR_SIZE, reg); 11988 11989 /* 11990 * Program dummy tail address for every receive context 11991 * before enabling any receive context 11992 */ 11993 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR, 11994 dd->rcvhdrtail_dummy_dma); 11995 } 11996 11997 void hfi1_rcvctrl(struct hfi1_devdata *dd, unsigned int op, 11998 struct hfi1_ctxtdata *rcd) 11999 { 12000 u64 rcvctrl, reg; 12001 int did_enable = 0; 12002 u16 ctxt; 12003 12004 if (!rcd) 12005 return; 12006 12007 ctxt = rcd->ctxt; 12008 12009 hfi1_cdbg(RCVCTRL, "ctxt %d op 0x%x", ctxt, op); 12010 12011 rcvctrl = read_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL); 12012 /* if the context already enabled, don't do the extra steps */ 12013 if ((op & HFI1_RCVCTRL_CTXT_ENB) && 12014 !(rcvctrl & RCV_CTXT_CTRL_ENABLE_SMASK)) { 12015 /* reset the tail and hdr addresses, and sequence count */ 12016 write_kctxt_csr(dd, ctxt, RCV_HDR_ADDR, 12017 rcd->rcvhdrq_dma); 12018 if (hfi1_rcvhdrtail_kvaddr(rcd)) 12019 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR, 12020 rcd->rcvhdrqtailaddr_dma); 12021 hfi1_set_seq_cnt(rcd, 1); 12022 12023 /* reset the cached receive header queue head value */ 12024 hfi1_set_rcd_head(rcd, 0); 12025 12026 /* 12027 * Zero the receive header queue so we don't get false 12028 * positives when checking the sequence number. The 12029 * sequence numbers could land exactly on the same spot. 12030 * E.g. a rcd restart before the receive header wrapped. 12031 */ 12032 memset(rcd->rcvhdrq, 0, rcvhdrq_size(rcd)); 12033 12034 /* starting timeout */ 12035 rcd->rcvavail_timeout = dd->rcv_intr_timeout_csr; 12036 12037 /* enable the context */ 12038 rcvctrl |= RCV_CTXT_CTRL_ENABLE_SMASK; 12039 12040 /* clean the egr buffer size first */ 12041 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK; 12042 rcvctrl |= ((u64)encoded_size(rcd->egrbufs.rcvtid_size) 12043 & RCV_CTXT_CTRL_EGR_BUF_SIZE_MASK) 12044 << RCV_CTXT_CTRL_EGR_BUF_SIZE_SHIFT; 12045 12046 /* zero RcvHdrHead - set RcvHdrHead.Counter after enable */ 12047 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0); 12048 did_enable = 1; 12049 12050 /* zero RcvEgrIndexHead */ 12051 write_uctxt_csr(dd, ctxt, RCV_EGR_INDEX_HEAD, 0); 12052 12053 /* set eager count and base index */ 12054 reg = (((u64)(rcd->egrbufs.alloced >> RCV_SHIFT) 12055 & RCV_EGR_CTRL_EGR_CNT_MASK) 12056 << RCV_EGR_CTRL_EGR_CNT_SHIFT) | 12057 (((rcd->eager_base >> RCV_SHIFT) 12058 & RCV_EGR_CTRL_EGR_BASE_INDEX_MASK) 12059 << RCV_EGR_CTRL_EGR_BASE_INDEX_SHIFT); 12060 write_kctxt_csr(dd, ctxt, RCV_EGR_CTRL, reg); 12061 12062 /* 12063 * Set TID (expected) count and base index. 12064 * rcd->expected_count is set to individual RcvArray entries, 12065 * not pairs, and the CSR takes a pair-count in groups of 12066 * four, so divide by 8. 12067 */ 12068 reg = (((rcd->expected_count >> RCV_SHIFT) 12069 & RCV_TID_CTRL_TID_PAIR_CNT_MASK) 12070 << RCV_TID_CTRL_TID_PAIR_CNT_SHIFT) | 12071 (((rcd->expected_base >> RCV_SHIFT) 12072 & RCV_TID_CTRL_TID_BASE_INDEX_MASK) 12073 << RCV_TID_CTRL_TID_BASE_INDEX_SHIFT); 12074 write_kctxt_csr(dd, ctxt, RCV_TID_CTRL, reg); 12075 if (ctxt == HFI1_CTRL_CTXT) 12076 write_csr(dd, RCV_VL15, HFI1_CTRL_CTXT); 12077 } 12078 if (op & HFI1_RCVCTRL_CTXT_DIS) { 12079 write_csr(dd, RCV_VL15, 0); 12080 /* 12081 * When receive context is being disabled turn on tail 12082 * update with a dummy tail address and then disable 12083 * receive context. 12084 */ 12085 if (dd->rcvhdrtail_dummy_dma) { 12086 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR, 12087 dd->rcvhdrtail_dummy_dma); 12088 /* Enabling RcvCtxtCtrl.TailUpd is intentional. */ 12089 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK; 12090 } 12091 12092 rcvctrl &= ~RCV_CTXT_CTRL_ENABLE_SMASK; 12093 } 12094 if (op & HFI1_RCVCTRL_INTRAVAIL_ENB) { 12095 set_intr_bits(dd, IS_RCVAVAIL_START + rcd->ctxt, 12096 IS_RCVAVAIL_START + rcd->ctxt, true); 12097 rcvctrl |= RCV_CTXT_CTRL_INTR_AVAIL_SMASK; 12098 } 12099 if (op & HFI1_RCVCTRL_INTRAVAIL_DIS) { 12100 set_intr_bits(dd, IS_RCVAVAIL_START + rcd->ctxt, 12101 IS_RCVAVAIL_START + rcd->ctxt, false); 12102 rcvctrl &= ~RCV_CTXT_CTRL_INTR_AVAIL_SMASK; 12103 } 12104 if ((op & HFI1_RCVCTRL_TAILUPD_ENB) && hfi1_rcvhdrtail_kvaddr(rcd)) 12105 rcvctrl |= RCV_CTXT_CTRL_TAIL_UPD_SMASK; 12106 if (op & HFI1_RCVCTRL_TAILUPD_DIS) { 12107 /* See comment on RcvCtxtCtrl.TailUpd above */ 12108 if (!(op & HFI1_RCVCTRL_CTXT_DIS)) 12109 rcvctrl &= ~RCV_CTXT_CTRL_TAIL_UPD_SMASK; 12110 } 12111 if (op & HFI1_RCVCTRL_TIDFLOW_ENB) 12112 rcvctrl |= RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK; 12113 if (op & HFI1_RCVCTRL_TIDFLOW_DIS) 12114 rcvctrl &= ~RCV_CTXT_CTRL_TID_FLOW_ENABLE_SMASK; 12115 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_ENB) { 12116 /* 12117 * In one-packet-per-eager mode, the size comes from 12118 * the RcvArray entry. 12119 */ 12120 rcvctrl &= ~RCV_CTXT_CTRL_EGR_BUF_SIZE_SMASK; 12121 rcvctrl |= RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK; 12122 } 12123 if (op & HFI1_RCVCTRL_ONE_PKT_EGR_DIS) 12124 rcvctrl &= ~RCV_CTXT_CTRL_ONE_PACKET_PER_EGR_BUFFER_SMASK; 12125 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_ENB) 12126 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK; 12127 if (op & HFI1_RCVCTRL_NO_RHQ_DROP_DIS) 12128 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK; 12129 if (op & HFI1_RCVCTRL_NO_EGR_DROP_ENB) 12130 rcvctrl |= RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK; 12131 if (op & HFI1_RCVCTRL_NO_EGR_DROP_DIS) 12132 rcvctrl &= ~RCV_CTXT_CTRL_DONT_DROP_EGR_FULL_SMASK; 12133 if (op & HFI1_RCVCTRL_URGENT_ENB) 12134 set_intr_bits(dd, IS_RCVURGENT_START + rcd->ctxt, 12135 IS_RCVURGENT_START + rcd->ctxt, true); 12136 if (op & HFI1_RCVCTRL_URGENT_DIS) 12137 set_intr_bits(dd, IS_RCVURGENT_START + rcd->ctxt, 12138 IS_RCVURGENT_START + rcd->ctxt, false); 12139 12140 hfi1_cdbg(RCVCTRL, "ctxt %d rcvctrl 0x%llx", ctxt, rcvctrl); 12141 write_kctxt_csr(dd, ctxt, RCV_CTXT_CTRL, rcvctrl); 12142 12143 /* work around sticky RcvCtxtStatus.BlockedRHQFull */ 12144 if (did_enable && 12145 (rcvctrl & RCV_CTXT_CTRL_DONT_DROP_RHQ_FULL_SMASK)) { 12146 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS); 12147 if (reg != 0) { 12148 dd_dev_info(dd, "ctxt %d status %lld (blocked)\n", 12149 ctxt, reg); 12150 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD); 12151 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x10); 12152 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, 0x00); 12153 read_uctxt_csr(dd, ctxt, RCV_HDR_HEAD); 12154 reg = read_kctxt_csr(dd, ctxt, RCV_CTXT_STATUS); 12155 dd_dev_info(dd, "ctxt %d status %lld (%s blocked)\n", 12156 ctxt, reg, reg == 0 ? "not" : "still"); 12157 } 12158 } 12159 12160 if (did_enable) { 12161 /* 12162 * The interrupt timeout and count must be set after 12163 * the context is enabled to take effect. 12164 */ 12165 /* set interrupt timeout */ 12166 write_kctxt_csr(dd, ctxt, RCV_AVAIL_TIME_OUT, 12167 (u64)rcd->rcvavail_timeout << 12168 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_SHIFT); 12169 12170 /* set RcvHdrHead.Counter, zero RcvHdrHead.Head (again) */ 12171 reg = (u64)rcv_intr_count << RCV_HDR_HEAD_COUNTER_SHIFT; 12172 write_uctxt_csr(dd, ctxt, RCV_HDR_HEAD, reg); 12173 } 12174 12175 if (op & (HFI1_RCVCTRL_TAILUPD_DIS | HFI1_RCVCTRL_CTXT_DIS)) 12176 /* 12177 * If the context has been disabled and the Tail Update has 12178 * been cleared, set the RCV_HDR_TAIL_ADDR CSR to dummy address 12179 * so it doesn't contain an address that is invalid. 12180 */ 12181 write_kctxt_csr(dd, ctxt, RCV_HDR_TAIL_ADDR, 12182 dd->rcvhdrtail_dummy_dma); 12183 } 12184 12185 u32 hfi1_read_cntrs(struct hfi1_devdata *dd, char **namep, u64 **cntrp) 12186 { 12187 int ret; 12188 u64 val = 0; 12189 12190 if (namep) { 12191 ret = dd->cntrnameslen; 12192 *namep = dd->cntrnames; 12193 } else { 12194 const struct cntr_entry *entry; 12195 int i, j; 12196 12197 ret = (dd->ndevcntrs) * sizeof(u64); 12198 12199 /* Get the start of the block of counters */ 12200 *cntrp = dd->cntrs; 12201 12202 /* 12203 * Now go and fill in each counter in the block. 12204 */ 12205 for (i = 0; i < DEV_CNTR_LAST; i++) { 12206 entry = &dev_cntrs[i]; 12207 hfi1_cdbg(CNTR, "reading %s", entry->name); 12208 if (entry->flags & CNTR_DISABLED) { 12209 /* Nothing */ 12210 hfi1_cdbg(CNTR, "\tDisabled"); 12211 } else { 12212 if (entry->flags & CNTR_VL) { 12213 hfi1_cdbg(CNTR, "\tPer VL"); 12214 for (j = 0; j < C_VL_COUNT; j++) { 12215 val = entry->rw_cntr(entry, 12216 dd, j, 12217 CNTR_MODE_R, 12218 0); 12219 hfi1_cdbg( 12220 CNTR, 12221 "\t\tRead 0x%llx for %d", 12222 val, j); 12223 dd->cntrs[entry->offset + j] = 12224 val; 12225 } 12226 } else if (entry->flags & CNTR_SDMA) { 12227 hfi1_cdbg(CNTR, 12228 "\t Per SDMA Engine"); 12229 for (j = 0; j < chip_sdma_engines(dd); 12230 j++) { 12231 val = 12232 entry->rw_cntr(entry, dd, j, 12233 CNTR_MODE_R, 0); 12234 hfi1_cdbg(CNTR, 12235 "\t\tRead 0x%llx for %d", 12236 val, j); 12237 dd->cntrs[entry->offset + j] = 12238 val; 12239 } 12240 } else { 12241 val = entry->rw_cntr(entry, dd, 12242 CNTR_INVALID_VL, 12243 CNTR_MODE_R, 0); 12244 dd->cntrs[entry->offset] = val; 12245 hfi1_cdbg(CNTR, "\tRead 0x%llx", val); 12246 } 12247 } 12248 } 12249 } 12250 return ret; 12251 } 12252 12253 /* 12254 * Used by sysfs to create files for hfi stats to read 12255 */ 12256 u32 hfi1_read_portcntrs(struct hfi1_pportdata *ppd, char **namep, u64 **cntrp) 12257 { 12258 int ret; 12259 u64 val = 0; 12260 12261 if (namep) { 12262 ret = ppd->dd->portcntrnameslen; 12263 *namep = ppd->dd->portcntrnames; 12264 } else { 12265 const struct cntr_entry *entry; 12266 int i, j; 12267 12268 ret = ppd->dd->nportcntrs * sizeof(u64); 12269 *cntrp = ppd->cntrs; 12270 12271 for (i = 0; i < PORT_CNTR_LAST; i++) { 12272 entry = &port_cntrs[i]; 12273 hfi1_cdbg(CNTR, "reading %s", entry->name); 12274 if (entry->flags & CNTR_DISABLED) { 12275 /* Nothing */ 12276 hfi1_cdbg(CNTR, "\tDisabled"); 12277 continue; 12278 } 12279 12280 if (entry->flags & CNTR_VL) { 12281 hfi1_cdbg(CNTR, "\tPer VL"); 12282 for (j = 0; j < C_VL_COUNT; j++) { 12283 val = entry->rw_cntr(entry, ppd, j, 12284 CNTR_MODE_R, 12285 0); 12286 hfi1_cdbg( 12287 CNTR, 12288 "\t\tRead 0x%llx for %d", 12289 val, j); 12290 ppd->cntrs[entry->offset + j] = val; 12291 } 12292 } else { 12293 val = entry->rw_cntr(entry, ppd, 12294 CNTR_INVALID_VL, 12295 CNTR_MODE_R, 12296 0); 12297 ppd->cntrs[entry->offset] = val; 12298 hfi1_cdbg(CNTR, "\tRead 0x%llx", val); 12299 } 12300 } 12301 } 12302 return ret; 12303 } 12304 12305 static void free_cntrs(struct hfi1_devdata *dd) 12306 { 12307 struct hfi1_pportdata *ppd; 12308 int i; 12309 12310 if (dd->synth_stats_timer.function) 12311 del_timer_sync(&dd->synth_stats_timer); 12312 cancel_work_sync(&dd->update_cntr_work); 12313 ppd = (struct hfi1_pportdata *)(dd + 1); 12314 for (i = 0; i < dd->num_pports; i++, ppd++) { 12315 kfree(ppd->cntrs); 12316 kfree(ppd->scntrs); 12317 free_percpu(ppd->ibport_data.rvp.rc_acks); 12318 free_percpu(ppd->ibport_data.rvp.rc_qacks); 12319 free_percpu(ppd->ibport_data.rvp.rc_delayed_comp); 12320 ppd->cntrs = NULL; 12321 ppd->scntrs = NULL; 12322 ppd->ibport_data.rvp.rc_acks = NULL; 12323 ppd->ibport_data.rvp.rc_qacks = NULL; 12324 ppd->ibport_data.rvp.rc_delayed_comp = NULL; 12325 } 12326 kfree(dd->portcntrnames); 12327 dd->portcntrnames = NULL; 12328 kfree(dd->cntrs); 12329 dd->cntrs = NULL; 12330 kfree(dd->scntrs); 12331 dd->scntrs = NULL; 12332 kfree(dd->cntrnames); 12333 dd->cntrnames = NULL; 12334 if (dd->update_cntr_wq) { 12335 destroy_workqueue(dd->update_cntr_wq); 12336 dd->update_cntr_wq = NULL; 12337 } 12338 } 12339 12340 static u64 read_dev_port_cntr(struct hfi1_devdata *dd, struct cntr_entry *entry, 12341 u64 *psval, void *context, int vl) 12342 { 12343 u64 val; 12344 u64 sval = *psval; 12345 12346 if (entry->flags & CNTR_DISABLED) { 12347 dd_dev_err(dd, "Counter %s not enabled", entry->name); 12348 return 0; 12349 } 12350 12351 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval); 12352 12353 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_R, 0); 12354 12355 /* If its a synthetic counter there is more work we need to do */ 12356 if (entry->flags & CNTR_SYNTH) { 12357 if (sval == CNTR_MAX) { 12358 /* No need to read already saturated */ 12359 return CNTR_MAX; 12360 } 12361 12362 if (entry->flags & CNTR_32BIT) { 12363 /* 32bit counters can wrap multiple times */ 12364 u64 upper = sval >> 32; 12365 u64 lower = (sval << 32) >> 32; 12366 12367 if (lower > val) { /* hw wrapped */ 12368 if (upper == CNTR_32BIT_MAX) 12369 val = CNTR_MAX; 12370 else 12371 upper++; 12372 } 12373 12374 if (val != CNTR_MAX) 12375 val = (upper << 32) | val; 12376 12377 } else { 12378 /* If we rolled we are saturated */ 12379 if ((val < sval) || (val > CNTR_MAX)) 12380 val = CNTR_MAX; 12381 } 12382 } 12383 12384 *psval = val; 12385 12386 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val); 12387 12388 return val; 12389 } 12390 12391 static u64 write_dev_port_cntr(struct hfi1_devdata *dd, 12392 struct cntr_entry *entry, 12393 u64 *psval, void *context, int vl, u64 data) 12394 { 12395 u64 val; 12396 12397 if (entry->flags & CNTR_DISABLED) { 12398 dd_dev_err(dd, "Counter %s not enabled", entry->name); 12399 return 0; 12400 } 12401 12402 hfi1_cdbg(CNTR, "cntr: %s vl %d psval 0x%llx", entry->name, vl, *psval); 12403 12404 if (entry->flags & CNTR_SYNTH) { 12405 *psval = data; 12406 if (entry->flags & CNTR_32BIT) { 12407 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, 12408 (data << 32) >> 32); 12409 val = data; /* return the full 64bit value */ 12410 } else { 12411 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, 12412 data); 12413 } 12414 } else { 12415 val = entry->rw_cntr(entry, context, vl, CNTR_MODE_W, data); 12416 } 12417 12418 *psval = val; 12419 12420 hfi1_cdbg(CNTR, "\tNew val=0x%llx", val); 12421 12422 return val; 12423 } 12424 12425 u64 read_dev_cntr(struct hfi1_devdata *dd, int index, int vl) 12426 { 12427 struct cntr_entry *entry; 12428 u64 *sval; 12429 12430 entry = &dev_cntrs[index]; 12431 sval = dd->scntrs + entry->offset; 12432 12433 if (vl != CNTR_INVALID_VL) 12434 sval += vl; 12435 12436 return read_dev_port_cntr(dd, entry, sval, dd, vl); 12437 } 12438 12439 u64 write_dev_cntr(struct hfi1_devdata *dd, int index, int vl, u64 data) 12440 { 12441 struct cntr_entry *entry; 12442 u64 *sval; 12443 12444 entry = &dev_cntrs[index]; 12445 sval = dd->scntrs + entry->offset; 12446 12447 if (vl != CNTR_INVALID_VL) 12448 sval += vl; 12449 12450 return write_dev_port_cntr(dd, entry, sval, dd, vl, data); 12451 } 12452 12453 u64 read_port_cntr(struct hfi1_pportdata *ppd, int index, int vl) 12454 { 12455 struct cntr_entry *entry; 12456 u64 *sval; 12457 12458 entry = &port_cntrs[index]; 12459 sval = ppd->scntrs + entry->offset; 12460 12461 if (vl != CNTR_INVALID_VL) 12462 sval += vl; 12463 12464 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) && 12465 (index <= C_RCV_HDR_OVF_LAST)) { 12466 /* We do not want to bother for disabled contexts */ 12467 return 0; 12468 } 12469 12470 return read_dev_port_cntr(ppd->dd, entry, sval, ppd, vl); 12471 } 12472 12473 u64 write_port_cntr(struct hfi1_pportdata *ppd, int index, int vl, u64 data) 12474 { 12475 struct cntr_entry *entry; 12476 u64 *sval; 12477 12478 entry = &port_cntrs[index]; 12479 sval = ppd->scntrs + entry->offset; 12480 12481 if (vl != CNTR_INVALID_VL) 12482 sval += vl; 12483 12484 if ((index >= C_RCV_HDR_OVF_FIRST + ppd->dd->num_rcv_contexts) && 12485 (index <= C_RCV_HDR_OVF_LAST)) { 12486 /* We do not want to bother for disabled contexts */ 12487 return 0; 12488 } 12489 12490 return write_dev_port_cntr(ppd->dd, entry, sval, ppd, vl, data); 12491 } 12492 12493 static void do_update_synth_timer(struct work_struct *work) 12494 { 12495 u64 cur_tx; 12496 u64 cur_rx; 12497 u64 total_flits; 12498 u8 update = 0; 12499 int i, j, vl; 12500 struct hfi1_pportdata *ppd; 12501 struct cntr_entry *entry; 12502 struct hfi1_devdata *dd = container_of(work, struct hfi1_devdata, 12503 update_cntr_work); 12504 12505 /* 12506 * Rather than keep beating on the CSRs pick a minimal set that we can 12507 * check to watch for potential roll over. We can do this by looking at 12508 * the number of flits sent/recv. If the total flits exceeds 32bits then 12509 * we have to iterate all the counters and update. 12510 */ 12511 entry = &dev_cntrs[C_DC_RCV_FLITS]; 12512 cur_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0); 12513 12514 entry = &dev_cntrs[C_DC_XMIT_FLITS]; 12515 cur_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, CNTR_MODE_R, 0); 12516 12517 hfi1_cdbg( 12518 CNTR, 12519 "[%d] curr tx=0x%llx rx=0x%llx :: last tx=0x%llx rx=0x%llx", 12520 dd->unit, cur_tx, cur_rx, dd->last_tx, dd->last_rx); 12521 12522 if ((cur_tx < dd->last_tx) || (cur_rx < dd->last_rx)) { 12523 /* 12524 * May not be strictly necessary to update but it won't hurt and 12525 * simplifies the logic here. 12526 */ 12527 update = 1; 12528 hfi1_cdbg(CNTR, "[%d] Tripwire counter rolled, updating", 12529 dd->unit); 12530 } else { 12531 total_flits = (cur_tx - dd->last_tx) + (cur_rx - dd->last_rx); 12532 hfi1_cdbg(CNTR, 12533 "[%d] total flits 0x%llx limit 0x%llx", dd->unit, 12534 total_flits, (u64)CNTR_32BIT_MAX); 12535 if (total_flits >= CNTR_32BIT_MAX) { 12536 hfi1_cdbg(CNTR, "[%d] 32bit limit hit, updating", 12537 dd->unit); 12538 update = 1; 12539 } 12540 } 12541 12542 if (update) { 12543 hfi1_cdbg(CNTR, "[%d] Updating dd and ppd counters", dd->unit); 12544 for (i = 0; i < DEV_CNTR_LAST; i++) { 12545 entry = &dev_cntrs[i]; 12546 if (entry->flags & CNTR_VL) { 12547 for (vl = 0; vl < C_VL_COUNT; vl++) 12548 read_dev_cntr(dd, i, vl); 12549 } else { 12550 read_dev_cntr(dd, i, CNTR_INVALID_VL); 12551 } 12552 } 12553 ppd = (struct hfi1_pportdata *)(dd + 1); 12554 for (i = 0; i < dd->num_pports; i++, ppd++) { 12555 for (j = 0; j < PORT_CNTR_LAST; j++) { 12556 entry = &port_cntrs[j]; 12557 if (entry->flags & CNTR_VL) { 12558 for (vl = 0; vl < C_VL_COUNT; vl++) 12559 read_port_cntr(ppd, j, vl); 12560 } else { 12561 read_port_cntr(ppd, j, CNTR_INVALID_VL); 12562 } 12563 } 12564 } 12565 12566 /* 12567 * We want the value in the register. The goal is to keep track 12568 * of the number of "ticks" not the counter value. In other 12569 * words if the register rolls we want to notice it and go ahead 12570 * and force an update. 12571 */ 12572 entry = &dev_cntrs[C_DC_XMIT_FLITS]; 12573 dd->last_tx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, 12574 CNTR_MODE_R, 0); 12575 12576 entry = &dev_cntrs[C_DC_RCV_FLITS]; 12577 dd->last_rx = entry->rw_cntr(entry, dd, CNTR_INVALID_VL, 12578 CNTR_MODE_R, 0); 12579 12580 hfi1_cdbg(CNTR, "[%d] setting last tx/rx to 0x%llx 0x%llx", 12581 dd->unit, dd->last_tx, dd->last_rx); 12582 12583 } else { 12584 hfi1_cdbg(CNTR, "[%d] No update necessary", dd->unit); 12585 } 12586 } 12587 12588 static void update_synth_timer(struct timer_list *t) 12589 { 12590 struct hfi1_devdata *dd = from_timer(dd, t, synth_stats_timer); 12591 12592 queue_work(dd->update_cntr_wq, &dd->update_cntr_work); 12593 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME); 12594 } 12595 12596 #define C_MAX_NAME 16 /* 15 chars + one for /0 */ 12597 static int init_cntrs(struct hfi1_devdata *dd) 12598 { 12599 int i, rcv_ctxts, j; 12600 size_t sz; 12601 char *p; 12602 char name[C_MAX_NAME]; 12603 struct hfi1_pportdata *ppd; 12604 const char *bit_type_32 = ",32"; 12605 const int bit_type_32_sz = strlen(bit_type_32); 12606 u32 sdma_engines = chip_sdma_engines(dd); 12607 12608 /* set up the stats timer; the add_timer is done at the end */ 12609 timer_setup(&dd->synth_stats_timer, update_synth_timer, 0); 12610 12611 /***********************/ 12612 /* per device counters */ 12613 /***********************/ 12614 12615 /* size names and determine how many we have*/ 12616 dd->ndevcntrs = 0; 12617 sz = 0; 12618 12619 for (i = 0; i < DEV_CNTR_LAST; i++) { 12620 if (dev_cntrs[i].flags & CNTR_DISABLED) { 12621 hfi1_dbg_early("\tSkipping %s\n", dev_cntrs[i].name); 12622 continue; 12623 } 12624 12625 if (dev_cntrs[i].flags & CNTR_VL) { 12626 dev_cntrs[i].offset = dd->ndevcntrs; 12627 for (j = 0; j < C_VL_COUNT; j++) { 12628 snprintf(name, C_MAX_NAME, "%s%d", 12629 dev_cntrs[i].name, vl_from_idx(j)); 12630 sz += strlen(name); 12631 /* Add ",32" for 32-bit counters */ 12632 if (dev_cntrs[i].flags & CNTR_32BIT) 12633 sz += bit_type_32_sz; 12634 sz++; 12635 dd->ndevcntrs++; 12636 } 12637 } else if (dev_cntrs[i].flags & CNTR_SDMA) { 12638 dev_cntrs[i].offset = dd->ndevcntrs; 12639 for (j = 0; j < sdma_engines; j++) { 12640 snprintf(name, C_MAX_NAME, "%s%d", 12641 dev_cntrs[i].name, j); 12642 sz += strlen(name); 12643 /* Add ",32" for 32-bit counters */ 12644 if (dev_cntrs[i].flags & CNTR_32BIT) 12645 sz += bit_type_32_sz; 12646 sz++; 12647 dd->ndevcntrs++; 12648 } 12649 } else { 12650 /* +1 for newline. */ 12651 sz += strlen(dev_cntrs[i].name) + 1; 12652 /* Add ",32" for 32-bit counters */ 12653 if (dev_cntrs[i].flags & CNTR_32BIT) 12654 sz += bit_type_32_sz; 12655 dev_cntrs[i].offset = dd->ndevcntrs; 12656 dd->ndevcntrs++; 12657 } 12658 } 12659 12660 /* allocate space for the counter values */ 12661 dd->cntrs = kcalloc(dd->ndevcntrs + num_driver_cntrs, sizeof(u64), 12662 GFP_KERNEL); 12663 if (!dd->cntrs) 12664 goto bail; 12665 12666 dd->scntrs = kcalloc(dd->ndevcntrs, sizeof(u64), GFP_KERNEL); 12667 if (!dd->scntrs) 12668 goto bail; 12669 12670 /* allocate space for the counter names */ 12671 dd->cntrnameslen = sz; 12672 dd->cntrnames = kmalloc(sz, GFP_KERNEL); 12673 if (!dd->cntrnames) 12674 goto bail; 12675 12676 /* fill in the names */ 12677 for (p = dd->cntrnames, i = 0; i < DEV_CNTR_LAST; i++) { 12678 if (dev_cntrs[i].flags & CNTR_DISABLED) { 12679 /* Nothing */ 12680 } else if (dev_cntrs[i].flags & CNTR_VL) { 12681 for (j = 0; j < C_VL_COUNT; j++) { 12682 snprintf(name, C_MAX_NAME, "%s%d", 12683 dev_cntrs[i].name, 12684 vl_from_idx(j)); 12685 memcpy(p, name, strlen(name)); 12686 p += strlen(name); 12687 12688 /* Counter is 32 bits */ 12689 if (dev_cntrs[i].flags & CNTR_32BIT) { 12690 memcpy(p, bit_type_32, bit_type_32_sz); 12691 p += bit_type_32_sz; 12692 } 12693 12694 *p++ = '\n'; 12695 } 12696 } else if (dev_cntrs[i].flags & CNTR_SDMA) { 12697 for (j = 0; j < sdma_engines; j++) { 12698 snprintf(name, C_MAX_NAME, "%s%d", 12699 dev_cntrs[i].name, j); 12700 memcpy(p, name, strlen(name)); 12701 p += strlen(name); 12702 12703 /* Counter is 32 bits */ 12704 if (dev_cntrs[i].flags & CNTR_32BIT) { 12705 memcpy(p, bit_type_32, bit_type_32_sz); 12706 p += bit_type_32_sz; 12707 } 12708 12709 *p++ = '\n'; 12710 } 12711 } else { 12712 memcpy(p, dev_cntrs[i].name, strlen(dev_cntrs[i].name)); 12713 p += strlen(dev_cntrs[i].name); 12714 12715 /* Counter is 32 bits */ 12716 if (dev_cntrs[i].flags & CNTR_32BIT) { 12717 memcpy(p, bit_type_32, bit_type_32_sz); 12718 p += bit_type_32_sz; 12719 } 12720 12721 *p++ = '\n'; 12722 } 12723 } 12724 12725 /*********************/ 12726 /* per port counters */ 12727 /*********************/ 12728 12729 /* 12730 * Go through the counters for the overflows and disable the ones we 12731 * don't need. This varies based on platform so we need to do it 12732 * dynamically here. 12733 */ 12734 rcv_ctxts = dd->num_rcv_contexts; 12735 for (i = C_RCV_HDR_OVF_FIRST + rcv_ctxts; 12736 i <= C_RCV_HDR_OVF_LAST; i++) { 12737 port_cntrs[i].flags |= CNTR_DISABLED; 12738 } 12739 12740 /* size port counter names and determine how many we have*/ 12741 sz = 0; 12742 dd->nportcntrs = 0; 12743 for (i = 0; i < PORT_CNTR_LAST; i++) { 12744 if (port_cntrs[i].flags & CNTR_DISABLED) { 12745 hfi1_dbg_early("\tSkipping %s\n", port_cntrs[i].name); 12746 continue; 12747 } 12748 12749 if (port_cntrs[i].flags & CNTR_VL) { 12750 port_cntrs[i].offset = dd->nportcntrs; 12751 for (j = 0; j < C_VL_COUNT; j++) { 12752 snprintf(name, C_MAX_NAME, "%s%d", 12753 port_cntrs[i].name, vl_from_idx(j)); 12754 sz += strlen(name); 12755 /* Add ",32" for 32-bit counters */ 12756 if (port_cntrs[i].flags & CNTR_32BIT) 12757 sz += bit_type_32_sz; 12758 sz++; 12759 dd->nportcntrs++; 12760 } 12761 } else { 12762 /* +1 for newline */ 12763 sz += strlen(port_cntrs[i].name) + 1; 12764 /* Add ",32" for 32-bit counters */ 12765 if (port_cntrs[i].flags & CNTR_32BIT) 12766 sz += bit_type_32_sz; 12767 port_cntrs[i].offset = dd->nportcntrs; 12768 dd->nportcntrs++; 12769 } 12770 } 12771 12772 /* allocate space for the counter names */ 12773 dd->portcntrnameslen = sz; 12774 dd->portcntrnames = kmalloc(sz, GFP_KERNEL); 12775 if (!dd->portcntrnames) 12776 goto bail; 12777 12778 /* fill in port cntr names */ 12779 for (p = dd->portcntrnames, i = 0; i < PORT_CNTR_LAST; i++) { 12780 if (port_cntrs[i].flags & CNTR_DISABLED) 12781 continue; 12782 12783 if (port_cntrs[i].flags & CNTR_VL) { 12784 for (j = 0; j < C_VL_COUNT; j++) { 12785 snprintf(name, C_MAX_NAME, "%s%d", 12786 port_cntrs[i].name, vl_from_idx(j)); 12787 memcpy(p, name, strlen(name)); 12788 p += strlen(name); 12789 12790 /* Counter is 32 bits */ 12791 if (port_cntrs[i].flags & CNTR_32BIT) { 12792 memcpy(p, bit_type_32, bit_type_32_sz); 12793 p += bit_type_32_sz; 12794 } 12795 12796 *p++ = '\n'; 12797 } 12798 } else { 12799 memcpy(p, port_cntrs[i].name, 12800 strlen(port_cntrs[i].name)); 12801 p += strlen(port_cntrs[i].name); 12802 12803 /* Counter is 32 bits */ 12804 if (port_cntrs[i].flags & CNTR_32BIT) { 12805 memcpy(p, bit_type_32, bit_type_32_sz); 12806 p += bit_type_32_sz; 12807 } 12808 12809 *p++ = '\n'; 12810 } 12811 } 12812 12813 /* allocate per port storage for counter values */ 12814 ppd = (struct hfi1_pportdata *)(dd + 1); 12815 for (i = 0; i < dd->num_pports; i++, ppd++) { 12816 ppd->cntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL); 12817 if (!ppd->cntrs) 12818 goto bail; 12819 12820 ppd->scntrs = kcalloc(dd->nportcntrs, sizeof(u64), GFP_KERNEL); 12821 if (!ppd->scntrs) 12822 goto bail; 12823 } 12824 12825 /* CPU counters need to be allocated and zeroed */ 12826 if (init_cpu_counters(dd)) 12827 goto bail; 12828 12829 dd->update_cntr_wq = alloc_ordered_workqueue("hfi1_update_cntr_%d", 12830 WQ_MEM_RECLAIM, dd->unit); 12831 if (!dd->update_cntr_wq) 12832 goto bail; 12833 12834 INIT_WORK(&dd->update_cntr_work, do_update_synth_timer); 12835 12836 mod_timer(&dd->synth_stats_timer, jiffies + HZ * SYNTH_CNT_TIME); 12837 return 0; 12838 bail: 12839 free_cntrs(dd); 12840 return -ENOMEM; 12841 } 12842 12843 static u32 chip_to_opa_lstate(struct hfi1_devdata *dd, u32 chip_lstate) 12844 { 12845 switch (chip_lstate) { 12846 case LSTATE_DOWN: 12847 return IB_PORT_DOWN; 12848 case LSTATE_INIT: 12849 return IB_PORT_INIT; 12850 case LSTATE_ARMED: 12851 return IB_PORT_ARMED; 12852 case LSTATE_ACTIVE: 12853 return IB_PORT_ACTIVE; 12854 default: 12855 dd_dev_err(dd, 12856 "Unknown logical state 0x%x, reporting IB_PORT_DOWN\n", 12857 chip_lstate); 12858 return IB_PORT_DOWN; 12859 } 12860 } 12861 12862 u32 chip_to_opa_pstate(struct hfi1_devdata *dd, u32 chip_pstate) 12863 { 12864 /* look at the HFI meta-states only */ 12865 switch (chip_pstate & 0xf0) { 12866 case PLS_DISABLED: 12867 return IB_PORTPHYSSTATE_DISABLED; 12868 case PLS_OFFLINE: 12869 return OPA_PORTPHYSSTATE_OFFLINE; 12870 case PLS_POLLING: 12871 return IB_PORTPHYSSTATE_POLLING; 12872 case PLS_CONFIGPHY: 12873 return IB_PORTPHYSSTATE_TRAINING; 12874 case PLS_LINKUP: 12875 return IB_PORTPHYSSTATE_LINKUP; 12876 case PLS_PHYTEST: 12877 return IB_PORTPHYSSTATE_PHY_TEST; 12878 default: 12879 dd_dev_err(dd, "Unexpected chip physical state of 0x%x\n", 12880 chip_pstate); 12881 return IB_PORTPHYSSTATE_DISABLED; 12882 } 12883 } 12884 12885 /* return the OPA port logical state name */ 12886 const char *opa_lstate_name(u32 lstate) 12887 { 12888 static const char * const port_logical_names[] = { 12889 "PORT_NOP", 12890 "PORT_DOWN", 12891 "PORT_INIT", 12892 "PORT_ARMED", 12893 "PORT_ACTIVE", 12894 "PORT_ACTIVE_DEFER", 12895 }; 12896 if (lstate < ARRAY_SIZE(port_logical_names)) 12897 return port_logical_names[lstate]; 12898 return "unknown"; 12899 } 12900 12901 /* return the OPA port physical state name */ 12902 const char *opa_pstate_name(u32 pstate) 12903 { 12904 static const char * const port_physical_names[] = { 12905 "PHYS_NOP", 12906 "reserved1", 12907 "PHYS_POLL", 12908 "PHYS_DISABLED", 12909 "PHYS_TRAINING", 12910 "PHYS_LINKUP", 12911 "PHYS_LINK_ERR_RECOVER", 12912 "PHYS_PHY_TEST", 12913 "reserved8", 12914 "PHYS_OFFLINE", 12915 "PHYS_GANGED", 12916 "PHYS_TEST", 12917 }; 12918 if (pstate < ARRAY_SIZE(port_physical_names)) 12919 return port_physical_names[pstate]; 12920 return "unknown"; 12921 } 12922 12923 /** 12924 * update_statusp - Update userspace status flag 12925 * @ppd: Port data structure 12926 * @state: port state information 12927 * 12928 * Actual port status is determined by the host_link_state value 12929 * in the ppd. 12930 * 12931 * host_link_state MUST be updated before updating the user space 12932 * statusp. 12933 */ 12934 static void update_statusp(struct hfi1_pportdata *ppd, u32 state) 12935 { 12936 /* 12937 * Set port status flags in the page mapped into userspace 12938 * memory. Do it here to ensure a reliable state - this is 12939 * the only function called by all state handling code. 12940 * Always set the flags due to the fact that the cache value 12941 * might have been changed explicitly outside of this 12942 * function. 12943 */ 12944 if (ppd->statusp) { 12945 switch (state) { 12946 case IB_PORT_DOWN: 12947 case IB_PORT_INIT: 12948 *ppd->statusp &= ~(HFI1_STATUS_IB_CONF | 12949 HFI1_STATUS_IB_READY); 12950 break; 12951 case IB_PORT_ARMED: 12952 *ppd->statusp |= HFI1_STATUS_IB_CONF; 12953 break; 12954 case IB_PORT_ACTIVE: 12955 *ppd->statusp |= HFI1_STATUS_IB_READY; 12956 break; 12957 } 12958 } 12959 dd_dev_info(ppd->dd, "logical state changed to %s (0x%x)\n", 12960 opa_lstate_name(state), state); 12961 } 12962 12963 /** 12964 * wait_logical_linkstate - wait for an IB link state change to occur 12965 * @ppd: port device 12966 * @state: the state to wait for 12967 * @msecs: the number of milliseconds to wait 12968 * 12969 * Wait up to msecs milliseconds for IB link state change to occur. 12970 * For now, take the easy polling route. 12971 * Returns 0 if state reached, otherwise -ETIMEDOUT. 12972 */ 12973 static int wait_logical_linkstate(struct hfi1_pportdata *ppd, u32 state, 12974 int msecs) 12975 { 12976 unsigned long timeout; 12977 u32 new_state; 12978 12979 timeout = jiffies + msecs_to_jiffies(msecs); 12980 while (1) { 12981 new_state = chip_to_opa_lstate(ppd->dd, 12982 read_logical_state(ppd->dd)); 12983 if (new_state == state) 12984 break; 12985 if (time_after(jiffies, timeout)) { 12986 dd_dev_err(ppd->dd, 12987 "timeout waiting for link state 0x%x\n", 12988 state); 12989 return -ETIMEDOUT; 12990 } 12991 msleep(20); 12992 } 12993 12994 return 0; 12995 } 12996 12997 static void log_state_transition(struct hfi1_pportdata *ppd, u32 state) 12998 { 12999 u32 ib_pstate = chip_to_opa_pstate(ppd->dd, state); 13000 13001 dd_dev_info(ppd->dd, 13002 "physical state changed to %s (0x%x), phy 0x%x\n", 13003 opa_pstate_name(ib_pstate), ib_pstate, state); 13004 } 13005 13006 /* 13007 * Read the physical hardware link state and check if it matches host 13008 * drivers anticipated state. 13009 */ 13010 static void log_physical_state(struct hfi1_pportdata *ppd, u32 state) 13011 { 13012 u32 read_state = read_physical_state(ppd->dd); 13013 13014 if (read_state == state) { 13015 log_state_transition(ppd, state); 13016 } else { 13017 dd_dev_err(ppd->dd, 13018 "anticipated phy link state 0x%x, read 0x%x\n", 13019 state, read_state); 13020 } 13021 } 13022 13023 /* 13024 * wait_physical_linkstate - wait for an physical link state change to occur 13025 * @ppd: port device 13026 * @state: the state to wait for 13027 * @msecs: the number of milliseconds to wait 13028 * 13029 * Wait up to msecs milliseconds for physical link state change to occur. 13030 * Returns 0 if state reached, otherwise -ETIMEDOUT. 13031 */ 13032 static int wait_physical_linkstate(struct hfi1_pportdata *ppd, u32 state, 13033 int msecs) 13034 { 13035 u32 read_state; 13036 unsigned long timeout; 13037 13038 timeout = jiffies + msecs_to_jiffies(msecs); 13039 while (1) { 13040 read_state = read_physical_state(ppd->dd); 13041 if (read_state == state) 13042 break; 13043 if (time_after(jiffies, timeout)) { 13044 dd_dev_err(ppd->dd, 13045 "timeout waiting for phy link state 0x%x\n", 13046 state); 13047 return -ETIMEDOUT; 13048 } 13049 usleep_range(1950, 2050); /* sleep 2ms-ish */ 13050 } 13051 13052 log_state_transition(ppd, state); 13053 return 0; 13054 } 13055 13056 /* 13057 * wait_phys_link_offline_quiet_substates - wait for any offline substate 13058 * @ppd: port device 13059 * @msecs: the number of milliseconds to wait 13060 * 13061 * Wait up to msecs milliseconds for any offline physical link 13062 * state change to occur. 13063 * Returns 0 if at least one state is reached, otherwise -ETIMEDOUT. 13064 */ 13065 static int wait_phys_link_offline_substates(struct hfi1_pportdata *ppd, 13066 int msecs) 13067 { 13068 u32 read_state; 13069 unsigned long timeout; 13070 13071 timeout = jiffies + msecs_to_jiffies(msecs); 13072 while (1) { 13073 read_state = read_physical_state(ppd->dd); 13074 if ((read_state & 0xF0) == PLS_OFFLINE) 13075 break; 13076 if (time_after(jiffies, timeout)) { 13077 dd_dev_err(ppd->dd, 13078 "timeout waiting for phy link offline.quiet substates. Read state 0x%x, %dms\n", 13079 read_state, msecs); 13080 return -ETIMEDOUT; 13081 } 13082 usleep_range(1950, 2050); /* sleep 2ms-ish */ 13083 } 13084 13085 log_state_transition(ppd, read_state); 13086 return read_state; 13087 } 13088 13089 /* 13090 * wait_phys_link_out_of_offline - wait for any out of offline state 13091 * @ppd: port device 13092 * @msecs: the number of milliseconds to wait 13093 * 13094 * Wait up to msecs milliseconds for any out of offline physical link 13095 * state change to occur. 13096 * Returns 0 if at least one state is reached, otherwise -ETIMEDOUT. 13097 */ 13098 static int wait_phys_link_out_of_offline(struct hfi1_pportdata *ppd, 13099 int msecs) 13100 { 13101 u32 read_state; 13102 unsigned long timeout; 13103 13104 timeout = jiffies + msecs_to_jiffies(msecs); 13105 while (1) { 13106 read_state = read_physical_state(ppd->dd); 13107 if ((read_state & 0xF0) != PLS_OFFLINE) 13108 break; 13109 if (time_after(jiffies, timeout)) { 13110 dd_dev_err(ppd->dd, 13111 "timeout waiting for phy link out of offline. Read state 0x%x, %dms\n", 13112 read_state, msecs); 13113 return -ETIMEDOUT; 13114 } 13115 usleep_range(1950, 2050); /* sleep 2ms-ish */ 13116 } 13117 13118 log_state_transition(ppd, read_state); 13119 return read_state; 13120 } 13121 13122 #define CLEAR_STATIC_RATE_CONTROL_SMASK(r) \ 13123 (r &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK) 13124 13125 #define SET_STATIC_RATE_CONTROL_SMASK(r) \ 13126 (r |= SEND_CTXT_CHECK_ENABLE_DISALLOW_PBC_STATIC_RATE_CONTROL_SMASK) 13127 13128 void hfi1_init_ctxt(struct send_context *sc) 13129 { 13130 if (sc) { 13131 struct hfi1_devdata *dd = sc->dd; 13132 u64 reg; 13133 u8 set = (sc->type == SC_USER ? 13134 HFI1_CAP_IS_USET(STATIC_RATE_CTRL) : 13135 HFI1_CAP_IS_KSET(STATIC_RATE_CTRL)); 13136 reg = read_kctxt_csr(dd, sc->hw_context, 13137 SEND_CTXT_CHECK_ENABLE); 13138 if (set) 13139 CLEAR_STATIC_RATE_CONTROL_SMASK(reg); 13140 else 13141 SET_STATIC_RATE_CONTROL_SMASK(reg); 13142 write_kctxt_csr(dd, sc->hw_context, 13143 SEND_CTXT_CHECK_ENABLE, reg); 13144 } 13145 } 13146 13147 int hfi1_tempsense_rd(struct hfi1_devdata *dd, struct hfi1_temp *temp) 13148 { 13149 int ret = 0; 13150 u64 reg; 13151 13152 if (dd->icode != ICODE_RTL_SILICON) { 13153 if (HFI1_CAP_IS_KSET(PRINT_UNIMPL)) 13154 dd_dev_info(dd, "%s: tempsense not supported by HW\n", 13155 __func__); 13156 return -EINVAL; 13157 } 13158 reg = read_csr(dd, ASIC_STS_THERM); 13159 temp->curr = ((reg >> ASIC_STS_THERM_CURR_TEMP_SHIFT) & 13160 ASIC_STS_THERM_CURR_TEMP_MASK); 13161 temp->lo_lim = ((reg >> ASIC_STS_THERM_LO_TEMP_SHIFT) & 13162 ASIC_STS_THERM_LO_TEMP_MASK); 13163 temp->hi_lim = ((reg >> ASIC_STS_THERM_HI_TEMP_SHIFT) & 13164 ASIC_STS_THERM_HI_TEMP_MASK); 13165 temp->crit_lim = ((reg >> ASIC_STS_THERM_CRIT_TEMP_SHIFT) & 13166 ASIC_STS_THERM_CRIT_TEMP_MASK); 13167 /* triggers is a 3-bit value - 1 bit per trigger. */ 13168 temp->triggers = (u8)((reg >> ASIC_STS_THERM_LOW_SHIFT) & 0x7); 13169 13170 return ret; 13171 } 13172 13173 /* ========================================================================= */ 13174 13175 /** 13176 * read_mod_write() - Calculate the IRQ register index and set/clear the bits 13177 * @dd: valid devdata 13178 * @src: IRQ source to determine register index from 13179 * @bits: the bits to set or clear 13180 * @set: true == set the bits, false == clear the bits 13181 * 13182 */ 13183 static void read_mod_write(struct hfi1_devdata *dd, u16 src, u64 bits, 13184 bool set) 13185 { 13186 u64 reg; 13187 u16 idx = src / BITS_PER_REGISTER; 13188 unsigned long flags; 13189 13190 spin_lock_irqsave(&dd->irq_src_lock, flags); 13191 reg = read_csr(dd, CCE_INT_MASK + (8 * idx)); 13192 if (set) 13193 reg |= bits; 13194 else 13195 reg &= ~bits; 13196 write_csr(dd, CCE_INT_MASK + (8 * idx), reg); 13197 spin_unlock_irqrestore(&dd->irq_src_lock, flags); 13198 } 13199 13200 /** 13201 * set_intr_bits() - Enable/disable a range (one or more) IRQ sources 13202 * @dd: valid devdata 13203 * @first: first IRQ source to set/clear 13204 * @last: last IRQ source (inclusive) to set/clear 13205 * @set: true == set the bits, false == clear the bits 13206 * 13207 * If first == last, set the exact source. 13208 */ 13209 int set_intr_bits(struct hfi1_devdata *dd, u16 first, u16 last, bool set) 13210 { 13211 u64 bits = 0; 13212 u64 bit; 13213 u16 src; 13214 13215 if (first > NUM_INTERRUPT_SOURCES || last > NUM_INTERRUPT_SOURCES) 13216 return -EINVAL; 13217 13218 if (last < first) 13219 return -ERANGE; 13220 13221 for (src = first; src <= last; src++) { 13222 bit = src % BITS_PER_REGISTER; 13223 /* wrapped to next register? */ 13224 if (!bit && bits) { 13225 read_mod_write(dd, src - 1, bits, set); 13226 bits = 0; 13227 } 13228 bits |= BIT_ULL(bit); 13229 } 13230 read_mod_write(dd, last, bits, set); 13231 13232 return 0; 13233 } 13234 13235 /* 13236 * Clear all interrupt sources on the chip. 13237 */ 13238 void clear_all_interrupts(struct hfi1_devdata *dd) 13239 { 13240 int i; 13241 13242 for (i = 0; i < CCE_NUM_INT_CSRS; i++) 13243 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~(u64)0); 13244 13245 write_csr(dd, CCE_ERR_CLEAR, ~(u64)0); 13246 write_csr(dd, MISC_ERR_CLEAR, ~(u64)0); 13247 write_csr(dd, RCV_ERR_CLEAR, ~(u64)0); 13248 write_csr(dd, SEND_ERR_CLEAR, ~(u64)0); 13249 write_csr(dd, SEND_PIO_ERR_CLEAR, ~(u64)0); 13250 write_csr(dd, SEND_DMA_ERR_CLEAR, ~(u64)0); 13251 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~(u64)0); 13252 for (i = 0; i < chip_send_contexts(dd); i++) 13253 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~(u64)0); 13254 for (i = 0; i < chip_sdma_engines(dd); i++) 13255 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~(u64)0); 13256 13257 write_csr(dd, DCC_ERR_FLG_CLR, ~(u64)0); 13258 write_csr(dd, DC_LCB_ERR_CLR, ~(u64)0); 13259 write_csr(dd, DC_DC8051_ERR_CLR, ~(u64)0); 13260 } 13261 13262 /* 13263 * Remap the interrupt source from the general handler to the given MSI-X 13264 * interrupt. 13265 */ 13266 void remap_intr(struct hfi1_devdata *dd, int isrc, int msix_intr) 13267 { 13268 u64 reg; 13269 int m, n; 13270 13271 /* clear from the handled mask of the general interrupt */ 13272 m = isrc / 64; 13273 n = isrc % 64; 13274 if (likely(m < CCE_NUM_INT_CSRS)) { 13275 dd->gi_mask[m] &= ~((u64)1 << n); 13276 } else { 13277 dd_dev_err(dd, "remap interrupt err\n"); 13278 return; 13279 } 13280 13281 /* direct the chip source to the given MSI-X interrupt */ 13282 m = isrc / 8; 13283 n = isrc % 8; 13284 reg = read_csr(dd, CCE_INT_MAP + (8 * m)); 13285 reg &= ~((u64)0xff << (8 * n)); 13286 reg |= ((u64)msix_intr & 0xff) << (8 * n); 13287 write_csr(dd, CCE_INT_MAP + (8 * m), reg); 13288 } 13289 13290 void remap_sdma_interrupts(struct hfi1_devdata *dd, int engine, int msix_intr) 13291 { 13292 /* 13293 * SDMA engine interrupt sources grouped by type, rather than 13294 * engine. Per-engine interrupts are as follows: 13295 * SDMA 13296 * SDMAProgress 13297 * SDMAIdle 13298 */ 13299 remap_intr(dd, IS_SDMA_START + engine, msix_intr); 13300 remap_intr(dd, IS_SDMA_PROGRESS_START + engine, msix_intr); 13301 remap_intr(dd, IS_SDMA_IDLE_START + engine, msix_intr); 13302 } 13303 13304 /* 13305 * Set the general handler to accept all interrupts, remap all 13306 * chip interrupts back to MSI-X 0. 13307 */ 13308 void reset_interrupts(struct hfi1_devdata *dd) 13309 { 13310 int i; 13311 13312 /* all interrupts handled by the general handler */ 13313 for (i = 0; i < CCE_NUM_INT_CSRS; i++) 13314 dd->gi_mask[i] = ~(u64)0; 13315 13316 /* all chip interrupts map to MSI-X 0 */ 13317 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++) 13318 write_csr(dd, CCE_INT_MAP + (8 * i), 0); 13319 } 13320 13321 /** 13322 * set_up_interrupts() - Initialize the IRQ resources and state 13323 * @dd: valid devdata 13324 * 13325 */ 13326 static int set_up_interrupts(struct hfi1_devdata *dd) 13327 { 13328 int ret; 13329 13330 /* mask all interrupts */ 13331 set_intr_bits(dd, IS_FIRST_SOURCE, IS_LAST_SOURCE, false); 13332 13333 /* clear all pending interrupts */ 13334 clear_all_interrupts(dd); 13335 13336 /* reset general handler mask, chip MSI-X mappings */ 13337 reset_interrupts(dd); 13338 13339 /* ask for MSI-X interrupts */ 13340 ret = msix_initialize(dd); 13341 if (ret) 13342 return ret; 13343 13344 ret = msix_request_irqs(dd); 13345 if (ret) 13346 msix_clean_up_interrupts(dd); 13347 13348 return ret; 13349 } 13350 13351 /* 13352 * Set up context values in dd. Sets: 13353 * 13354 * num_rcv_contexts - number of contexts being used 13355 * n_krcv_queues - number of kernel contexts 13356 * first_dyn_alloc_ctxt - first dynamically allocated context 13357 * in array of contexts 13358 * freectxts - number of free user contexts 13359 * num_send_contexts - number of PIO send contexts being used 13360 * num_netdev_contexts - number of contexts reserved for netdev 13361 */ 13362 static int set_up_context_variables(struct hfi1_devdata *dd) 13363 { 13364 unsigned long num_kernel_contexts; 13365 u16 num_netdev_contexts; 13366 int ret; 13367 unsigned ngroups; 13368 int rmt_count; 13369 u32 n_usr_ctxts; 13370 u32 send_contexts = chip_send_contexts(dd); 13371 u32 rcv_contexts = chip_rcv_contexts(dd); 13372 13373 /* 13374 * Kernel receive contexts: 13375 * - Context 0 - control context (VL15/multicast/error) 13376 * - Context 1 - first kernel context 13377 * - Context 2 - second kernel context 13378 * ... 13379 */ 13380 if (n_krcvqs) 13381 /* 13382 * n_krcvqs is the sum of module parameter kernel receive 13383 * contexts, krcvqs[]. It does not include the control 13384 * context, so add that. 13385 */ 13386 num_kernel_contexts = n_krcvqs + 1; 13387 else 13388 num_kernel_contexts = DEFAULT_KRCVQS + 1; 13389 /* 13390 * Every kernel receive context needs an ACK send context. 13391 * one send context is allocated for each VL{0-7} and VL15 13392 */ 13393 if (num_kernel_contexts > (send_contexts - num_vls - 1)) { 13394 dd_dev_err(dd, 13395 "Reducing # kernel rcv contexts to: %d, from %lu\n", 13396 send_contexts - num_vls - 1, 13397 num_kernel_contexts); 13398 num_kernel_contexts = send_contexts - num_vls - 1; 13399 } 13400 13401 /* 13402 * User contexts: 13403 * - default to 1 user context per real (non-HT) CPU core if 13404 * num_user_contexts is negative 13405 */ 13406 if (num_user_contexts < 0) 13407 n_usr_ctxts = cpumask_weight(&node_affinity.real_cpu_mask); 13408 else 13409 n_usr_ctxts = num_user_contexts; 13410 /* 13411 * Adjust the counts given a global max. 13412 */ 13413 if (num_kernel_contexts + n_usr_ctxts > rcv_contexts) { 13414 dd_dev_err(dd, 13415 "Reducing # user receive contexts to: %u, from %u\n", 13416 (u32)(rcv_contexts - num_kernel_contexts), 13417 n_usr_ctxts); 13418 /* recalculate */ 13419 n_usr_ctxts = rcv_contexts - num_kernel_contexts; 13420 } 13421 13422 num_netdev_contexts = 13423 hfi1_num_netdev_contexts(dd, rcv_contexts - 13424 (num_kernel_contexts + n_usr_ctxts), 13425 &node_affinity.real_cpu_mask); 13426 /* 13427 * RMT entries are allocated as follows: 13428 * 1. QOS (0 to 128 entries) 13429 * 2. FECN (num_kernel_context - 1 [a] + num_user_contexts + 13430 * num_netdev_contexts [b]) 13431 * 3. netdev (NUM_NETDEV_MAP_ENTRIES) 13432 * 13433 * Notes: 13434 * [a] Kernel contexts (except control) are included in FECN if kernel 13435 * TID_RDMA is active. 13436 * [b] Netdev and user contexts are randomly allocated from the same 13437 * context pool, so FECN must cover all contexts in the pool. 13438 */ 13439 rmt_count = qos_rmt_entries(num_kernel_contexts - 1, NULL, NULL) 13440 + (HFI1_CAP_IS_KSET(TID_RDMA) ? num_kernel_contexts - 1 13441 : 0) 13442 + n_usr_ctxts 13443 + num_netdev_contexts 13444 + NUM_NETDEV_MAP_ENTRIES; 13445 if (rmt_count > NUM_MAP_ENTRIES) { 13446 int over = rmt_count - NUM_MAP_ENTRIES; 13447 /* try to squish user contexts, minimum of 1 */ 13448 if (over >= n_usr_ctxts) { 13449 dd_dev_err(dd, "RMT overflow: reduce the requested number of contexts\n"); 13450 return -EINVAL; 13451 } 13452 dd_dev_err(dd, "RMT overflow: reducing # user contexts from %u to %u\n", 13453 n_usr_ctxts, n_usr_ctxts - over); 13454 n_usr_ctxts -= over; 13455 } 13456 13457 /* the first N are kernel contexts, the rest are user/netdev contexts */ 13458 dd->num_rcv_contexts = 13459 num_kernel_contexts + n_usr_ctxts + num_netdev_contexts; 13460 dd->n_krcv_queues = num_kernel_contexts; 13461 dd->first_dyn_alloc_ctxt = num_kernel_contexts; 13462 dd->num_netdev_contexts = num_netdev_contexts; 13463 dd->num_user_contexts = n_usr_ctxts; 13464 dd->freectxts = n_usr_ctxts; 13465 dd_dev_info(dd, 13466 "rcv contexts: chip %d, used %d (kernel %d, netdev %u, user %u)\n", 13467 rcv_contexts, 13468 (int)dd->num_rcv_contexts, 13469 (int)dd->n_krcv_queues, 13470 dd->num_netdev_contexts, 13471 dd->num_user_contexts); 13472 13473 /* 13474 * Receive array allocation: 13475 * All RcvArray entries are divided into groups of 8. This 13476 * is required by the hardware and will speed up writes to 13477 * consecutive entries by using write-combining of the entire 13478 * cacheline. 13479 * 13480 * The number of groups are evenly divided among all contexts. 13481 * any left over groups will be given to the first N user 13482 * contexts. 13483 */ 13484 dd->rcv_entries.group_size = RCV_INCREMENT; 13485 ngroups = chip_rcv_array_count(dd) / dd->rcv_entries.group_size; 13486 dd->rcv_entries.ngroups = ngroups / dd->num_rcv_contexts; 13487 dd->rcv_entries.nctxt_extra = ngroups - 13488 (dd->num_rcv_contexts * dd->rcv_entries.ngroups); 13489 dd_dev_info(dd, "RcvArray groups %u, ctxts extra %u\n", 13490 dd->rcv_entries.ngroups, 13491 dd->rcv_entries.nctxt_extra); 13492 if (dd->rcv_entries.ngroups * dd->rcv_entries.group_size > 13493 MAX_EAGER_ENTRIES * 2) { 13494 dd->rcv_entries.ngroups = (MAX_EAGER_ENTRIES * 2) / 13495 dd->rcv_entries.group_size; 13496 dd_dev_info(dd, 13497 "RcvArray group count too high, change to %u\n", 13498 dd->rcv_entries.ngroups); 13499 dd->rcv_entries.nctxt_extra = 0; 13500 } 13501 /* 13502 * PIO send contexts 13503 */ 13504 ret = init_sc_pools_and_sizes(dd); 13505 if (ret >= 0) { /* success */ 13506 dd->num_send_contexts = ret; 13507 dd_dev_info( 13508 dd, 13509 "send contexts: chip %d, used %d (kernel %d, ack %d, user %d, vl15 %d)\n", 13510 send_contexts, 13511 dd->num_send_contexts, 13512 dd->sc_sizes[SC_KERNEL].count, 13513 dd->sc_sizes[SC_ACK].count, 13514 dd->sc_sizes[SC_USER].count, 13515 dd->sc_sizes[SC_VL15].count); 13516 ret = 0; /* success */ 13517 } 13518 13519 return ret; 13520 } 13521 13522 /* 13523 * Set the device/port partition key table. The MAD code 13524 * will ensure that, at least, the partial management 13525 * partition key is present in the table. 13526 */ 13527 static void set_partition_keys(struct hfi1_pportdata *ppd) 13528 { 13529 struct hfi1_devdata *dd = ppd->dd; 13530 u64 reg = 0; 13531 int i; 13532 13533 dd_dev_info(dd, "Setting partition keys\n"); 13534 for (i = 0; i < hfi1_get_npkeys(dd); i++) { 13535 reg |= (ppd->pkeys[i] & 13536 RCV_PARTITION_KEY_PARTITION_KEY_A_MASK) << 13537 ((i % 4) * 13538 RCV_PARTITION_KEY_PARTITION_KEY_B_SHIFT); 13539 /* Each register holds 4 PKey values. */ 13540 if ((i % 4) == 3) { 13541 write_csr(dd, RCV_PARTITION_KEY + 13542 ((i - 3) * 2), reg); 13543 reg = 0; 13544 } 13545 } 13546 13547 /* Always enable HW pkeys check when pkeys table is set */ 13548 add_rcvctrl(dd, RCV_CTRL_RCV_PARTITION_KEY_ENABLE_SMASK); 13549 } 13550 13551 /* 13552 * These CSRs and memories are uninitialized on reset and must be 13553 * written before reading to set the ECC/parity bits. 13554 * 13555 * NOTE: All user context CSRs that are not mmaped write-only 13556 * (e.g. the TID flows) must be initialized even if the driver never 13557 * reads them. 13558 */ 13559 static void write_uninitialized_csrs_and_memories(struct hfi1_devdata *dd) 13560 { 13561 int i, j; 13562 13563 /* CceIntMap */ 13564 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++) 13565 write_csr(dd, CCE_INT_MAP + (8 * i), 0); 13566 13567 /* SendCtxtCreditReturnAddr */ 13568 for (i = 0; i < chip_send_contexts(dd); i++) 13569 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0); 13570 13571 /* PIO Send buffers */ 13572 /* SDMA Send buffers */ 13573 /* 13574 * These are not normally read, and (presently) have no method 13575 * to be read, so are not pre-initialized 13576 */ 13577 13578 /* RcvHdrAddr */ 13579 /* RcvHdrTailAddr */ 13580 /* RcvTidFlowTable */ 13581 for (i = 0; i < chip_rcv_contexts(dd); i++) { 13582 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0); 13583 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0); 13584 for (j = 0; j < RXE_NUM_TID_FLOWS; j++) 13585 write_uctxt_csr(dd, i, RCV_TID_FLOW_TABLE + (8 * j), 0); 13586 } 13587 13588 /* RcvArray */ 13589 for (i = 0; i < chip_rcv_array_count(dd); i++) 13590 hfi1_put_tid(dd, i, PT_INVALID_FLUSH, 0, 0); 13591 13592 /* RcvQPMapTable */ 13593 for (i = 0; i < 32; i++) 13594 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0); 13595 } 13596 13597 /* 13598 * Use the ctrl_bits in CceCtrl to clear the status_bits in CceStatus. 13599 */ 13600 static void clear_cce_status(struct hfi1_devdata *dd, u64 status_bits, 13601 u64 ctrl_bits) 13602 { 13603 unsigned long timeout; 13604 u64 reg; 13605 13606 /* is the condition present? */ 13607 reg = read_csr(dd, CCE_STATUS); 13608 if ((reg & status_bits) == 0) 13609 return; 13610 13611 /* clear the condition */ 13612 write_csr(dd, CCE_CTRL, ctrl_bits); 13613 13614 /* wait for the condition to clear */ 13615 timeout = jiffies + msecs_to_jiffies(CCE_STATUS_TIMEOUT); 13616 while (1) { 13617 reg = read_csr(dd, CCE_STATUS); 13618 if ((reg & status_bits) == 0) 13619 return; 13620 if (time_after(jiffies, timeout)) { 13621 dd_dev_err(dd, 13622 "Timeout waiting for CceStatus to clear bits 0x%llx, remaining 0x%llx\n", 13623 status_bits, reg & status_bits); 13624 return; 13625 } 13626 udelay(1); 13627 } 13628 } 13629 13630 /* set CCE CSRs to chip reset defaults */ 13631 static void reset_cce_csrs(struct hfi1_devdata *dd) 13632 { 13633 int i; 13634 13635 /* CCE_REVISION read-only */ 13636 /* CCE_REVISION2 read-only */ 13637 /* CCE_CTRL - bits clear automatically */ 13638 /* CCE_STATUS read-only, use CceCtrl to clear */ 13639 clear_cce_status(dd, ALL_FROZE, CCE_CTRL_SPC_UNFREEZE_SMASK); 13640 clear_cce_status(dd, ALL_TXE_PAUSE, CCE_CTRL_TXE_RESUME_SMASK); 13641 clear_cce_status(dd, ALL_RXE_PAUSE, CCE_CTRL_RXE_RESUME_SMASK); 13642 for (i = 0; i < CCE_NUM_SCRATCH; i++) 13643 write_csr(dd, CCE_SCRATCH + (8 * i), 0); 13644 /* CCE_ERR_STATUS read-only */ 13645 write_csr(dd, CCE_ERR_MASK, 0); 13646 write_csr(dd, CCE_ERR_CLEAR, ~0ull); 13647 /* CCE_ERR_FORCE leave alone */ 13648 for (i = 0; i < CCE_NUM_32_BIT_COUNTERS; i++) 13649 write_csr(dd, CCE_COUNTER_ARRAY32 + (8 * i), 0); 13650 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_RESETCSR); 13651 /* CCE_PCIE_CTRL leave alone */ 13652 for (i = 0; i < CCE_NUM_MSIX_VECTORS; i++) { 13653 write_csr(dd, CCE_MSIX_TABLE_LOWER + (8 * i), 0); 13654 write_csr(dd, CCE_MSIX_TABLE_UPPER + (8 * i), 13655 CCE_MSIX_TABLE_UPPER_RESETCSR); 13656 } 13657 for (i = 0; i < CCE_NUM_MSIX_PBAS; i++) { 13658 /* CCE_MSIX_PBA read-only */ 13659 write_csr(dd, CCE_MSIX_INT_GRANTED, ~0ull); 13660 write_csr(dd, CCE_MSIX_VEC_CLR_WITHOUT_INT, ~0ull); 13661 } 13662 for (i = 0; i < CCE_NUM_INT_MAP_CSRS; i++) 13663 write_csr(dd, CCE_INT_MAP, 0); 13664 for (i = 0; i < CCE_NUM_INT_CSRS; i++) { 13665 /* CCE_INT_STATUS read-only */ 13666 write_csr(dd, CCE_INT_MASK + (8 * i), 0); 13667 write_csr(dd, CCE_INT_CLEAR + (8 * i), ~0ull); 13668 /* CCE_INT_FORCE leave alone */ 13669 /* CCE_INT_BLOCKED read-only */ 13670 } 13671 for (i = 0; i < CCE_NUM_32_BIT_INT_COUNTERS; i++) 13672 write_csr(dd, CCE_INT_COUNTER_ARRAY32 + (8 * i), 0); 13673 } 13674 13675 /* set MISC CSRs to chip reset defaults */ 13676 static void reset_misc_csrs(struct hfi1_devdata *dd) 13677 { 13678 int i; 13679 13680 for (i = 0; i < 32; i++) { 13681 write_csr(dd, MISC_CFG_RSA_R2 + (8 * i), 0); 13682 write_csr(dd, MISC_CFG_RSA_SIGNATURE + (8 * i), 0); 13683 write_csr(dd, MISC_CFG_RSA_MODULUS + (8 * i), 0); 13684 } 13685 /* 13686 * MISC_CFG_SHA_PRELOAD leave alone - always reads 0 and can 13687 * only be written 128-byte chunks 13688 */ 13689 /* init RSA engine to clear lingering errors */ 13690 write_csr(dd, MISC_CFG_RSA_CMD, 1); 13691 write_csr(dd, MISC_CFG_RSA_MU, 0); 13692 write_csr(dd, MISC_CFG_FW_CTRL, 0); 13693 /* MISC_STS_8051_DIGEST read-only */ 13694 /* MISC_STS_SBM_DIGEST read-only */ 13695 /* MISC_STS_PCIE_DIGEST read-only */ 13696 /* MISC_STS_FAB_DIGEST read-only */ 13697 /* MISC_ERR_STATUS read-only */ 13698 write_csr(dd, MISC_ERR_MASK, 0); 13699 write_csr(dd, MISC_ERR_CLEAR, ~0ull); 13700 /* MISC_ERR_FORCE leave alone */ 13701 } 13702 13703 /* set TXE CSRs to chip reset defaults */ 13704 static void reset_txe_csrs(struct hfi1_devdata *dd) 13705 { 13706 int i; 13707 13708 /* 13709 * TXE Kernel CSRs 13710 */ 13711 write_csr(dd, SEND_CTRL, 0); 13712 __cm_reset(dd, 0); /* reset CM internal state */ 13713 /* SEND_CONTEXTS read-only */ 13714 /* SEND_DMA_ENGINES read-only */ 13715 /* SEND_PIO_MEM_SIZE read-only */ 13716 /* SEND_DMA_MEM_SIZE read-only */ 13717 write_csr(dd, SEND_HIGH_PRIORITY_LIMIT, 0); 13718 pio_reset_all(dd); /* SEND_PIO_INIT_CTXT */ 13719 /* SEND_PIO_ERR_STATUS read-only */ 13720 write_csr(dd, SEND_PIO_ERR_MASK, 0); 13721 write_csr(dd, SEND_PIO_ERR_CLEAR, ~0ull); 13722 /* SEND_PIO_ERR_FORCE leave alone */ 13723 /* SEND_DMA_ERR_STATUS read-only */ 13724 write_csr(dd, SEND_DMA_ERR_MASK, 0); 13725 write_csr(dd, SEND_DMA_ERR_CLEAR, ~0ull); 13726 /* SEND_DMA_ERR_FORCE leave alone */ 13727 /* SEND_EGRESS_ERR_STATUS read-only */ 13728 write_csr(dd, SEND_EGRESS_ERR_MASK, 0); 13729 write_csr(dd, SEND_EGRESS_ERR_CLEAR, ~0ull); 13730 /* SEND_EGRESS_ERR_FORCE leave alone */ 13731 write_csr(dd, SEND_BTH_QP, 0); 13732 write_csr(dd, SEND_STATIC_RATE_CONTROL, 0); 13733 write_csr(dd, SEND_SC2VLT0, 0); 13734 write_csr(dd, SEND_SC2VLT1, 0); 13735 write_csr(dd, SEND_SC2VLT2, 0); 13736 write_csr(dd, SEND_SC2VLT3, 0); 13737 write_csr(dd, SEND_LEN_CHECK0, 0); 13738 write_csr(dd, SEND_LEN_CHECK1, 0); 13739 /* SEND_ERR_STATUS read-only */ 13740 write_csr(dd, SEND_ERR_MASK, 0); 13741 write_csr(dd, SEND_ERR_CLEAR, ~0ull); 13742 /* SEND_ERR_FORCE read-only */ 13743 for (i = 0; i < VL_ARB_LOW_PRIO_TABLE_SIZE; i++) 13744 write_csr(dd, SEND_LOW_PRIORITY_LIST + (8 * i), 0); 13745 for (i = 0; i < VL_ARB_HIGH_PRIO_TABLE_SIZE; i++) 13746 write_csr(dd, SEND_HIGH_PRIORITY_LIST + (8 * i), 0); 13747 for (i = 0; i < chip_send_contexts(dd) / NUM_CONTEXTS_PER_SET; i++) 13748 write_csr(dd, SEND_CONTEXT_SET_CTRL + (8 * i), 0); 13749 for (i = 0; i < TXE_NUM_32_BIT_COUNTER; i++) 13750 write_csr(dd, SEND_COUNTER_ARRAY32 + (8 * i), 0); 13751 for (i = 0; i < TXE_NUM_64_BIT_COUNTER; i++) 13752 write_csr(dd, SEND_COUNTER_ARRAY64 + (8 * i), 0); 13753 write_csr(dd, SEND_CM_CTRL, SEND_CM_CTRL_RESETCSR); 13754 write_csr(dd, SEND_CM_GLOBAL_CREDIT, SEND_CM_GLOBAL_CREDIT_RESETCSR); 13755 /* SEND_CM_CREDIT_USED_STATUS read-only */ 13756 write_csr(dd, SEND_CM_TIMER_CTRL, 0); 13757 write_csr(dd, SEND_CM_LOCAL_AU_TABLE0_TO3, 0); 13758 write_csr(dd, SEND_CM_LOCAL_AU_TABLE4_TO7, 0); 13759 write_csr(dd, SEND_CM_REMOTE_AU_TABLE0_TO3, 0); 13760 write_csr(dd, SEND_CM_REMOTE_AU_TABLE4_TO7, 0); 13761 for (i = 0; i < TXE_NUM_DATA_VL; i++) 13762 write_csr(dd, SEND_CM_CREDIT_VL + (8 * i), 0); 13763 write_csr(dd, SEND_CM_CREDIT_VL15, 0); 13764 /* SEND_CM_CREDIT_USED_VL read-only */ 13765 /* SEND_CM_CREDIT_USED_VL15 read-only */ 13766 /* SEND_EGRESS_CTXT_STATUS read-only */ 13767 /* SEND_EGRESS_SEND_DMA_STATUS read-only */ 13768 write_csr(dd, SEND_EGRESS_ERR_INFO, ~0ull); 13769 /* SEND_EGRESS_ERR_INFO read-only */ 13770 /* SEND_EGRESS_ERR_SOURCE read-only */ 13771 13772 /* 13773 * TXE Per-Context CSRs 13774 */ 13775 for (i = 0; i < chip_send_contexts(dd); i++) { 13776 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0); 13777 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_CTRL, 0); 13778 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_RETURN_ADDR, 0); 13779 write_kctxt_csr(dd, i, SEND_CTXT_CREDIT_FORCE, 0); 13780 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, 0); 13781 write_kctxt_csr(dd, i, SEND_CTXT_ERR_CLEAR, ~0ull); 13782 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_ENABLE, 0); 13783 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_VL, 0); 13784 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_JOB_KEY, 0); 13785 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_PARTITION_KEY, 0); 13786 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_SLID, 0); 13787 write_kctxt_csr(dd, i, SEND_CTXT_CHECK_OPCODE, 0); 13788 } 13789 13790 /* 13791 * TXE Per-SDMA CSRs 13792 */ 13793 for (i = 0; i < chip_sdma_engines(dd); i++) { 13794 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0); 13795 /* SEND_DMA_STATUS read-only */ 13796 write_kctxt_csr(dd, i, SEND_DMA_BASE_ADDR, 0); 13797 write_kctxt_csr(dd, i, SEND_DMA_LEN_GEN, 0); 13798 write_kctxt_csr(dd, i, SEND_DMA_TAIL, 0); 13799 /* SEND_DMA_HEAD read-only */ 13800 write_kctxt_csr(dd, i, SEND_DMA_HEAD_ADDR, 0); 13801 write_kctxt_csr(dd, i, SEND_DMA_PRIORITY_THLD, 0); 13802 /* SEND_DMA_IDLE_CNT read-only */ 13803 write_kctxt_csr(dd, i, SEND_DMA_RELOAD_CNT, 0); 13804 write_kctxt_csr(dd, i, SEND_DMA_DESC_CNT, 0); 13805 /* SEND_DMA_DESC_FETCHED_CNT read-only */ 13806 /* SEND_DMA_ENG_ERR_STATUS read-only */ 13807 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, 0); 13808 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_CLEAR, ~0ull); 13809 /* SEND_DMA_ENG_ERR_FORCE leave alone */ 13810 write_kctxt_csr(dd, i, SEND_DMA_CHECK_ENABLE, 0); 13811 write_kctxt_csr(dd, i, SEND_DMA_CHECK_VL, 0); 13812 write_kctxt_csr(dd, i, SEND_DMA_CHECK_JOB_KEY, 0); 13813 write_kctxt_csr(dd, i, SEND_DMA_CHECK_PARTITION_KEY, 0); 13814 write_kctxt_csr(dd, i, SEND_DMA_CHECK_SLID, 0); 13815 write_kctxt_csr(dd, i, SEND_DMA_CHECK_OPCODE, 0); 13816 write_kctxt_csr(dd, i, SEND_DMA_MEMORY, 0); 13817 } 13818 } 13819 13820 /* 13821 * Expect on entry: 13822 * o Packet ingress is disabled, i.e. RcvCtrl.RcvPortEnable == 0 13823 */ 13824 static void init_rbufs(struct hfi1_devdata *dd) 13825 { 13826 u64 reg; 13827 int count; 13828 13829 /* 13830 * Wait for DMA to stop: RxRbufPktPending and RxPktInProgress are 13831 * clear. 13832 */ 13833 count = 0; 13834 while (1) { 13835 reg = read_csr(dd, RCV_STATUS); 13836 if ((reg & (RCV_STATUS_RX_RBUF_PKT_PENDING_SMASK 13837 | RCV_STATUS_RX_PKT_IN_PROGRESS_SMASK)) == 0) 13838 break; 13839 /* 13840 * Give up after 1ms - maximum wait time. 13841 * 13842 * RBuf size is 136KiB. Slowest possible is PCIe Gen1 x1 at 13843 * 250MB/s bandwidth. Lower rate to 66% for overhead to get: 13844 * 136 KB / (66% * 250MB/s) = 844us 13845 */ 13846 if (count++ > 500) { 13847 dd_dev_err(dd, 13848 "%s: in-progress DMA not clearing: RcvStatus 0x%llx, continuing\n", 13849 __func__, reg); 13850 break; 13851 } 13852 udelay(2); /* do not busy-wait the CSR */ 13853 } 13854 13855 /* start the init - expect RcvCtrl to be 0 */ 13856 write_csr(dd, RCV_CTRL, RCV_CTRL_RX_RBUF_INIT_SMASK); 13857 13858 /* 13859 * Read to force the write of Rcvtrl.RxRbufInit. There is a brief 13860 * period after the write before RcvStatus.RxRbufInitDone is valid. 13861 * The delay in the first run through the loop below is sufficient and 13862 * required before the first read of RcvStatus.RxRbufInintDone. 13863 */ 13864 read_csr(dd, RCV_CTRL); 13865 13866 /* wait for the init to finish */ 13867 count = 0; 13868 while (1) { 13869 /* delay is required first time through - see above */ 13870 udelay(2); /* do not busy-wait the CSR */ 13871 reg = read_csr(dd, RCV_STATUS); 13872 if (reg & (RCV_STATUS_RX_RBUF_INIT_DONE_SMASK)) 13873 break; 13874 13875 /* give up after 100us - slowest possible at 33MHz is 73us */ 13876 if (count++ > 50) { 13877 dd_dev_err(dd, 13878 "%s: RcvStatus.RxRbufInit not set, continuing\n", 13879 __func__); 13880 break; 13881 } 13882 } 13883 } 13884 13885 /* set RXE CSRs to chip reset defaults */ 13886 static void reset_rxe_csrs(struct hfi1_devdata *dd) 13887 { 13888 int i, j; 13889 13890 /* 13891 * RXE Kernel CSRs 13892 */ 13893 write_csr(dd, RCV_CTRL, 0); 13894 init_rbufs(dd); 13895 /* RCV_STATUS read-only */ 13896 /* RCV_CONTEXTS read-only */ 13897 /* RCV_ARRAY_CNT read-only */ 13898 /* RCV_BUF_SIZE read-only */ 13899 write_csr(dd, RCV_BTH_QP, 0); 13900 write_csr(dd, RCV_MULTICAST, 0); 13901 write_csr(dd, RCV_BYPASS, 0); 13902 write_csr(dd, RCV_VL15, 0); 13903 /* this is a clear-down */ 13904 write_csr(dd, RCV_ERR_INFO, 13905 RCV_ERR_INFO_RCV_EXCESS_BUFFER_OVERRUN_SMASK); 13906 /* RCV_ERR_STATUS read-only */ 13907 write_csr(dd, RCV_ERR_MASK, 0); 13908 write_csr(dd, RCV_ERR_CLEAR, ~0ull); 13909 /* RCV_ERR_FORCE leave alone */ 13910 for (i = 0; i < 32; i++) 13911 write_csr(dd, RCV_QP_MAP_TABLE + (8 * i), 0); 13912 for (i = 0; i < 4; i++) 13913 write_csr(dd, RCV_PARTITION_KEY + (8 * i), 0); 13914 for (i = 0; i < RXE_NUM_32_BIT_COUNTERS; i++) 13915 write_csr(dd, RCV_COUNTER_ARRAY32 + (8 * i), 0); 13916 for (i = 0; i < RXE_NUM_64_BIT_COUNTERS; i++) 13917 write_csr(dd, RCV_COUNTER_ARRAY64 + (8 * i), 0); 13918 for (i = 0; i < RXE_NUM_RSM_INSTANCES; i++) 13919 clear_rsm_rule(dd, i); 13920 for (i = 0; i < 32; i++) 13921 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), 0); 13922 13923 /* 13924 * RXE Kernel and User Per-Context CSRs 13925 */ 13926 for (i = 0; i < chip_rcv_contexts(dd); i++) { 13927 /* kernel */ 13928 write_kctxt_csr(dd, i, RCV_CTXT_CTRL, 0); 13929 /* RCV_CTXT_STATUS read-only */ 13930 write_kctxt_csr(dd, i, RCV_EGR_CTRL, 0); 13931 write_kctxt_csr(dd, i, RCV_TID_CTRL, 0); 13932 write_kctxt_csr(dd, i, RCV_KEY_CTRL, 0); 13933 write_kctxt_csr(dd, i, RCV_HDR_ADDR, 0); 13934 write_kctxt_csr(dd, i, RCV_HDR_CNT, 0); 13935 write_kctxt_csr(dd, i, RCV_HDR_ENT_SIZE, 0); 13936 write_kctxt_csr(dd, i, RCV_HDR_SIZE, 0); 13937 write_kctxt_csr(dd, i, RCV_HDR_TAIL_ADDR, 0); 13938 write_kctxt_csr(dd, i, RCV_AVAIL_TIME_OUT, 0); 13939 write_kctxt_csr(dd, i, RCV_HDR_OVFL_CNT, 0); 13940 13941 /* user */ 13942 /* RCV_HDR_TAIL read-only */ 13943 write_uctxt_csr(dd, i, RCV_HDR_HEAD, 0); 13944 /* RCV_EGR_INDEX_TAIL read-only */ 13945 write_uctxt_csr(dd, i, RCV_EGR_INDEX_HEAD, 0); 13946 /* RCV_EGR_OFFSET_TAIL read-only */ 13947 for (j = 0; j < RXE_NUM_TID_FLOWS; j++) { 13948 write_uctxt_csr(dd, i, 13949 RCV_TID_FLOW_TABLE + (8 * j), 0); 13950 } 13951 } 13952 } 13953 13954 /* 13955 * Set sc2vl tables. 13956 * 13957 * They power on to zeros, so to avoid send context errors 13958 * they need to be set: 13959 * 13960 * SC 0-7 -> VL 0-7 (respectively) 13961 * SC 15 -> VL 15 13962 * otherwise 13963 * -> VL 0 13964 */ 13965 static void init_sc2vl_tables(struct hfi1_devdata *dd) 13966 { 13967 int i; 13968 /* init per architecture spec, constrained by hardware capability */ 13969 13970 /* HFI maps sent packets */ 13971 write_csr(dd, SEND_SC2VLT0, SC2VL_VAL( 13972 0, 13973 0, 0, 1, 1, 13974 2, 2, 3, 3, 13975 4, 4, 5, 5, 13976 6, 6, 7, 7)); 13977 write_csr(dd, SEND_SC2VLT1, SC2VL_VAL( 13978 1, 13979 8, 0, 9, 0, 13980 10, 0, 11, 0, 13981 12, 0, 13, 0, 13982 14, 0, 15, 15)); 13983 write_csr(dd, SEND_SC2VLT2, SC2VL_VAL( 13984 2, 13985 16, 0, 17, 0, 13986 18, 0, 19, 0, 13987 20, 0, 21, 0, 13988 22, 0, 23, 0)); 13989 write_csr(dd, SEND_SC2VLT3, SC2VL_VAL( 13990 3, 13991 24, 0, 25, 0, 13992 26, 0, 27, 0, 13993 28, 0, 29, 0, 13994 30, 0, 31, 0)); 13995 13996 /* DC maps received packets */ 13997 write_csr(dd, DCC_CFG_SC_VL_TABLE_15_0, DC_SC_VL_VAL( 13998 15_0, 13999 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 14000 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, 13, 0, 14, 0, 15, 15)); 14001 write_csr(dd, DCC_CFG_SC_VL_TABLE_31_16, DC_SC_VL_VAL( 14002 31_16, 14003 16, 0, 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0, 14004 24, 0, 25, 0, 26, 0, 27, 0, 28, 0, 29, 0, 30, 0, 31, 0)); 14005 14006 /* initialize the cached sc2vl values consistently with h/w */ 14007 for (i = 0; i < 32; i++) { 14008 if (i < 8 || i == 15) 14009 *((u8 *)(dd->sc2vl) + i) = (u8)i; 14010 else 14011 *((u8 *)(dd->sc2vl) + i) = 0; 14012 } 14013 } 14014 14015 /* 14016 * Read chip sizes and then reset parts to sane, disabled, values. We cannot 14017 * depend on the chip going through a power-on reset - a driver may be loaded 14018 * and unloaded many times. 14019 * 14020 * Do not write any CSR values to the chip in this routine - there may be 14021 * a reset following the (possible) FLR in this routine. 14022 * 14023 */ 14024 static int init_chip(struct hfi1_devdata *dd) 14025 { 14026 int i; 14027 int ret = 0; 14028 14029 /* 14030 * Put the HFI CSRs in a known state. 14031 * Combine this with a DC reset. 14032 * 14033 * Stop the device from doing anything while we do a 14034 * reset. We know there are no other active users of 14035 * the device since we are now in charge. Turn off 14036 * off all outbound and inbound traffic and make sure 14037 * the device does not generate any interrupts. 14038 */ 14039 14040 /* disable send contexts and SDMA engines */ 14041 write_csr(dd, SEND_CTRL, 0); 14042 for (i = 0; i < chip_send_contexts(dd); i++) 14043 write_kctxt_csr(dd, i, SEND_CTXT_CTRL, 0); 14044 for (i = 0; i < chip_sdma_engines(dd); i++) 14045 write_kctxt_csr(dd, i, SEND_DMA_CTRL, 0); 14046 /* disable port (turn off RXE inbound traffic) and contexts */ 14047 write_csr(dd, RCV_CTRL, 0); 14048 for (i = 0; i < chip_rcv_contexts(dd); i++) 14049 write_csr(dd, RCV_CTXT_CTRL, 0); 14050 /* mask all interrupt sources */ 14051 for (i = 0; i < CCE_NUM_INT_CSRS; i++) 14052 write_csr(dd, CCE_INT_MASK + (8 * i), 0ull); 14053 14054 /* 14055 * DC Reset: do a full DC reset before the register clear. 14056 * A recommended length of time to hold is one CSR read, 14057 * so reread the CceDcCtrl. Then, hold the DC in reset 14058 * across the clear. 14059 */ 14060 write_csr(dd, CCE_DC_CTRL, CCE_DC_CTRL_DC_RESET_SMASK); 14061 (void)read_csr(dd, CCE_DC_CTRL); 14062 14063 if (use_flr) { 14064 /* 14065 * A FLR will reset the SPC core and part of the PCIe. 14066 * The parts that need to be restored have already been 14067 * saved. 14068 */ 14069 dd_dev_info(dd, "Resetting CSRs with FLR\n"); 14070 14071 /* do the FLR, the DC reset will remain */ 14072 pcie_flr(dd->pcidev); 14073 14074 /* restore command and BARs */ 14075 ret = restore_pci_variables(dd); 14076 if (ret) { 14077 dd_dev_err(dd, "%s: Could not restore PCI variables\n", 14078 __func__); 14079 return ret; 14080 } 14081 14082 if (is_ax(dd)) { 14083 dd_dev_info(dd, "Resetting CSRs with FLR\n"); 14084 pcie_flr(dd->pcidev); 14085 ret = restore_pci_variables(dd); 14086 if (ret) { 14087 dd_dev_err(dd, "%s: Could not restore PCI variables\n", 14088 __func__); 14089 return ret; 14090 } 14091 } 14092 } else { 14093 dd_dev_info(dd, "Resetting CSRs with writes\n"); 14094 reset_cce_csrs(dd); 14095 reset_txe_csrs(dd); 14096 reset_rxe_csrs(dd); 14097 reset_misc_csrs(dd); 14098 } 14099 /* clear the DC reset */ 14100 write_csr(dd, CCE_DC_CTRL, 0); 14101 14102 /* Set the LED off */ 14103 setextled(dd, 0); 14104 14105 /* 14106 * Clear the QSFP reset. 14107 * An FLR enforces a 0 on all out pins. The driver does not touch 14108 * ASIC_QSFPn_OUT otherwise. This leaves RESET_N low and 14109 * anything plugged constantly in reset, if it pays attention 14110 * to RESET_N. 14111 * Prime examples of this are optical cables. Set all pins high. 14112 * I2CCLK and I2CDAT will change per direction, and INT_N and 14113 * MODPRS_N are input only and their value is ignored. 14114 */ 14115 write_csr(dd, ASIC_QSFP1_OUT, 0x1f); 14116 write_csr(dd, ASIC_QSFP2_OUT, 0x1f); 14117 init_chip_resources(dd); 14118 return ret; 14119 } 14120 14121 static void init_early_variables(struct hfi1_devdata *dd) 14122 { 14123 int i; 14124 14125 /* assign link credit variables */ 14126 dd->vau = CM_VAU; 14127 dd->link_credits = CM_GLOBAL_CREDITS; 14128 if (is_ax(dd)) 14129 dd->link_credits--; 14130 dd->vcu = cu_to_vcu(hfi1_cu); 14131 /* enough room for 8 MAD packets plus header - 17K */ 14132 dd->vl15_init = (8 * (2048 + 128)) / vau_to_au(dd->vau); 14133 if (dd->vl15_init > dd->link_credits) 14134 dd->vl15_init = dd->link_credits; 14135 14136 write_uninitialized_csrs_and_memories(dd); 14137 14138 if (HFI1_CAP_IS_KSET(PKEY_CHECK)) 14139 for (i = 0; i < dd->num_pports; i++) { 14140 struct hfi1_pportdata *ppd = &dd->pport[i]; 14141 14142 set_partition_keys(ppd); 14143 } 14144 init_sc2vl_tables(dd); 14145 } 14146 14147 static void init_kdeth_qp(struct hfi1_devdata *dd) 14148 { 14149 write_csr(dd, SEND_BTH_QP, 14150 (RVT_KDETH_QP_PREFIX & SEND_BTH_QP_KDETH_QP_MASK) << 14151 SEND_BTH_QP_KDETH_QP_SHIFT); 14152 14153 write_csr(dd, RCV_BTH_QP, 14154 (RVT_KDETH_QP_PREFIX & RCV_BTH_QP_KDETH_QP_MASK) << 14155 RCV_BTH_QP_KDETH_QP_SHIFT); 14156 } 14157 14158 /** 14159 * hfi1_get_qp_map - get qp map 14160 * @dd: device data 14161 * @idx: index to read 14162 */ 14163 u8 hfi1_get_qp_map(struct hfi1_devdata *dd, u8 idx) 14164 { 14165 u64 reg = read_csr(dd, RCV_QP_MAP_TABLE + (idx / 8) * 8); 14166 14167 reg >>= (idx % 8) * 8; 14168 return reg; 14169 } 14170 14171 /** 14172 * init_qpmap_table - init qp map 14173 * @dd: device data 14174 * @first_ctxt: first context 14175 * @last_ctxt: first context 14176 * 14177 * This return sets the qpn mapping table that 14178 * is indexed by qpn[8:1]. 14179 * 14180 * The routine will round robin the 256 settings 14181 * from first_ctxt to last_ctxt. 14182 * 14183 * The first/last looks ahead to having specialized 14184 * receive contexts for mgmt and bypass. Normal 14185 * verbs traffic will assumed to be on a range 14186 * of receive contexts. 14187 */ 14188 static void init_qpmap_table(struct hfi1_devdata *dd, 14189 u32 first_ctxt, 14190 u32 last_ctxt) 14191 { 14192 u64 reg = 0; 14193 u64 regno = RCV_QP_MAP_TABLE; 14194 int i; 14195 u64 ctxt = first_ctxt; 14196 14197 for (i = 0; i < 256; i++) { 14198 reg |= ctxt << (8 * (i % 8)); 14199 ctxt++; 14200 if (ctxt > last_ctxt) 14201 ctxt = first_ctxt; 14202 if (i % 8 == 7) { 14203 write_csr(dd, regno, reg); 14204 reg = 0; 14205 regno += 8; 14206 } 14207 } 14208 14209 add_rcvctrl(dd, RCV_CTRL_RCV_QP_MAP_ENABLE_SMASK 14210 | RCV_CTRL_RCV_BYPASS_ENABLE_SMASK); 14211 } 14212 14213 struct rsm_map_table { 14214 u64 map[NUM_MAP_REGS]; 14215 unsigned int used; 14216 }; 14217 14218 struct rsm_rule_data { 14219 u8 offset; 14220 u8 pkt_type; 14221 u32 field1_off; 14222 u32 field2_off; 14223 u32 index1_off; 14224 u32 index1_width; 14225 u32 index2_off; 14226 u32 index2_width; 14227 u32 mask1; 14228 u32 value1; 14229 u32 mask2; 14230 u32 value2; 14231 }; 14232 14233 /* 14234 * Return an initialized RMT map table for users to fill in. OK if it 14235 * returns NULL, indicating no table. 14236 */ 14237 static struct rsm_map_table *alloc_rsm_map_table(struct hfi1_devdata *dd) 14238 { 14239 struct rsm_map_table *rmt; 14240 u8 rxcontext = is_ax(dd) ? 0 : 0xff; /* 0 is default if a0 ver. */ 14241 14242 rmt = kmalloc(sizeof(*rmt), GFP_KERNEL); 14243 if (rmt) { 14244 memset(rmt->map, rxcontext, sizeof(rmt->map)); 14245 rmt->used = 0; 14246 } 14247 14248 return rmt; 14249 } 14250 14251 /* 14252 * Write the final RMT map table to the chip and free the table. OK if 14253 * table is NULL. 14254 */ 14255 static void complete_rsm_map_table(struct hfi1_devdata *dd, 14256 struct rsm_map_table *rmt) 14257 { 14258 int i; 14259 14260 if (rmt) { 14261 /* write table to chip */ 14262 for (i = 0; i < NUM_MAP_REGS; i++) 14263 write_csr(dd, RCV_RSM_MAP_TABLE + (8 * i), rmt->map[i]); 14264 14265 /* enable RSM */ 14266 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK); 14267 } 14268 } 14269 14270 /* Is a receive side mapping rule */ 14271 static bool has_rsm_rule(struct hfi1_devdata *dd, u8 rule_index) 14272 { 14273 return read_csr(dd, RCV_RSM_CFG + (8 * rule_index)) != 0; 14274 } 14275 14276 /* 14277 * Add a receive side mapping rule. 14278 */ 14279 static void add_rsm_rule(struct hfi1_devdata *dd, u8 rule_index, 14280 struct rsm_rule_data *rrd) 14281 { 14282 write_csr(dd, RCV_RSM_CFG + (8 * rule_index), 14283 (u64)rrd->offset << RCV_RSM_CFG_OFFSET_SHIFT | 14284 1ull << rule_index | /* enable bit */ 14285 (u64)rrd->pkt_type << RCV_RSM_CFG_PACKET_TYPE_SHIFT); 14286 write_csr(dd, RCV_RSM_SELECT + (8 * rule_index), 14287 (u64)rrd->field1_off << RCV_RSM_SELECT_FIELD1_OFFSET_SHIFT | 14288 (u64)rrd->field2_off << RCV_RSM_SELECT_FIELD2_OFFSET_SHIFT | 14289 (u64)rrd->index1_off << RCV_RSM_SELECT_INDEX1_OFFSET_SHIFT | 14290 (u64)rrd->index1_width << RCV_RSM_SELECT_INDEX1_WIDTH_SHIFT | 14291 (u64)rrd->index2_off << RCV_RSM_SELECT_INDEX2_OFFSET_SHIFT | 14292 (u64)rrd->index2_width << RCV_RSM_SELECT_INDEX2_WIDTH_SHIFT); 14293 write_csr(dd, RCV_RSM_MATCH + (8 * rule_index), 14294 (u64)rrd->mask1 << RCV_RSM_MATCH_MASK1_SHIFT | 14295 (u64)rrd->value1 << RCV_RSM_MATCH_VALUE1_SHIFT | 14296 (u64)rrd->mask2 << RCV_RSM_MATCH_MASK2_SHIFT | 14297 (u64)rrd->value2 << RCV_RSM_MATCH_VALUE2_SHIFT); 14298 } 14299 14300 /* 14301 * Clear a receive side mapping rule. 14302 */ 14303 static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index) 14304 { 14305 write_csr(dd, RCV_RSM_CFG + (8 * rule_index), 0); 14306 write_csr(dd, RCV_RSM_SELECT + (8 * rule_index), 0); 14307 write_csr(dd, RCV_RSM_MATCH + (8 * rule_index), 0); 14308 } 14309 14310 /* return the number of RSM map table entries that will be used for QOS */ 14311 static int qos_rmt_entries(unsigned int n_krcv_queues, unsigned int *mp, 14312 unsigned int *np) 14313 { 14314 int i; 14315 unsigned int m, n; 14316 uint max_by_vl = 0; 14317 14318 /* is QOS active at all? */ 14319 if (n_krcv_queues < MIN_KERNEL_KCTXTS || 14320 num_vls == 1 || 14321 krcvqsset <= 1) 14322 goto no_qos; 14323 14324 /* determine bits for qpn */ 14325 for (i = 0; i < min_t(unsigned int, num_vls, krcvqsset); i++) 14326 if (krcvqs[i] > max_by_vl) 14327 max_by_vl = krcvqs[i]; 14328 if (max_by_vl > 32) 14329 goto no_qos; 14330 m = ilog2(__roundup_pow_of_two(max_by_vl)); 14331 14332 /* determine bits for vl */ 14333 n = ilog2(__roundup_pow_of_two(num_vls)); 14334 14335 /* reject if too much is used */ 14336 if ((m + n) > 7) 14337 goto no_qos; 14338 14339 if (mp) 14340 *mp = m; 14341 if (np) 14342 *np = n; 14343 14344 return 1 << (m + n); 14345 14346 no_qos: 14347 if (mp) 14348 *mp = 0; 14349 if (np) 14350 *np = 0; 14351 return 0; 14352 } 14353 14354 /** 14355 * init_qos - init RX qos 14356 * @dd: device data 14357 * @rmt: RSM map table 14358 * 14359 * This routine initializes Rule 0 and the RSM map table to implement 14360 * quality of service (qos). 14361 * 14362 * If all of the limit tests succeed, qos is applied based on the array 14363 * interpretation of krcvqs where entry 0 is VL0. 14364 * 14365 * The number of vl bits (n) and the number of qpn bits (m) are computed to 14366 * feed both the RSM map table and the single rule. 14367 */ 14368 static void init_qos(struct hfi1_devdata *dd, struct rsm_map_table *rmt) 14369 { 14370 struct rsm_rule_data rrd; 14371 unsigned qpns_per_vl, ctxt, i, qpn, n = 1, m; 14372 unsigned int rmt_entries; 14373 u64 reg; 14374 14375 if (!rmt) 14376 goto bail; 14377 rmt_entries = qos_rmt_entries(dd->n_krcv_queues - 1, &m, &n); 14378 if (rmt_entries == 0) 14379 goto bail; 14380 qpns_per_vl = 1 << m; 14381 14382 /* enough room in the map table? */ 14383 rmt_entries = 1 << (m + n); 14384 if (rmt->used + rmt_entries >= NUM_MAP_ENTRIES) 14385 goto bail; 14386 14387 /* add qos entries to the RSM map table */ 14388 for (i = 0, ctxt = FIRST_KERNEL_KCTXT; i < num_vls; i++) { 14389 unsigned tctxt; 14390 14391 for (qpn = 0, tctxt = ctxt; 14392 krcvqs[i] && qpn < qpns_per_vl; qpn++) { 14393 unsigned idx, regoff, regidx; 14394 14395 /* generate the index the hardware will produce */ 14396 idx = rmt->used + ((qpn << n) ^ i); 14397 regoff = (idx % 8) * 8; 14398 regidx = idx / 8; 14399 /* replace default with context number */ 14400 reg = rmt->map[regidx]; 14401 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK 14402 << regoff); 14403 reg |= (u64)(tctxt++) << regoff; 14404 rmt->map[regidx] = reg; 14405 if (tctxt == ctxt + krcvqs[i]) 14406 tctxt = ctxt; 14407 } 14408 ctxt += krcvqs[i]; 14409 } 14410 14411 rrd.offset = rmt->used; 14412 rrd.pkt_type = 2; 14413 rrd.field1_off = LRH_BTH_MATCH_OFFSET; 14414 rrd.field2_off = LRH_SC_MATCH_OFFSET; 14415 rrd.index1_off = LRH_SC_SELECT_OFFSET; 14416 rrd.index1_width = n; 14417 rrd.index2_off = QPN_SELECT_OFFSET; 14418 rrd.index2_width = m + n; 14419 rrd.mask1 = LRH_BTH_MASK; 14420 rrd.value1 = LRH_BTH_VALUE; 14421 rrd.mask2 = LRH_SC_MASK; 14422 rrd.value2 = LRH_SC_VALUE; 14423 14424 /* add rule 0 */ 14425 add_rsm_rule(dd, RSM_INS_VERBS, &rrd); 14426 14427 /* mark RSM map entries as used */ 14428 rmt->used += rmt_entries; 14429 /* map everything else to the mcast/err/vl15 context */ 14430 init_qpmap_table(dd, HFI1_CTRL_CTXT, HFI1_CTRL_CTXT); 14431 dd->qos_shift = n + 1; 14432 return; 14433 bail: 14434 dd->qos_shift = 1; 14435 init_qpmap_table(dd, FIRST_KERNEL_KCTXT, dd->n_krcv_queues - 1); 14436 } 14437 14438 static void init_fecn_handling(struct hfi1_devdata *dd, 14439 struct rsm_map_table *rmt) 14440 { 14441 struct rsm_rule_data rrd; 14442 u64 reg; 14443 int i, idx, regoff, regidx, start; 14444 u8 offset; 14445 u32 total_cnt; 14446 14447 if (HFI1_CAP_IS_KSET(TID_RDMA)) 14448 /* Exclude context 0 */ 14449 start = 1; 14450 else 14451 start = dd->first_dyn_alloc_ctxt; 14452 14453 total_cnt = dd->num_rcv_contexts - start; 14454 14455 /* there needs to be enough room in the map table */ 14456 if (rmt->used + total_cnt >= NUM_MAP_ENTRIES) { 14457 dd_dev_err(dd, "FECN handling disabled - too many contexts allocated\n"); 14458 return; 14459 } 14460 14461 /* 14462 * RSM will extract the destination context as an index into the 14463 * map table. The destination contexts are a sequential block 14464 * in the range start...num_rcv_contexts-1 (inclusive). 14465 * Map entries are accessed as offset + extracted value. Adjust 14466 * the added offset so this sequence can be placed anywhere in 14467 * the table - as long as the entries themselves do not wrap. 14468 * There are only enough bits in offset for the table size, so 14469 * start with that to allow for a "negative" offset. 14470 */ 14471 offset = (u8)(NUM_MAP_ENTRIES + rmt->used - start); 14472 14473 for (i = start, idx = rmt->used; i < dd->num_rcv_contexts; 14474 i++, idx++) { 14475 /* replace with identity mapping */ 14476 regoff = (idx % 8) * 8; 14477 regidx = idx / 8; 14478 reg = rmt->map[regidx]; 14479 reg &= ~(RCV_RSM_MAP_TABLE_RCV_CONTEXT_A_MASK << regoff); 14480 reg |= (u64)i << regoff; 14481 rmt->map[regidx] = reg; 14482 } 14483 14484 /* 14485 * For RSM intercept of Expected FECN packets: 14486 * o packet type 0 - expected 14487 * o match on F (bit 95), using select/match 1, and 14488 * o match on SH (bit 133), using select/match 2. 14489 * 14490 * Use index 1 to extract the 8-bit receive context from DestQP 14491 * (start at bit 64). Use that as the RSM map table index. 14492 */ 14493 rrd.offset = offset; 14494 rrd.pkt_type = 0; 14495 rrd.field1_off = 95; 14496 rrd.field2_off = 133; 14497 rrd.index1_off = 64; 14498 rrd.index1_width = 8; 14499 rrd.index2_off = 0; 14500 rrd.index2_width = 0; 14501 rrd.mask1 = 1; 14502 rrd.value1 = 1; 14503 rrd.mask2 = 1; 14504 rrd.value2 = 1; 14505 14506 /* add rule 1 */ 14507 add_rsm_rule(dd, RSM_INS_FECN, &rrd); 14508 14509 rmt->used += total_cnt; 14510 } 14511 14512 static inline bool hfi1_is_rmt_full(int start, int spare) 14513 { 14514 return (start + spare) > NUM_MAP_ENTRIES; 14515 } 14516 14517 static bool hfi1_netdev_update_rmt(struct hfi1_devdata *dd) 14518 { 14519 u8 i, j; 14520 u8 ctx_id = 0; 14521 u64 reg; 14522 u32 regoff; 14523 int rmt_start = hfi1_netdev_get_free_rmt_idx(dd); 14524 int ctxt_count = hfi1_netdev_ctxt_count(dd); 14525 14526 /* We already have contexts mapped in RMT */ 14527 if (has_rsm_rule(dd, RSM_INS_VNIC) || has_rsm_rule(dd, RSM_INS_AIP)) { 14528 dd_dev_info(dd, "Contexts are already mapped in RMT\n"); 14529 return true; 14530 } 14531 14532 if (hfi1_is_rmt_full(rmt_start, NUM_NETDEV_MAP_ENTRIES)) { 14533 dd_dev_err(dd, "Not enough RMT entries used = %d\n", 14534 rmt_start); 14535 return false; 14536 } 14537 14538 dev_dbg(&(dd)->pcidev->dev, "RMT start = %d, end %d\n", 14539 rmt_start, 14540 rmt_start + NUM_NETDEV_MAP_ENTRIES); 14541 14542 /* Update RSM mapping table, 32 regs, 256 entries - 1 ctx per byte */ 14543 regoff = RCV_RSM_MAP_TABLE + (rmt_start / 8) * 8; 14544 reg = read_csr(dd, regoff); 14545 for (i = 0; i < NUM_NETDEV_MAP_ENTRIES; i++) { 14546 /* Update map register with netdev context */ 14547 j = (rmt_start + i) % 8; 14548 reg &= ~(0xffllu << (j * 8)); 14549 reg |= (u64)hfi1_netdev_get_ctxt(dd, ctx_id++)->ctxt << (j * 8); 14550 /* Wrap up netdev ctx index */ 14551 ctx_id %= ctxt_count; 14552 /* Write back map register */ 14553 if (j == 7 || ((i + 1) == NUM_NETDEV_MAP_ENTRIES)) { 14554 dev_dbg(&(dd)->pcidev->dev, 14555 "RMT[%d] =0x%llx\n", 14556 regoff - RCV_RSM_MAP_TABLE, reg); 14557 14558 write_csr(dd, regoff, reg); 14559 regoff += 8; 14560 if (i < (NUM_NETDEV_MAP_ENTRIES - 1)) 14561 reg = read_csr(dd, regoff); 14562 } 14563 } 14564 14565 return true; 14566 } 14567 14568 static void hfi1_enable_rsm_rule(struct hfi1_devdata *dd, 14569 int rule, struct rsm_rule_data *rrd) 14570 { 14571 if (!hfi1_netdev_update_rmt(dd)) { 14572 dd_dev_err(dd, "Failed to update RMT for RSM%d rule\n", rule); 14573 return; 14574 } 14575 14576 add_rsm_rule(dd, rule, rrd); 14577 add_rcvctrl(dd, RCV_CTRL_RCV_RSM_ENABLE_SMASK); 14578 } 14579 14580 void hfi1_init_aip_rsm(struct hfi1_devdata *dd) 14581 { 14582 /* 14583 * go through with the initialisation only if this rule actually doesn't 14584 * exist yet 14585 */ 14586 if (atomic_fetch_inc(&dd->ipoib_rsm_usr_num) == 0) { 14587 int rmt_start = hfi1_netdev_get_free_rmt_idx(dd); 14588 struct rsm_rule_data rrd = { 14589 .offset = rmt_start, 14590 .pkt_type = IB_PACKET_TYPE, 14591 .field1_off = LRH_BTH_MATCH_OFFSET, 14592 .mask1 = LRH_BTH_MASK, 14593 .value1 = LRH_BTH_VALUE, 14594 .field2_off = BTH_DESTQP_MATCH_OFFSET, 14595 .mask2 = BTH_DESTQP_MASK, 14596 .value2 = BTH_DESTQP_VALUE, 14597 .index1_off = DETH_AIP_SQPN_SELECT_OFFSET + 14598 ilog2(NUM_NETDEV_MAP_ENTRIES), 14599 .index1_width = ilog2(NUM_NETDEV_MAP_ENTRIES), 14600 .index2_off = DETH_AIP_SQPN_SELECT_OFFSET, 14601 .index2_width = ilog2(NUM_NETDEV_MAP_ENTRIES) 14602 }; 14603 14604 hfi1_enable_rsm_rule(dd, RSM_INS_AIP, &rrd); 14605 } 14606 } 14607 14608 /* Initialize RSM for VNIC */ 14609 void hfi1_init_vnic_rsm(struct hfi1_devdata *dd) 14610 { 14611 int rmt_start = hfi1_netdev_get_free_rmt_idx(dd); 14612 struct rsm_rule_data rrd = { 14613 /* Add rule for vnic */ 14614 .offset = rmt_start, 14615 .pkt_type = 4, 14616 /* Match 16B packets */ 14617 .field1_off = L2_TYPE_MATCH_OFFSET, 14618 .mask1 = L2_TYPE_MASK, 14619 .value1 = L2_16B_VALUE, 14620 /* Match ETH L4 packets */ 14621 .field2_off = L4_TYPE_MATCH_OFFSET, 14622 .mask2 = L4_16B_TYPE_MASK, 14623 .value2 = L4_16B_ETH_VALUE, 14624 /* Calc context from veswid and entropy */ 14625 .index1_off = L4_16B_HDR_VESWID_OFFSET, 14626 .index1_width = ilog2(NUM_NETDEV_MAP_ENTRIES), 14627 .index2_off = L2_16B_ENTROPY_OFFSET, 14628 .index2_width = ilog2(NUM_NETDEV_MAP_ENTRIES) 14629 }; 14630 14631 hfi1_enable_rsm_rule(dd, RSM_INS_VNIC, &rrd); 14632 } 14633 14634 void hfi1_deinit_vnic_rsm(struct hfi1_devdata *dd) 14635 { 14636 clear_rsm_rule(dd, RSM_INS_VNIC); 14637 } 14638 14639 void hfi1_deinit_aip_rsm(struct hfi1_devdata *dd) 14640 { 14641 /* only actually clear the rule if it's the last user asking to do so */ 14642 if (atomic_fetch_add_unless(&dd->ipoib_rsm_usr_num, -1, 0) == 1) 14643 clear_rsm_rule(dd, RSM_INS_AIP); 14644 } 14645 14646 static int init_rxe(struct hfi1_devdata *dd) 14647 { 14648 struct rsm_map_table *rmt; 14649 u64 val; 14650 14651 /* enable all receive errors */ 14652 write_csr(dd, RCV_ERR_MASK, ~0ull); 14653 14654 rmt = alloc_rsm_map_table(dd); 14655 if (!rmt) 14656 return -ENOMEM; 14657 14658 /* set up QOS, including the QPN map table */ 14659 init_qos(dd, rmt); 14660 init_fecn_handling(dd, rmt); 14661 complete_rsm_map_table(dd, rmt); 14662 /* record number of used rsm map entries for netdev */ 14663 hfi1_netdev_set_free_rmt_idx(dd, rmt->used); 14664 kfree(rmt); 14665 14666 /* 14667 * make sure RcvCtrl.RcvWcb <= PCIe Device Control 14668 * Register Max_Payload_Size (PCI_EXP_DEVCTL in Linux PCIe config 14669 * space, PciCfgCap2.MaxPayloadSize in HFI). There is only one 14670 * invalid configuration: RcvCtrl.RcvWcb set to its max of 256 and 14671 * Max_PayLoad_Size set to its minimum of 128. 14672 * 14673 * Presently, RcvCtrl.RcvWcb is not modified from its default of 0 14674 * (64 bytes). Max_Payload_Size is possibly modified upward in 14675 * tune_pcie_caps() which is called after this routine. 14676 */ 14677 14678 /* Have 16 bytes (4DW) of bypass header available in header queue */ 14679 val = read_csr(dd, RCV_BYPASS); 14680 val &= ~RCV_BYPASS_HDR_SIZE_SMASK; 14681 val |= ((4ull & RCV_BYPASS_HDR_SIZE_MASK) << 14682 RCV_BYPASS_HDR_SIZE_SHIFT); 14683 write_csr(dd, RCV_BYPASS, val); 14684 return 0; 14685 } 14686 14687 static void init_other(struct hfi1_devdata *dd) 14688 { 14689 /* enable all CCE errors */ 14690 write_csr(dd, CCE_ERR_MASK, ~0ull); 14691 /* enable *some* Misc errors */ 14692 write_csr(dd, MISC_ERR_MASK, DRIVER_MISC_MASK); 14693 /* enable all DC errors, except LCB */ 14694 write_csr(dd, DCC_ERR_FLG_EN, ~0ull); 14695 write_csr(dd, DC_DC8051_ERR_EN, ~0ull); 14696 } 14697 14698 /* 14699 * Fill out the given AU table using the given CU. A CU is defined in terms 14700 * AUs. The table is a an encoding: given the index, how many AUs does that 14701 * represent? 14702 * 14703 * NOTE: Assumes that the register layout is the same for the 14704 * local and remote tables. 14705 */ 14706 static void assign_cm_au_table(struct hfi1_devdata *dd, u32 cu, 14707 u32 csr0to3, u32 csr4to7) 14708 { 14709 write_csr(dd, csr0to3, 14710 0ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE0_SHIFT | 14711 1ull << SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE1_SHIFT | 14712 2ull * cu << 14713 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE2_SHIFT | 14714 4ull * cu << 14715 SEND_CM_LOCAL_AU_TABLE0_TO3_LOCAL_AU_TABLE3_SHIFT); 14716 write_csr(dd, csr4to7, 14717 8ull * cu << 14718 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE4_SHIFT | 14719 16ull * cu << 14720 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE5_SHIFT | 14721 32ull * cu << 14722 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE6_SHIFT | 14723 64ull * cu << 14724 SEND_CM_LOCAL_AU_TABLE4_TO7_LOCAL_AU_TABLE7_SHIFT); 14725 } 14726 14727 static void assign_local_cm_au_table(struct hfi1_devdata *dd, u8 vcu) 14728 { 14729 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_LOCAL_AU_TABLE0_TO3, 14730 SEND_CM_LOCAL_AU_TABLE4_TO7); 14731 } 14732 14733 void assign_remote_cm_au_table(struct hfi1_devdata *dd, u8 vcu) 14734 { 14735 assign_cm_au_table(dd, vcu_to_cu(vcu), SEND_CM_REMOTE_AU_TABLE0_TO3, 14736 SEND_CM_REMOTE_AU_TABLE4_TO7); 14737 } 14738 14739 static void init_txe(struct hfi1_devdata *dd) 14740 { 14741 int i; 14742 14743 /* enable all PIO, SDMA, general, and Egress errors */ 14744 write_csr(dd, SEND_PIO_ERR_MASK, ~0ull); 14745 write_csr(dd, SEND_DMA_ERR_MASK, ~0ull); 14746 write_csr(dd, SEND_ERR_MASK, ~0ull); 14747 write_csr(dd, SEND_EGRESS_ERR_MASK, ~0ull); 14748 14749 /* enable all per-context and per-SDMA engine errors */ 14750 for (i = 0; i < chip_send_contexts(dd); i++) 14751 write_kctxt_csr(dd, i, SEND_CTXT_ERR_MASK, ~0ull); 14752 for (i = 0; i < chip_sdma_engines(dd); i++) 14753 write_kctxt_csr(dd, i, SEND_DMA_ENG_ERR_MASK, ~0ull); 14754 14755 /* set the local CU to AU mapping */ 14756 assign_local_cm_au_table(dd, dd->vcu); 14757 14758 /* 14759 * Set reasonable default for Credit Return Timer 14760 * Don't set on Simulator - causes it to choke. 14761 */ 14762 if (dd->icode != ICODE_FUNCTIONAL_SIMULATOR) 14763 write_csr(dd, SEND_CM_TIMER_CTRL, HFI1_CREDIT_RETURN_RATE); 14764 } 14765 14766 int hfi1_set_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd, 14767 u16 jkey) 14768 { 14769 u8 hw_ctxt; 14770 u64 reg; 14771 14772 if (!rcd || !rcd->sc) 14773 return -EINVAL; 14774 14775 hw_ctxt = rcd->sc->hw_context; 14776 reg = SEND_CTXT_CHECK_JOB_KEY_MASK_SMASK | /* mask is always 1's */ 14777 ((jkey & SEND_CTXT_CHECK_JOB_KEY_VALUE_MASK) << 14778 SEND_CTXT_CHECK_JOB_KEY_VALUE_SHIFT); 14779 /* JOB_KEY_ALLOW_PERMISSIVE is not allowed by default */ 14780 if (HFI1_CAP_KGET_MASK(rcd->flags, ALLOW_PERM_JKEY)) 14781 reg |= SEND_CTXT_CHECK_JOB_KEY_ALLOW_PERMISSIVE_SMASK; 14782 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_JOB_KEY, reg); 14783 /* 14784 * Enable send-side J_KEY integrity check, unless this is A0 h/w 14785 */ 14786 if (!is_ax(dd)) { 14787 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE); 14788 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK; 14789 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg); 14790 } 14791 14792 /* Enable J_KEY check on receive context. */ 14793 reg = RCV_KEY_CTRL_JOB_KEY_ENABLE_SMASK | 14794 ((jkey & RCV_KEY_CTRL_JOB_KEY_VALUE_MASK) << 14795 RCV_KEY_CTRL_JOB_KEY_VALUE_SHIFT); 14796 write_kctxt_csr(dd, rcd->ctxt, RCV_KEY_CTRL, reg); 14797 14798 return 0; 14799 } 14800 14801 int hfi1_clear_ctxt_jkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd) 14802 { 14803 u8 hw_ctxt; 14804 u64 reg; 14805 14806 if (!rcd || !rcd->sc) 14807 return -EINVAL; 14808 14809 hw_ctxt = rcd->sc->hw_context; 14810 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_JOB_KEY, 0); 14811 /* 14812 * Disable send-side J_KEY integrity check, unless this is A0 h/w. 14813 * This check would not have been enabled for A0 h/w, see 14814 * set_ctxt_jkey(). 14815 */ 14816 if (!is_ax(dd)) { 14817 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE); 14818 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_JOB_KEY_SMASK; 14819 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg); 14820 } 14821 /* Turn off the J_KEY on the receive side */ 14822 write_kctxt_csr(dd, rcd->ctxt, RCV_KEY_CTRL, 0); 14823 14824 return 0; 14825 } 14826 14827 int hfi1_set_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *rcd, 14828 u16 pkey) 14829 { 14830 u8 hw_ctxt; 14831 u64 reg; 14832 14833 if (!rcd || !rcd->sc) 14834 return -EINVAL; 14835 14836 hw_ctxt = rcd->sc->hw_context; 14837 reg = ((u64)pkey & SEND_CTXT_CHECK_PARTITION_KEY_VALUE_MASK) << 14838 SEND_CTXT_CHECK_PARTITION_KEY_VALUE_SHIFT; 14839 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_PARTITION_KEY, reg); 14840 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE); 14841 reg |= SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK; 14842 reg &= ~SEND_CTXT_CHECK_ENABLE_DISALLOW_KDETH_PACKETS_SMASK; 14843 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg); 14844 14845 return 0; 14846 } 14847 14848 int hfi1_clear_ctxt_pkey(struct hfi1_devdata *dd, struct hfi1_ctxtdata *ctxt) 14849 { 14850 u8 hw_ctxt; 14851 u64 reg; 14852 14853 if (!ctxt || !ctxt->sc) 14854 return -EINVAL; 14855 14856 hw_ctxt = ctxt->sc->hw_context; 14857 reg = read_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE); 14858 reg &= ~SEND_CTXT_CHECK_ENABLE_CHECK_PARTITION_KEY_SMASK; 14859 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_ENABLE, reg); 14860 write_kctxt_csr(dd, hw_ctxt, SEND_CTXT_CHECK_PARTITION_KEY, 0); 14861 14862 return 0; 14863 } 14864 14865 /* 14866 * Start doing the clean up the chip. Our clean up happens in multiple 14867 * stages and this is just the first. 14868 */ 14869 void hfi1_start_cleanup(struct hfi1_devdata *dd) 14870 { 14871 aspm_exit(dd); 14872 free_cntrs(dd); 14873 free_rcverr(dd); 14874 finish_chip_resources(dd); 14875 } 14876 14877 #define HFI_BASE_GUID(dev) \ 14878 ((dev)->base_guid & ~(1ULL << GUID_HFI_INDEX_SHIFT)) 14879 14880 /* 14881 * Information can be shared between the two HFIs on the same ASIC 14882 * in the same OS. This function finds the peer device and sets 14883 * up a shared structure. 14884 */ 14885 static int init_asic_data(struct hfi1_devdata *dd) 14886 { 14887 unsigned long index; 14888 struct hfi1_devdata *peer; 14889 struct hfi1_asic_data *asic_data; 14890 int ret = 0; 14891 14892 /* pre-allocate the asic structure in case we are the first device */ 14893 asic_data = kzalloc(sizeof(*dd->asic_data), GFP_KERNEL); 14894 if (!asic_data) 14895 return -ENOMEM; 14896 14897 xa_lock_irq(&hfi1_dev_table); 14898 /* Find our peer device */ 14899 xa_for_each(&hfi1_dev_table, index, peer) { 14900 if ((HFI_BASE_GUID(dd) == HFI_BASE_GUID(peer)) && 14901 dd->unit != peer->unit) 14902 break; 14903 } 14904 14905 if (peer) { 14906 /* use already allocated structure */ 14907 dd->asic_data = peer->asic_data; 14908 kfree(asic_data); 14909 } else { 14910 dd->asic_data = asic_data; 14911 mutex_init(&dd->asic_data->asic_resource_mutex); 14912 } 14913 dd->asic_data->dds[dd->hfi1_id] = dd; /* self back-pointer */ 14914 xa_unlock_irq(&hfi1_dev_table); 14915 14916 /* first one through - set up i2c devices */ 14917 if (!peer) 14918 ret = set_up_i2c(dd, dd->asic_data); 14919 14920 return ret; 14921 } 14922 14923 /* 14924 * Set dd->boardname. Use a generic name if a name is not returned from 14925 * EFI variable space. 14926 * 14927 * Return 0 on success, -ENOMEM if space could not be allocated. 14928 */ 14929 static int obtain_boardname(struct hfi1_devdata *dd) 14930 { 14931 /* generic board description */ 14932 const char generic[] = 14933 "Cornelis Omni-Path Host Fabric Interface Adapter 100 Series"; 14934 unsigned long size; 14935 int ret; 14936 14937 ret = read_hfi1_efi_var(dd, "description", &size, 14938 (void **)&dd->boardname); 14939 if (ret) { 14940 dd_dev_info(dd, "Board description not found\n"); 14941 /* use generic description */ 14942 dd->boardname = kstrdup(generic, GFP_KERNEL); 14943 if (!dd->boardname) 14944 return -ENOMEM; 14945 } 14946 return 0; 14947 } 14948 14949 /* 14950 * Check the interrupt registers to make sure that they are mapped correctly. 14951 * It is intended to help user identify any mismapping by VMM when the driver 14952 * is running in a VM. This function should only be called before interrupt 14953 * is set up properly. 14954 * 14955 * Return 0 on success, -EINVAL on failure. 14956 */ 14957 static int check_int_registers(struct hfi1_devdata *dd) 14958 { 14959 u64 reg; 14960 u64 all_bits = ~(u64)0; 14961 u64 mask; 14962 14963 /* Clear CceIntMask[0] to avoid raising any interrupts */ 14964 mask = read_csr(dd, CCE_INT_MASK); 14965 write_csr(dd, CCE_INT_MASK, 0ull); 14966 reg = read_csr(dd, CCE_INT_MASK); 14967 if (reg) 14968 goto err_exit; 14969 14970 /* Clear all interrupt status bits */ 14971 write_csr(dd, CCE_INT_CLEAR, all_bits); 14972 reg = read_csr(dd, CCE_INT_STATUS); 14973 if (reg) 14974 goto err_exit; 14975 14976 /* Set all interrupt status bits */ 14977 write_csr(dd, CCE_INT_FORCE, all_bits); 14978 reg = read_csr(dd, CCE_INT_STATUS); 14979 if (reg != all_bits) 14980 goto err_exit; 14981 14982 /* Restore the interrupt mask */ 14983 write_csr(dd, CCE_INT_CLEAR, all_bits); 14984 write_csr(dd, CCE_INT_MASK, mask); 14985 14986 return 0; 14987 err_exit: 14988 write_csr(dd, CCE_INT_MASK, mask); 14989 dd_dev_err(dd, "Interrupt registers not properly mapped by VMM\n"); 14990 return -EINVAL; 14991 } 14992 14993 /** 14994 * hfi1_init_dd() - Initialize most of the dd structure. 14995 * @dd: the dd device 14996 * 14997 * This is global, and is called directly at init to set up the 14998 * chip-specific function pointers for later use. 14999 */ 15000 int hfi1_init_dd(struct hfi1_devdata *dd) 15001 { 15002 struct pci_dev *pdev = dd->pcidev; 15003 struct hfi1_pportdata *ppd; 15004 u64 reg; 15005 int i, ret; 15006 static const char * const inames[] = { /* implementation names */ 15007 "RTL silicon", 15008 "RTL VCS simulation", 15009 "RTL FPGA emulation", 15010 "Functional simulator" 15011 }; 15012 struct pci_dev *parent = pdev->bus->self; 15013 u32 sdma_engines = chip_sdma_engines(dd); 15014 15015 ppd = dd->pport; 15016 for (i = 0; i < dd->num_pports; i++, ppd++) { 15017 int vl; 15018 /* init common fields */ 15019 hfi1_init_pportdata(pdev, ppd, dd, 0, 1); 15020 /* DC supports 4 link widths */ 15021 ppd->link_width_supported = 15022 OPA_LINK_WIDTH_1X | OPA_LINK_WIDTH_2X | 15023 OPA_LINK_WIDTH_3X | OPA_LINK_WIDTH_4X; 15024 ppd->link_width_downgrade_supported = 15025 ppd->link_width_supported; 15026 /* start out enabling only 4X */ 15027 ppd->link_width_enabled = OPA_LINK_WIDTH_4X; 15028 ppd->link_width_downgrade_enabled = 15029 ppd->link_width_downgrade_supported; 15030 /* link width active is 0 when link is down */ 15031 /* link width downgrade active is 0 when link is down */ 15032 15033 if (num_vls < HFI1_MIN_VLS_SUPPORTED || 15034 num_vls > HFI1_MAX_VLS_SUPPORTED) { 15035 dd_dev_err(dd, "Invalid num_vls %u, using %u VLs\n", 15036 num_vls, HFI1_MAX_VLS_SUPPORTED); 15037 num_vls = HFI1_MAX_VLS_SUPPORTED; 15038 } 15039 ppd->vls_supported = num_vls; 15040 ppd->vls_operational = ppd->vls_supported; 15041 /* Set the default MTU. */ 15042 for (vl = 0; vl < num_vls; vl++) 15043 dd->vld[vl].mtu = hfi1_max_mtu; 15044 dd->vld[15].mtu = MAX_MAD_PACKET; 15045 /* 15046 * Set the initial values to reasonable default, will be set 15047 * for real when link is up. 15048 */ 15049 ppd->overrun_threshold = 0x4; 15050 ppd->phy_error_threshold = 0xf; 15051 ppd->port_crc_mode_enabled = link_crc_mask; 15052 /* initialize supported LTP CRC mode */ 15053 ppd->port_ltp_crc_mode = cap_to_port_ltp(link_crc_mask) << 8; 15054 /* initialize enabled LTP CRC mode */ 15055 ppd->port_ltp_crc_mode |= cap_to_port_ltp(link_crc_mask) << 4; 15056 /* start in offline */ 15057 ppd->host_link_state = HLS_DN_OFFLINE; 15058 init_vl_arb_caches(ppd); 15059 } 15060 15061 /* 15062 * Do remaining PCIe setup and save PCIe values in dd. 15063 * Any error printing is already done by the init code. 15064 * On return, we have the chip mapped. 15065 */ 15066 ret = hfi1_pcie_ddinit(dd, pdev); 15067 if (ret < 0) 15068 goto bail_free; 15069 15070 /* Save PCI space registers to rewrite after device reset */ 15071 ret = save_pci_variables(dd); 15072 if (ret < 0) 15073 goto bail_cleanup; 15074 15075 dd->majrev = (dd->revision >> CCE_REVISION_CHIP_REV_MAJOR_SHIFT) 15076 & CCE_REVISION_CHIP_REV_MAJOR_MASK; 15077 dd->minrev = (dd->revision >> CCE_REVISION_CHIP_REV_MINOR_SHIFT) 15078 & CCE_REVISION_CHIP_REV_MINOR_MASK; 15079 15080 /* 15081 * Check interrupt registers mapping if the driver has no access to 15082 * the upstream component. In this case, it is likely that the driver 15083 * is running in a VM. 15084 */ 15085 if (!parent) { 15086 ret = check_int_registers(dd); 15087 if (ret) 15088 goto bail_cleanup; 15089 } 15090 15091 /* 15092 * obtain the hardware ID - NOT related to unit, which is a 15093 * software enumeration 15094 */ 15095 reg = read_csr(dd, CCE_REVISION2); 15096 dd->hfi1_id = (reg >> CCE_REVISION2_HFI_ID_SHIFT) 15097 & CCE_REVISION2_HFI_ID_MASK; 15098 /* the variable size will remove unwanted bits */ 15099 dd->icode = reg >> CCE_REVISION2_IMPL_CODE_SHIFT; 15100 dd->irev = reg >> CCE_REVISION2_IMPL_REVISION_SHIFT; 15101 dd_dev_info(dd, "Implementation: %s, revision 0x%x\n", 15102 dd->icode < ARRAY_SIZE(inames) ? 15103 inames[dd->icode] : "unknown", (int)dd->irev); 15104 15105 /* speeds the hardware can support */ 15106 dd->pport->link_speed_supported = OPA_LINK_SPEED_25G; 15107 /* speeds allowed to run at */ 15108 dd->pport->link_speed_enabled = dd->pport->link_speed_supported; 15109 /* give a reasonable active value, will be set on link up */ 15110 dd->pport->link_speed_active = OPA_LINK_SPEED_25G; 15111 15112 /* fix up link widths for emulation _p */ 15113 ppd = dd->pport; 15114 if (dd->icode == ICODE_FPGA_EMULATION && is_emulator_p(dd)) { 15115 ppd->link_width_supported = 15116 ppd->link_width_enabled = 15117 ppd->link_width_downgrade_supported = 15118 ppd->link_width_downgrade_enabled = 15119 OPA_LINK_WIDTH_1X; 15120 } 15121 /* insure num_vls isn't larger than number of sdma engines */ 15122 if (HFI1_CAP_IS_KSET(SDMA) && num_vls > sdma_engines) { 15123 dd_dev_err(dd, "num_vls %u too large, using %u VLs\n", 15124 num_vls, sdma_engines); 15125 num_vls = sdma_engines; 15126 ppd->vls_supported = sdma_engines; 15127 ppd->vls_operational = ppd->vls_supported; 15128 } 15129 15130 /* 15131 * Convert the ns parameter to the 64 * cclocks used in the CSR. 15132 * Limit the max if larger than the field holds. If timeout is 15133 * non-zero, then the calculated field will be at least 1. 15134 * 15135 * Must be after icode is set up - the cclock rate depends 15136 * on knowing the hardware being used. 15137 */ 15138 dd->rcv_intr_timeout_csr = ns_to_cclock(dd, rcv_intr_timeout) / 64; 15139 if (dd->rcv_intr_timeout_csr > 15140 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK) 15141 dd->rcv_intr_timeout_csr = 15142 RCV_AVAIL_TIME_OUT_TIME_OUT_RELOAD_MASK; 15143 else if (dd->rcv_intr_timeout_csr == 0 && rcv_intr_timeout) 15144 dd->rcv_intr_timeout_csr = 1; 15145 15146 /* needs to be done before we look for the peer device */ 15147 read_guid(dd); 15148 15149 /* set up shared ASIC data with peer device */ 15150 ret = init_asic_data(dd); 15151 if (ret) 15152 goto bail_cleanup; 15153 15154 /* obtain chip sizes, reset chip CSRs */ 15155 ret = init_chip(dd); 15156 if (ret) 15157 goto bail_cleanup; 15158 15159 /* read in the PCIe link speed information */ 15160 ret = pcie_speeds(dd); 15161 if (ret) 15162 goto bail_cleanup; 15163 15164 /* call before get_platform_config(), after init_chip_resources() */ 15165 ret = eprom_init(dd); 15166 if (ret) 15167 goto bail_free_rcverr; 15168 15169 /* Needs to be called before hfi1_firmware_init */ 15170 get_platform_config(dd); 15171 15172 /* read in firmware */ 15173 ret = hfi1_firmware_init(dd); 15174 if (ret) 15175 goto bail_cleanup; 15176 15177 /* 15178 * In general, the PCIe Gen3 transition must occur after the 15179 * chip has been idled (so it won't initiate any PCIe transactions 15180 * e.g. an interrupt) and before the driver changes any registers 15181 * (the transition will reset the registers). 15182 * 15183 * In particular, place this call after: 15184 * - init_chip() - the chip will not initiate any PCIe transactions 15185 * - pcie_speeds() - reads the current link speed 15186 * - hfi1_firmware_init() - the needed firmware is ready to be 15187 * downloaded 15188 */ 15189 ret = do_pcie_gen3_transition(dd); 15190 if (ret) 15191 goto bail_cleanup; 15192 15193 /* 15194 * This should probably occur in hfi1_pcie_init(), but historically 15195 * occurs after the do_pcie_gen3_transition() code. 15196 */ 15197 tune_pcie_caps(dd); 15198 15199 /* start setting dd values and adjusting CSRs */ 15200 init_early_variables(dd); 15201 15202 parse_platform_config(dd); 15203 15204 ret = obtain_boardname(dd); 15205 if (ret) 15206 goto bail_cleanup; 15207 15208 snprintf(dd->boardversion, BOARD_VERS_MAX, 15209 "ChipABI %u.%u, ChipRev %u.%u, SW Compat %llu\n", 15210 HFI1_CHIP_VERS_MAJ, HFI1_CHIP_VERS_MIN, 15211 (u32)dd->majrev, 15212 (u32)dd->minrev, 15213 (dd->revision >> CCE_REVISION_SW_SHIFT) 15214 & CCE_REVISION_SW_MASK); 15215 15216 /* alloc VNIC/AIP rx data */ 15217 ret = hfi1_alloc_rx(dd); 15218 if (ret) 15219 goto bail_cleanup; 15220 15221 ret = set_up_context_variables(dd); 15222 if (ret) 15223 goto bail_cleanup; 15224 15225 /* set initial RXE CSRs */ 15226 ret = init_rxe(dd); 15227 if (ret) 15228 goto bail_cleanup; 15229 15230 /* set initial TXE CSRs */ 15231 init_txe(dd); 15232 /* set initial non-RXE, non-TXE CSRs */ 15233 init_other(dd); 15234 /* set up KDETH QP prefix in both RX and TX CSRs */ 15235 init_kdeth_qp(dd); 15236 15237 ret = hfi1_dev_affinity_init(dd); 15238 if (ret) 15239 goto bail_cleanup; 15240 15241 /* send contexts must be set up before receive contexts */ 15242 ret = init_send_contexts(dd); 15243 if (ret) 15244 goto bail_cleanup; 15245 15246 ret = hfi1_create_kctxts(dd); 15247 if (ret) 15248 goto bail_cleanup; 15249 15250 /* 15251 * Initialize aspm, to be done after gen3 transition and setting up 15252 * contexts and before enabling interrupts 15253 */ 15254 aspm_init(dd); 15255 15256 ret = init_pervl_scs(dd); 15257 if (ret) 15258 goto bail_cleanup; 15259 15260 /* sdma init */ 15261 for (i = 0; i < dd->num_pports; ++i) { 15262 ret = sdma_init(dd, i); 15263 if (ret) 15264 goto bail_cleanup; 15265 } 15266 15267 /* use contexts created by hfi1_create_kctxts */ 15268 ret = set_up_interrupts(dd); 15269 if (ret) 15270 goto bail_cleanup; 15271 15272 ret = hfi1_comp_vectors_set_up(dd); 15273 if (ret) 15274 goto bail_clear_intr; 15275 15276 /* set up LCB access - must be after set_up_interrupts() */ 15277 init_lcb_access(dd); 15278 15279 /* 15280 * Serial number is created from the base guid: 15281 * [27:24] = base guid [38:35] 15282 * [23: 0] = base guid [23: 0] 15283 */ 15284 snprintf(dd->serial, SERIAL_MAX, "0x%08llx\n", 15285 (dd->base_guid & 0xFFFFFF) | 15286 ((dd->base_guid >> 11) & 0xF000000)); 15287 15288 dd->oui1 = dd->base_guid >> 56 & 0xFF; 15289 dd->oui2 = dd->base_guid >> 48 & 0xFF; 15290 dd->oui3 = dd->base_guid >> 40 & 0xFF; 15291 15292 ret = load_firmware(dd); /* asymmetric with dispose_firmware() */ 15293 if (ret) 15294 goto bail_clear_intr; 15295 15296 thermal_init(dd); 15297 15298 ret = init_cntrs(dd); 15299 if (ret) 15300 goto bail_clear_intr; 15301 15302 ret = init_rcverr(dd); 15303 if (ret) 15304 goto bail_free_cntrs; 15305 15306 init_completion(&dd->user_comp); 15307 15308 /* The user refcount starts with one to inidicate an active device */ 15309 refcount_set(&dd->user_refcount, 1); 15310 15311 goto bail; 15312 15313 bail_free_rcverr: 15314 free_rcverr(dd); 15315 bail_free_cntrs: 15316 free_cntrs(dd); 15317 bail_clear_intr: 15318 hfi1_comp_vectors_clean_up(dd); 15319 msix_clean_up_interrupts(dd); 15320 bail_cleanup: 15321 hfi1_free_rx(dd); 15322 hfi1_pcie_ddcleanup(dd); 15323 bail_free: 15324 hfi1_free_devdata(dd); 15325 bail: 15326 return ret; 15327 } 15328 15329 static u16 delay_cycles(struct hfi1_pportdata *ppd, u32 desired_egress_rate, 15330 u32 dw_len) 15331 { 15332 u32 delta_cycles; 15333 u32 current_egress_rate = ppd->current_egress_rate; 15334 /* rates here are in units of 10^6 bits/sec */ 15335 15336 if (desired_egress_rate == -1) 15337 return 0; /* shouldn't happen */ 15338 15339 if (desired_egress_rate >= current_egress_rate) 15340 return 0; /* we can't help go faster, only slower */ 15341 15342 delta_cycles = egress_cycles(dw_len * 4, desired_egress_rate) - 15343 egress_cycles(dw_len * 4, current_egress_rate); 15344 15345 return (u16)delta_cycles; 15346 } 15347 15348 /** 15349 * create_pbc - build a pbc for transmission 15350 * @ppd: info of physical Hfi port 15351 * @flags: special case flags or-ed in built pbc 15352 * @srate_mbs: static rate 15353 * @vl: vl 15354 * @dw_len: dword length (header words + data words + pbc words) 15355 * 15356 * Create a PBC with the given flags, rate, VL, and length. 15357 * 15358 * NOTE: The PBC created will not insert any HCRC - all callers but one are 15359 * for verbs, which does not use this PSM feature. The lone other caller 15360 * is for the diagnostic interface which calls this if the user does not 15361 * supply their own PBC. 15362 */ 15363 u64 create_pbc(struct hfi1_pportdata *ppd, u64 flags, int srate_mbs, u32 vl, 15364 u32 dw_len) 15365 { 15366 u64 pbc, delay = 0; 15367 15368 if (unlikely(srate_mbs)) 15369 delay = delay_cycles(ppd, srate_mbs, dw_len); 15370 15371 pbc = flags 15372 | (delay << PBC_STATIC_RATE_CONTROL_COUNT_SHIFT) 15373 | ((u64)PBC_IHCRC_NONE << PBC_INSERT_HCRC_SHIFT) 15374 | (vl & PBC_VL_MASK) << PBC_VL_SHIFT 15375 | (dw_len & PBC_LENGTH_DWS_MASK) 15376 << PBC_LENGTH_DWS_SHIFT; 15377 15378 return pbc; 15379 } 15380 15381 #define SBUS_THERMAL 0x4f 15382 #define SBUS_THERM_MONITOR_MODE 0x1 15383 15384 #define THERM_FAILURE(dev, ret, reason) \ 15385 dd_dev_err((dd), \ 15386 "Thermal sensor initialization failed: %s (%d)\n", \ 15387 (reason), (ret)) 15388 15389 /* 15390 * Initialize the thermal sensor. 15391 * 15392 * After initialization, enable polling of thermal sensor through 15393 * SBus interface. In order for this to work, the SBus Master 15394 * firmware has to be loaded due to the fact that the HW polling 15395 * logic uses SBus interrupts, which are not supported with 15396 * default firmware. Otherwise, no data will be returned through 15397 * the ASIC_STS_THERM CSR. 15398 */ 15399 static int thermal_init(struct hfi1_devdata *dd) 15400 { 15401 int ret = 0; 15402 15403 if (dd->icode != ICODE_RTL_SILICON || 15404 check_chip_resource(dd, CR_THERM_INIT, NULL)) 15405 return ret; 15406 15407 ret = acquire_chip_resource(dd, CR_SBUS, SBUS_TIMEOUT); 15408 if (ret) { 15409 THERM_FAILURE(dd, ret, "Acquire SBus"); 15410 return ret; 15411 } 15412 15413 dd_dev_info(dd, "Initializing thermal sensor\n"); 15414 /* Disable polling of thermal readings */ 15415 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x0); 15416 msleep(100); 15417 /* Thermal Sensor Initialization */ 15418 /* Step 1: Reset the Thermal SBus Receiver */ 15419 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0, 15420 RESET_SBUS_RECEIVER, 0); 15421 if (ret) { 15422 THERM_FAILURE(dd, ret, "Bus Reset"); 15423 goto done; 15424 } 15425 /* Step 2: Set Reset bit in Thermal block */ 15426 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0, 15427 WRITE_SBUS_RECEIVER, 0x1); 15428 if (ret) { 15429 THERM_FAILURE(dd, ret, "Therm Block Reset"); 15430 goto done; 15431 } 15432 /* Step 3: Write clock divider value (100MHz -> 2MHz) */ 15433 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x1, 15434 WRITE_SBUS_RECEIVER, 0x32); 15435 if (ret) { 15436 THERM_FAILURE(dd, ret, "Write Clock Div"); 15437 goto done; 15438 } 15439 /* Step 4: Select temperature mode */ 15440 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x3, 15441 WRITE_SBUS_RECEIVER, 15442 SBUS_THERM_MONITOR_MODE); 15443 if (ret) { 15444 THERM_FAILURE(dd, ret, "Write Mode Sel"); 15445 goto done; 15446 } 15447 /* Step 5: De-assert block reset and start conversion */ 15448 ret = sbus_request_slow(dd, SBUS_THERMAL, 0x0, 15449 WRITE_SBUS_RECEIVER, 0x2); 15450 if (ret) { 15451 THERM_FAILURE(dd, ret, "Write Reset Deassert"); 15452 goto done; 15453 } 15454 /* Step 5.1: Wait for first conversion (21.5ms per spec) */ 15455 msleep(22); 15456 15457 /* Enable polling of thermal readings */ 15458 write_csr(dd, ASIC_CFG_THERM_POLL_EN, 0x1); 15459 15460 /* Set initialized flag */ 15461 ret = acquire_chip_resource(dd, CR_THERM_INIT, 0); 15462 if (ret) 15463 THERM_FAILURE(dd, ret, "Unable to set thermal init flag"); 15464 15465 done: 15466 release_chip_resource(dd, CR_SBUS); 15467 return ret; 15468 } 15469 15470 static void handle_temp_err(struct hfi1_devdata *dd) 15471 { 15472 struct hfi1_pportdata *ppd = &dd->pport[0]; 15473 /* 15474 * Thermal Critical Interrupt 15475 * Put the device into forced freeze mode, take link down to 15476 * offline, and put DC into reset. 15477 */ 15478 dd_dev_emerg(dd, 15479 "Critical temperature reached! Forcing device into freeze mode!\n"); 15480 dd->flags |= HFI1_FORCED_FREEZE; 15481 start_freeze_handling(ppd, FREEZE_SELF | FREEZE_ABORT); 15482 /* 15483 * Shut DC down as much and as quickly as possible. 15484 * 15485 * Step 1: Take the link down to OFFLINE. This will cause the 15486 * 8051 to put the Serdes in reset. However, we don't want to 15487 * go through the entire link state machine since we want to 15488 * shutdown ASAP. Furthermore, this is not a graceful shutdown 15489 * but rather an attempt to save the chip. 15490 * Code below is almost the same as quiet_serdes() but avoids 15491 * all the extra work and the sleeps. 15492 */ 15493 ppd->driver_link_ready = 0; 15494 ppd->link_enabled = 0; 15495 set_physical_link_state(dd, (OPA_LINKDOWN_REASON_SMA_DISABLED << 8) | 15496 PLS_OFFLINE); 15497 /* 15498 * Step 2: Shutdown LCB and 8051 15499 * After shutdown, do not restore DC_CFG_RESET value. 15500 */ 15501 dc_shutdown(dd); 15502 } 15503